Changeset 252024 in webkit
- Timestamp:
- Nov 4, 2019 3:57:34 PM (5 years ago)
- Location:
- trunk
- Files:
-
- 2 added
- 13 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/JSTests/ChangeLog
r251978 r252024 1 2019-11-04 Saam Barati <sbarati@apple.com> 2 3 Don't use memmove/memcpy/memset for memory that can be scanned concurrently 4 https://bugs.webkit.org/show_bug.cgi?id=203228 5 <rdar://problem/56401852> 6 7 Reviewed by Robin Morisset. 8 9 * stress/torn-js-value-concurrent-collector.js: Added. 10 (foo): 11 1 12 2019-11-03 Tadeu Zagallo <tzagallo@apple.com> 2 13 -
trunk/Source/JavaScriptCore/CMakeLists.txt
r251967 r252024 596 596 heap/GCIncomingRefCountedSet.h 597 597 heap/GCLogging.h 598 heap/GCMemoryOperations.h 598 599 heap/GCRequest.h 599 600 heap/GCSegmentedArray.h -
trunk/Source/JavaScriptCore/ChangeLog
r252021 r252024 1 2019-11-04 Saam Barati <sbarati@apple.com> 2 3 Don't use memmove/memcpy/memset for memory that can be scanned concurrently 4 https://bugs.webkit.org/show_bug.cgi?id=203228 5 <rdar://problem/56401852> 6 7 Reviewed by Robin Morisset. 8 9 We had code inside various places of the runtime which would call into system 10 memcpy/memmove/memset when updating a live butterfly. This means that the 11 concurrent collector could be scanning such butterflies while a memcpy/memmove/memset 12 was running. Those functions don't guarantee anything about the minimum 13 alignment of the stores they do. And implementations for them frequently have 14 byte copy loops for low byte copy counts. This lead to us seeing torn JSValues 15 inside the concurrent collector during Array.prototype.splice. This patch 16 introduces new functions for doing memcpy/memmove/memset for data structures 17 which may be concurrently scanned. The loops are written using inline assembly 18 for gcc compatible compilers on 64 bit platforms. The inline assembly 19 ensures we never write to memory using instructions that store fewer 20 than 8 bytes. On other platforms, we just use a volatile pointer to 21 ensure the compiler doesn't turn the loop into a function call or a 22 series of stores which may be smaller than 8 bytes. 23 24 * CMakeLists.txt: 25 * JavaScriptCore.xcodeproj/project.pbxproj: 26 * heap/GCMemoryOperations.h: Added. 27 (JSC::gcSafeMemcpy): 28 (JSC::gcSafeMemmove): 29 (JSC::gcSafeZeroMemory): 30 * heap/Heap.h: 31 * runtime/ArrayConventions.cpp: 32 (JSC::clearArrayMemset): 33 * runtime/ArrayPrototype.cpp: 34 (JSC::copyElements): 35 * runtime/ButterflyInlines.h: 36 (JSC::Butterfly::tryCreate): 37 (JSC::Butterfly::createOrGrowPropertyStorage): 38 (JSC::Butterfly::growArrayRight): 39 (JSC::Butterfly::reallocArrayRightIfPossible): 40 (JSC::Butterfly::resizeArray): 41 (JSC::Butterfly::unshift): 42 (JSC::Butterfly::shift): 43 * runtime/JSArray.cpp: 44 (JSC::JSArray::unshiftCountSlowCase): 45 (JSC::JSArray::appendMemcpy): 46 (JSC::JSArray::fastSlice): 47 (JSC::JSArray::shiftCountWithArrayStorage): 48 (JSC::JSArray::shiftCountWithAnyIndexingType): 49 (JSC::JSArray::unshiftCountWithArrayStorage): 50 * runtime/JSObject.cpp: 51 (JSC::JSObject::constructConvertedArrayStorageWithoutCopyingElements): 52 (JSC::JSObject::convertFromCopyOnWrite): 53 (JSC::JSObject::shiftButterflyAfterFlattening): 54 * runtime/JSObject.h: 55 * runtime/RegExpMatchesArray.h: 56 (JSC::createRegExpMatchesArray): 57 * runtime/Structure.cpp: 58 (JSC::Structure::flattenDictionaryStructure): 59 1 60 2019-11-04 Truitt Savell <tsavell@apple.com> 2 61 -
trunk/Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
r251967 r252024 893 893 521131F71F82BF14007CCEEE /* PolyProtoAccessChain.h in Headers */ = {isa = PBXBuildFile; fileRef = 521131F61F82BF11007CCEEE /* PolyProtoAccessChain.h */; settings = {ATTRIBUTES = (Private, ); }; }; 894 894 521322461ECBCE8200F65615 /* WebAssemblyFunctionBase.h in Headers */ = {isa = PBXBuildFile; fileRef = 521322441ECBCE8200F65615 /* WebAssemblyFunctionBase.h */; }; 895 522927D5235FD0B9005CB169 /* GCMemoryOperations.h in Headers */ = {isa = PBXBuildFile; fileRef = 5272987B235FC8BA005C982C /* GCMemoryOperations.h */; settings = {ATTRIBUTES = (Private, ); }; }; 895 896 523FD88E225566C9003B3DCC /* WebAssemblyFunctionHeapCellType.h in Headers */ = {isa = PBXBuildFile; fileRef = 523FD88C225566C3003B3DCC /* WebAssemblyFunctionHeapCellType.h */; }; 896 897 524E9D7322092B5200A6BEEE /* AirAllocateRegistersAndStackAndGenerateCode.h in Headers */ = {isa = PBXBuildFile; fileRef = 524E9D7222092B4600A6BEEE /* AirAllocateRegistersAndStackAndGenerateCode.h */; }; … … 3552 3553 526AC4B41E977C5D003500E1 /* WasmCodeBlock.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = WasmCodeBlock.cpp; sourceTree = "<group>"; }; 3553 3554 526AC4B51E977C5D003500E1 /* WasmCodeBlock.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WasmCodeBlock.h; sourceTree = "<group>"; }; 3555 5272987B235FC8BA005C982C /* GCMemoryOperations.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GCMemoryOperations.h; sourceTree = "<group>"; }; 3554 3556 527773DD1AAF83AC00BDE7E8 /* RuntimeType.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RuntimeType.cpp; sourceTree = "<group>"; }; 3555 3557 527CE35222555FDD00C6F382 /* JSToWasmICCallee.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = JSToWasmICCallee.cpp; path = js/JSToWasmICCallee.cpp; sourceTree = "<group>"; }; … … 6150 6152 2ADFA26218EF3540004F9FCC /* GCLogging.cpp */, 6151 6153 2AABCDE618EF294200002096 /* GCLogging.h */, 6154 5272987B235FC8BA005C982C /* GCMemoryOperations.h */, 6152 6155 0F97152E1EB28BE900A1645D /* GCRequest.cpp */, 6153 6156 0F97152F1EB28BE900A1645D /* GCRequest.h */, … … 8928 8931 3395C70722555F6D00BDBFAD /* B3EliminateDeadCode.h in Headers */, 8929 8932 0F5BF1711F23A5A10029D91D /* B3EnsureLoopPreHeaders.h in Headers */, 8933 522927D5235FD0B9005CB169 /* GCMemoryOperations.h in Headers */, 8930 8934 5318045C22EAAC4B004A7342 /* B3ExtractValue.h in Headers */, 8931 8935 0F6971EA1D92F42400BA02A5 /* B3FenceValue.h in Headers */, -
trunk/Source/JavaScriptCore/heap/Heap.h
r251967 r252024 29 29 #include "GCConductor.h" 30 30 #include "GCIncomingRefCountedSet.h" 31 #include "GCMemoryOperations.h" 31 32 #include "GCRequest.h" 32 33 #include "HandleSet.h" -
trunk/Source/JavaScriptCore/runtime/ArrayConventions.cpp
r251967 r252024 34 34 void clearArrayMemset(WriteBarrier<Unknown>* base, unsigned count) 35 35 { 36 #if CPU(X86_64) && COMPILER(GCC_COMPATIBLE) 37 uint64_t zero = 0; 38 asm volatile ( 39 "rep stosq\n\t" 40 : "+D"(base), "+c"(count) 41 : "a"(zero) 42 : "memory" 43 ); 44 #else // not CPU(X86_64) 45 memset(base, 0, count * sizeof(WriteBarrier<Unknown>)); 46 #endif // generic CPU 36 gcSafeZeroMemory(base, count * sizeof(WriteBarrier<Unknown>)); 47 37 } 48 38 -
trunk/Source/JavaScriptCore/runtime/ArrayPrototype.cpp
r251967 r252024 1488 1488 1489 1489 template<typename T> 1490 ALWAYS_INLINE void copyElements(T* buffer, unsigned offset, void* source, unsigned sourceSize, IndexingType sourceType)1490 ALWAYS_INLINE void copyElements(T* buffer, unsigned offset, T* source, unsigned sourceSize, IndexingType sourceType) 1491 1491 { 1492 1492 if (sourceType != ArrayWithUndecided) { 1493 memcpy(buffer + offset, source, sizeof(JSValue) * sourceSize);1493 gcSafeMemcpy(buffer + offset, source, sizeof(JSValue) * sourceSize); 1494 1494 return; 1495 1495 } -
trunk/Source/JavaScriptCore/runtime/ButterflyInlines.h
r251967 r252024 105 105 if (hasIndexingHeader) 106 106 *result->indexingHeader() = indexingHeader; 107 memset(result->propertyStorage() - propertyCapacity, 0, propertyCapacity * sizeof(EncodedJSValue));107 gcSafeZeroMemory(result->propertyStorage() - propertyCapacity, propertyCapacity * sizeof(EncodedJSValue)); 108 108 return result; 109 109 } … … 140 140 bool hasIndexingHeader = structure->hasIndexingHeader(intendedOwner); 141 141 Butterfly* result = createUninitialized(vm, intendedOwner, preCapacity, newPropertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes); 142 memcpy(142 gcSafeMemcpy( 143 143 result->propertyStorage() - oldPropertyCapacity, 144 144 oldButterfly->propertyStorage() - oldPropertyCapacity, 145 145 totalSize(0, oldPropertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes)); 146 memset(146 gcSafeZeroMemory( 147 147 result->propertyStorage() - newPropertyCapacity, 148 0,149 148 (newPropertyCapacity - oldPropertyCapacity) * sizeof(EncodedJSValue)); 150 149 return result; … … 180 179 return nullptr; 181 180 // FIXME: This probably shouldn't be a memcpy. 182 memcpy(newBase, theBase, oldSize);181 gcSafeMemcpy(static_cast<JSValue*>(newBase), static_cast<JSValue*>(theBase), oldSize); 183 182 return fromBase(newBase, 0, propertyCapacity); 184 183 } … … 222 221 if (!newBase) 223 222 return nullptr; 224 memcpy(newBase, theBase, oldSize);223 gcSafeMemcpy(static_cast<JSValue*>(newBase), static_cast<JSValue*>(theBase), oldSize); 225 224 return fromBase(newBase, 0, propertyCapacity); 226 225 } … … 239 238 totalSize(0, propertyCapacity, oldHasIndexingHeader, oldIndexingPayloadSizeInBytes), 240 239 totalSize(0, propertyCapacity, newHasIndexingHeader, newIndexingPayloadSizeInBytes)); 241 memcpy(to, from, size);240 gcSafeMemcpy(static_cast<JSValue*>(to), static_cast<JSValue*>(from), size); 242 241 return result; 243 242 } … … 266 265 // we rely on the compiler to recognize the ordering of the pointer arguments (since 267 266 // propertyCapacity is variable and could cause wrap-around as far as the compiler knows). 268 memmove(267 gcSafeMemmove( 269 268 propertyStorage() - numberOfSlots - propertyCapacity, 270 269 propertyStorage() - propertyCapacity, … … 278 277 unsigned propertyCapacity = structure->outOfLineCapacity(); 279 278 // FIXME: See comment in unshift(), above. 280 memmove(279 gcSafeMemmove( 281 280 propertyStorage() - propertyCapacity + numberOfSlots, 282 281 propertyStorage() - propertyCapacity, -
trunk/Source/JavaScriptCore/runtime/JSArray.cpp
r251967 r252024 422 422 if (addToFront) { 423 423 ASSERT(count + usedVectorLength <= newVectorLength); 424 memmove(newButterfly->arrayStorage()->m_vector + count, storage->m_vector, sizeof(JSValue) * usedVectorLength);425 memmove(newButterfly->propertyStorage() - propertySize, butterfly->propertyStorage() - propertySize, sizeof(JSValue) * propertySize + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));424 gcSafeMemmove(newButterfly->arrayStorage()->m_vector + count, storage->m_vector, sizeof(JSValue) * usedVectorLength); 425 gcSafeMemmove(newButterfly->propertyStorage() - propertySize, butterfly->propertyStorage() - propertySize, sizeof(JSValue) * propertySize + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0)); 426 426 427 427 // We don't need to zero the pre-capacity for the concurrent GC because it is not available to use as property storage. 428 memset(newButterfly->base(0, propertyCapacity), 0, (propertyCapacity - propertySize) * sizeof(JSValue));428 gcSafeZeroMemory(static_cast<JSValue*>(newButterfly->base(0, propertyCapacity)), (propertyCapacity - propertySize) * sizeof(JSValue)); 429 429 430 430 if (allocatedNewStorage) { … … 435 435 } 436 436 } else if ((newAllocBase != butterfly->base(structure)) || (preCapacity != storage->m_indexBias)) { 437 memmove(newButterfly->propertyStorage() - propertyCapacity, butterfly->propertyStorage() - propertyCapacity, sizeof(JSValue) * propertyCapacity + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));438 memmove(newButterfly->arrayStorage()->m_vector, storage->m_vector, sizeof(JSValue) * usedVectorLength);437 gcSafeMemmove(newButterfly->propertyStorage() - propertyCapacity, butterfly->propertyStorage() - propertyCapacity, sizeof(JSValue) * propertyCapacity + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0)); 438 gcSafeMemmove(newButterfly->arrayStorage()->m_vector, storage->m_vector, sizeof(JSValue) * usedVectorLength); 439 439 440 440 for (unsigned i = requiredVectorLength; i < newVectorLength; i++) … … 570 570 } 571 571 } else if (type == ArrayWithDouble) 572 memcpy(butterfly()->contiguousDouble().data() + startIndex, otherArray->butterfly()->contiguousDouble().data(), sizeof(JSValue) * otherLength);572 gcSafeMemcpy(butterfly()->contiguousDouble().data() + startIndex, otherArray->butterfly()->contiguousDouble().data(), sizeof(JSValue) * otherLength); 573 573 else { 574 memcpy(butterfly()->contiguous().data() + startIndex, otherArray->butterfly()->contiguous().data(), sizeof(JSValue) * otherLength);574 gcSafeMemcpy(butterfly()->contiguous().data() + startIndex, otherArray->butterfly()->contiguous().data(), sizeof(JSValue) * otherLength); 575 575 vm.heap.writeBarrier(this); 576 576 } … … 790 790 auto& resultButterfly = *resultArray->butterfly(); 791 791 if (arrayType == ArrayWithDouble) 792 memcpy(resultButterfly.contiguousDouble().data(), butterfly()->contiguousDouble().data() + startIndex, sizeof(JSValue) * count);792 gcSafeMemcpy(resultButterfly.contiguousDouble().data(), butterfly()->contiguousDouble().data() + startIndex, sizeof(JSValue) * count); 793 793 else 794 memcpy(resultButterfly.contiguous().data(), butterfly()->contiguous().data() + startIndex, sizeof(JSValue) * count);794 gcSafeMemcpy(resultButterfly.contiguous().data(), butterfly()->contiguous().data() + startIndex, sizeof(JSValue) * count); 795 795 796 796 ASSERT(resultButterfly.publicLength() == count); … … 850 850 if (numElementsBeforeShiftRegion) { 851 851 RELEASE_ASSERT(count + startIndex <= vectorLength); 852 memmove(storage->m_vector + count,852 gcSafeMemmove(storage->m_vector + count, 853 853 storage->m_vector, 854 854 sizeof(JSValue) * startIndex); … … 868 868 // The number of elements before the shift region is greater than or equal to the number 869 869 // of elements after the shift region, so we move the elements after the shift region to the left. 870 memmove(storage->m_vector + startIndex,870 gcSafeMemmove(storage->m_vector + startIndex, 871 871 storage->m_vector + firstIndexAfterShiftRegion, 872 872 sizeof(JSValue) * numElementsAfterShiftRegion); … … 928 928 } 929 929 } else { 930 memmove(butterfly->contiguous().data() + startIndex,930 gcSafeMemmove(butterfly->contiguous().data() + startIndex, 931 931 butterfly->contiguous().data() + startIndex + count, 932 932 sizeof(JSValue) * (end - startIndex)); … … 970 970 } 971 971 } else { 972 memmove(butterfly->contiguousDouble().data() + startIndex,972 gcSafeMemmove(butterfly->contiguousDouble().data() + startIndex, 973 973 butterfly->contiguousDouble().data() + startIndex + count, 974 974 sizeof(JSValue) * (end - startIndex)); … … 1034 1034 if (startIndex) { 1035 1035 if (moveFront) 1036 memmove(vector, vector + count, startIndex * sizeof(JSValue));1036 gcSafeMemmove(vector, vector + count, startIndex * sizeof(JSValue)); 1037 1037 else if (length - startIndex) 1038 memmove(vector + startIndex + count, vector + startIndex, (length - startIndex) * sizeof(JSValue));1038 gcSafeMemmove(vector + startIndex + count, vector + startIndex, (length - startIndex) * sizeof(JSValue)); 1039 1039 } 1040 1040 -
trunk/Source/JavaScriptCore/runtime/JSObject.cpp
r252019 r252024 1208 1208 Butterfly* newButterfly = Butterfly::createUninitialized(vm, this, 0, propertyCapacity, true, ArrayStorage::sizeFor(neededLength)); 1209 1209 1210 memcpy(1211 newButterfly->base(0, propertyCapacity),1212 m_butterfly->base(0, propertyCapacity),1210 gcSafeMemcpy( 1211 static_cast<JSValue*>(newButterfly->base(0, propertyCapacity)), 1212 static_cast<JSValue*>(m_butterfly->base(0, propertyCapacity)), 1213 1213 propertyCapacity * sizeof(EncodedJSValue)); 1214 1214 … … 1501 1501 Butterfly* newButterfly = Butterfly::createUninitialized(vm, this, 0, propertyCapacity, hasIndexingHeader, newVectorLength * sizeof(JSValue)); 1502 1502 1503 memcpy(newButterfly->propertyStorage(), oldButterfly->propertyStorage(), oldButterfly->vectorLength() * sizeof(JSValue) + sizeof(IndexingHeader));1503 gcSafeMemcpy(newButterfly->propertyStorage(), oldButterfly->propertyStorage(), oldButterfly->vectorLength() * sizeof(JSValue) + sizeof(IndexingHeader)); 1504 1504 1505 1505 WTF::storeStoreFence(); … … 3780 3780 void* newBase = newButterfly->base(0, outOfLineCapacityAfter); 3781 3781 3782 memcpy(newBase, currentBase, Butterfly::totalSize(0, outOfLineCapacityAfter, hasIndexingHeader, indexingPayloadSizeInBytes));3782 gcSafeMemcpy(static_cast<JSValue*>(newBase), static_cast<JSValue*>(currentBase), Butterfly::totalSize(0, outOfLineCapacityAfter, hasIndexingHeader, indexingPayloadSizeInBytes)); 3783 3783 3784 3784 setButterfly(vm, newButterfly); -
trunk/Source/JavaScriptCore/runtime/JSObject.h
r251967 r252024 1181 1181 : JSObject(vm, structure, butterfly) 1182 1182 { 1183 memset(inlineStorageUnsafe(), 0, structure->inlineCapacity() * sizeof(EncodedJSValue));1183 gcSafeZeroMemory(inlineStorageUnsafe(), structure->inlineCapacity() * sizeof(EncodedJSValue)); 1184 1184 } 1185 1185 }; -
trunk/Source/JavaScriptCore/runtime/RegExpMatchesArray.h
r251967 r252024 99 99 auto capacity = matchStructure->outOfLineCapacity(); 100 100 auto size = matchStructure->outOfLineSize(); 101 memset(array->butterfly()->base(0, capacity), 0, (capacity - size) * sizeof(JSValue));101 gcSafeZeroMemory(static_cast<JSValue*>(array->butterfly()->base(0, capacity)), (capacity - size) * sizeof(JSValue)); 102 102 }; 103 103 -
trunk/Source/JavaScriptCore/runtime/Structure.cpp
r251967 r252024 778 778 // We need to zero our unused property space; otherwise the GC might see a 779 779 // stale pointer when we add properties in the future. 780 memset(780 gcSafeZeroMemory( 781 781 object->inlineStorageUnsafe() + inlineSize(), 782 0,783 782 (inlineCapacity() - inlineSize()) * sizeof(EncodedJSValue)); 784 783 … … 787 786 void* base = butterfly->base(preCapacity, beforeOutOfLineCapacity); 788 787 void* startOfPropertyStorageSlots = reinterpret_cast<EncodedJSValue*>(base) + preCapacity; 789 memset(startOfPropertyStorageSlots, 0, (beforeOutOfLineCapacity - outOfLineSize()) * sizeof(EncodedJSValue));788 gcSafeZeroMemory(static_cast<JSValue*>(startOfPropertyStorageSlots), (beforeOutOfLineCapacity - outOfLineSize()) * sizeof(EncodedJSValue)); 790 789 checkOffsetConsistency(); 791 790 }
Note: See TracChangeset
for help on using the changeset viewer.