From 517de9c0e9fa7236b70b00765e42f547cbb9a863 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Sun, 2 Aug 2026 17:59:07 -0500 Subject: [PATCH 01/10] Add opt-in thread-local GC backend (tgc) Introduce --DRT-gcopt=gc:tgc with per-thread heaps and local collection that avoids global stop-the-world pauses, for mixed GC/@nogc realtime work. Co-authored-by: Cursor --- changelog/druntime.tgc.dd | 17 + druntime/Makefile | 3 + druntime/mak/COPY | 1 + druntime/mak/DOCS | 1 + druntime/mak/SRCS | 1 + druntime/src/core/internal/gc/impl/tgc/gc.d | 805 ++++++++++++++++++++ druntime/src/core/internal/gc/proxy.d | 5 +- druntime/src/core/internal/parseoptions.d | 1 + druntime/test/gc/Makefile | 3 +- druntime/test/gc/tgc.d | 73 ++ spec/garbage.dd | 5 +- 11 files changed, 912 insertions(+), 3 deletions(-) create mode 100644 changelog/druntime.tgc.dd create mode 100644 druntime/src/core/internal/gc/impl/tgc/gc.d create mode 100644 druntime/test/gc/tgc.d diff --git a/changelog/druntime.tgc.dd b/changelog/druntime.tgc.dd new file mode 100644 index 000000000000..1e7f8e66dea2 --- /dev/null +++ b/changelog/druntime.tgc.dd @@ -0,0 +1,17 @@ +Opt-in thread-local GC (`tgc`) + +A new garbage collector implementation can be selected with +`--DRT-gcopt=gc:tgc`. + +Each attached thread owns a private heap arena. Collection scans and sweeps only +that thread's stack, TLS, and blocks — it does not issue a process-wide +stop-the-world pause. Threads detached via `thread_detachThis()` are never +paused by `tgc` collections. + +Cross-thread sharing of GC pointers is unsupported in this first version except +through ownership transfer that returns memory via a remote free list. Prefer +copy or `immutable` message passing (`std.concurrency`). Partitioned shared +regions are planned as a later phase. + +Informally this collector is sometimes called a "realtime GC" because it avoids +global pauses; the registered name is `tgc` (thread garbage collection). diff --git a/druntime/Makefile b/druntime/Makefile index 95ca8cf7c58d..2e55fb0f4074 100644 --- a/druntime/Makefile +++ b/druntime/Makefile @@ -203,6 +203,9 @@ $(DOC_OUTPUT_DIR)/core_internal_gc_impl_conservative_%.html : import/core/intern $(DOC_OUTPUT_DIR)/core_internal_gc_impl_manual_%.html : import/core/internal/gc/impl/manual/%.d $(DMD) $(DMD) $(DDOCFLAGS) -Df$@ project.ddoc $(DOCFMT) $< +$(DOC_OUTPUT_DIR)/core_internal_gc_impl_tgc_%.html : import/core/internal/gc/impl/tgc/%.d $(DMD) + $(DMD) $(DDOCFLAGS) -Df$@ project.ddoc $(DOCFMT) $< + $(DOC_OUTPUT_DIR)/core_internal_gc_impl_proto_%.html : import/core/internal/gc/impl/proto/%.d $(DMD) $(DMD) $(DDOCFLAGS) -Df$@ project.ddoc $(DOCFMT) $< diff --git a/druntime/mak/COPY b/druntime/mak/COPY index 90dc5e23f599..c96129730769 100644 --- a/druntime/mak/COPY +++ b/druntime/mak/COPY @@ -79,6 +79,7 @@ COPY=\ $(IMPDIR)\core\internal\gc\proxy.d \ $(IMPDIR)\core\internal\gc\impl\conservative\gc.d \ $(IMPDIR)\core\internal\gc\impl\manual\gc.d \ + $(IMPDIR)\core\internal\gc\impl\tgc\gc.d \ $(IMPDIR)\core\internal\gc\impl\proto\gc.d \ \ $(IMPDIR)\core\internal\container\array.d \ diff --git a/druntime/mak/DOCS b/druntime/mak/DOCS index cb5049bf9d0d..c5d36b8aeea1 100644 --- a/druntime/mak/DOCS +++ b/druntime/mak/DOCS @@ -551,6 +551,7 @@ DOCS=\ $(DOCDIR)\core_internal_gc_proxy.html \ $(DOCDIR)\core_internal_gc_impl_conservative_gc.html \ $(DOCDIR)\core_internal_gc_impl_manual_gc.html \ + $(DOCDIR)\core_internal_gc_impl_tgc_gc.html \ $(DOCDIR)\core_internal_gc_impl_proto_gc.html \ \ $(DOCDIR)\rt_aApply.html \ diff --git a/druntime/mak/SRCS b/druntime/mak/SRCS index eff0ab33b62c..8222ea923a83 100644 --- a/druntime/mak/SRCS +++ b/druntime/mak/SRCS @@ -82,6 +82,7 @@ SRCS=\ src\core\internal\gc\proxy.d \ src\core\internal\gc\impl\conservative\gc.d \ src\core\internal\gc\impl\manual\gc.d \ + src\core\internal\gc\impl\tgc\gc.d \ src\core\internal\gc\impl\proto\gc.d \ \ src\core\internal\util\array.d \ diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d new file mode 100644 index 000000000000..f26891c0a44a --- /dev/null +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -0,0 +1,805 @@ +/** + * Opt-in thread-local garbage collector (`tgc`). + * + * Each attached thread owns a private heap arena. Collection scans and sweeps + * only that thread's stack, TLS, roots/ranges, and blocks — it does not call + * `thread_suspendAll`. Detached `@nogc` threads are never paused by `tgc`. + * + * Cross-thread pointer sharing of GC blocks is unsupported in v1 except via + * explicit ownership transfer that returns memory through a remote free list. + * Prefer copy or `immutable` message passing (`std.concurrency`). Partitioned + * shared regions are planned as Phase 2. + * + * Select with `--DRT-gcopt=gc:tgc`. Informal side-name: "realtime GC". + * + * Copyright: Copyright dlang-supplemental contributors 2026. + * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). + */ +module core.internal.gc.impl.tgc.gc; + +import core.gc.gcinterface; + +import core.internal.container.array; +import core.internal.spinlock; + +import core.thread.threadbase : ThreadBase; + +import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; +import core.stdc.string : memcpy, memset; +static import core.memory; + +extern (C) noreturn onOutOfMemoryError(void* pretend_sideffect = null, string file = __FILE__, size_t line = __LINE__) @trusted pure nothrow @nogc; /* dmd @@@BUG11461@@@ */ +extern (C) void rt_finalizeFromGC(void* p, size_t size, uint attr, const TypeInfo typeInfo) nothrow; +extern (C) void* thread_stackTop() nothrow @nogc; +extern (C) void* thread_stackBottom() nothrow @nogc; + +private enum size_t headerAlign = (void*).sizeof; +private enum size_t collectThresholdInit = 256 * 1024; + +private struct BlkHeader +{ + size_t size; /// user-visible allocation size + uint attr; + uint marked; /// non-zero when marked during collect + ThreadHeap* heap; /// owning thread heap + BlkHeader* next; /// intrusive list in owner heap + BlkHeader* prev; +} + +private struct ThreadHeap +{ + BlkHeader* head; + size_t usedBytes; + size_t allocatedTotal; /// bytes allocated on this thread since start + size_t collectThreshold = collectThresholdInit; + size_t numCollections; + + // Remote frees pushed by other threads (ownership transfer). + void** remotePtrs; + size_t remoteLen; + size_t remoteCap; + SpinLock remoteLock; + + bool collecting; + + static ThreadHeap* create() nothrow @nogc + { + auto h = cast(ThreadHeap*) cstdlib.calloc(1, ThreadHeap.sizeof); + if (!h) + onOutOfMemoryError(); + h.collectThreshold = collectThresholdInit; + h.remoteLock = SpinLock(SpinLock.Contention.brief); + return h; + } + + void pushRemote(void* p) nothrow @nogc + { + remoteLock.lock(); + if (remoteLen == remoteCap) + { + size_t ncap = remoteCap ? remoteCap * 2 : 16; + auto np = cast(void**) cstdlib.realloc(remotePtrs, ncap * (void*).sizeof); + if (!np) + { + remoteLock.unlock(); + onOutOfMemoryError(); + } + remotePtrs = np; + remoteCap = ncap; + } + remotePtrs[remoteLen++] = p; + remoteLock.unlock(); + } + + void drainRemote() nothrow @nogc + { + remoteLock.lock(); + size_t n = remoteLen; + void** ptrs = remotePtrs; + remoteLen = 0; + remoteLock.unlock(); + + foreach (i; 0 .. n) + { + auto p = ptrs[i]; + if (!p) + continue; + auto h = headerOf(p); + if (h && h.heap is &this) + unlinkAndFree(h); + } + } + + void link(BlkHeader* h) nothrow @nogc + { + h.prev = null; + h.next = head; + if (head) + head.prev = h; + head = h; + usedBytes += h.size; + } + + void unlink(BlkHeader* h) nothrow @nogc + { + if (h.prev) + h.prev.next = h.next; + else + head = h.next; + if (h.next) + h.next.prev = h.prev; + if (usedBytes >= h.size) + usedBytes -= h.size; + else + usedBytes = 0; + } + + void unlinkAndFree(BlkHeader* h) nothrow @nogc + { + unlink(h); + // GC.free does not run finalizers; call destroy() first if needed. + cstdlib.free(h); + } + + void unlinkAndFreeFinalize(BlkHeader* h) nothrow + { + unlink(h); + if (h.attr & BlkAttr.FINALIZE) + rt_finalizeFromGC(h + 1, h.size, h.attr, null); + cstdlib.free(h); + } + + static BlkHeader* headerOf(void* p) nothrow @nogc + { + if (!p) + return null; + return cast(BlkHeader*) p - 1; + } + + BlkHeader* findBlock(void* p) nothrow @nogc + { + if (!p) + return null; + for (auto h = head; h; h = h.next) + { + void* base = h + 1; + void* end = base + h.size; + if (p >= base && p < end) + return h; + } + return null; + } +} + +// TLS pointer to the calling thread's heap +private static ThreadHeap* tlsHeap; + +private __gshared ThreadHeap*[] allHeaps; +private __gshared size_t allHeapsLen; +private __gshared size_t allHeapsCap; +private __gshared SpinLock heapsLock; + +private void registerHeap(ThreadHeap* h) nothrow @nogc +{ + heapsLock.lock(); + if (allHeapsLen == allHeapsCap) + { + size_t ncap = allHeapsCap ? allHeapsCap * 2 : 8; + auto np = cast(ThreadHeap**) cstdlib.realloc(allHeaps.ptr, ncap * (ThreadHeap*).sizeof); + if (!np) + { + heapsLock.unlock(); + onOutOfMemoryError(); + } + allHeaps = np[0 .. ncap]; + allHeapsCap = ncap; + } + allHeaps[allHeapsLen++] = h; + heapsLock.unlock(); +} + +private void unregisterHeap(ThreadHeap* h) nothrow @nogc +{ + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + if (allHeaps[i] is h) + { + allHeaps[i] = allHeaps[allHeapsLen - 1]; + allHeapsLen--; + break; + } + } + heapsLock.unlock(); +} + +private ThreadHeap* currentHeap() nothrow @nogc +{ + if (tlsHeap) + return tlsHeap; + tlsHeap = ThreadHeap.create(); + registerHeap(tlsHeap); + auto t = ThreadBase.getThis(); + if (t !is null) + t.tlsGCData() = tlsHeap; + return tlsHeap; +} + +// register GC in C constructor +private pragma(crt_constructor) void gc_tgc_ctor() +{ + heapsLock = SpinLock(SpinLock.Contention.brief); + _d_register_tgc_gc(); +} + +extern (C) void _d_register_tgc_gc() +{ + import core.gc.registry; + registerGCFactory("tgc", &initialize, &threadInitHook); +} + +private void threadInitHook(ThreadBase base) nothrow @nogc +{ + // Called before the thread is fully registered; ensure a heap exists. + if (tlsHeap is null) + { + tlsHeap = ThreadHeap.create(); + registerHeap(tlsHeap); + } + base.tlsGCData() = tlsHeap; +} + +private GC initialize() +{ + import core.lifetime : emplace; + + auto gc = cast(ThreadGC) cstdlib.malloc(__traits(classInstanceSize, ThreadGC)); + if (!gc) + onOutOfMemoryError(); + + return emplace(gc); +} + +/** + * Thread-local GC implementation. + * + * Also known informally as a "realtime GC" because collection does not + * globally stop-the-world; the name registered with the runtime is `tgc`. + */ +class ThreadGC : GC +{ + Array!Root roots; + Array!Range ranges; + SpinLock rootsLock; + bool disabled; + size_t profileCollections; + ulong profilePauseTicks; + + this() + { + rootsLock = SpinLock(SpinLock.Contention.brief); + // Ensure the initializing thread has a heap. + cast(void) currentHeap(); + } + + ~this() + { + } + + void enable() + { + disabled = false; + } + + void disable() + { + disabled = true; + } + + void collect() nothrow + { + collectHeap(currentHeap()); + } + + void minimize() nothrow + { + auto h = currentHeap(); + h.drainRemote(); + } + + uint getAttr(void* p) nothrow + { + auto blk = queryBlock(p); + return blk ? blk.attr : 0; + } + + uint setAttr(void* p, uint mask) nothrow + { + auto blk = queryBlock(p); + if (!blk) + return 0; + blk.attr |= mask; + return blk.attr; + } + + uint clrAttr(void* p, uint mask) nothrow + { + auto blk = queryBlock(p); + if (!blk) + return 0; + blk.attr &= ~mask; + return blk.attr; + } + + void* malloc(size_t size, uint bits, const TypeInfo ti) nothrow + { + return alloc(size, bits, false); + } + + BlkInfo qalloc(size_t size, uint bits, const scope TypeInfo ti) nothrow + { + BlkInfo retval; + retval.base = alloc(size, bits, false); + retval.size = size; + retval.attr = bits; + return retval; + } + + void* calloc(size_t size, uint bits, const TypeInfo ti) nothrow + { + return alloc(size, bits, true); + } + + void* realloc(void* p, size_t size, uint bits, const TypeInfo ti) nothrow + { + if (!p) + return alloc(size, bits, false); + if (!size) + { + free(p); + return null; + } + + auto blk = queryBlock(p); + if (!blk) + { + // Unknown pointer — allocate fresh + return alloc(size, bits, false); + } + + auto heap = blk.heap; + if (heap !is currentHeap()) + { + // Cannot realloc foreign block in place; copy into local heap. + auto np = alloc(size, bits ? bits : blk.attr, false); + auto n = size < blk.size ? size : blk.size; + memcpy(np, p, n); + free(p); + return np; + } + + if (size <= blk.size) + { + blk.size = size; + if (bits) + blk.attr = bits; + return p; + } + + auto np = alloc(size, bits ? bits : blk.attr, false); + memcpy(np, p, blk.size); + heap.unlinkAndFree(blk); + return np; + } + + size_t extend(void* p, size_t minsize, size_t maxsize, const TypeInfo ti) nothrow + { + return 0; + } + + size_t reserve(size_t size) nothrow + { + return 0; + } + + void free(void* p) nothrow @nogc + { + if (!p) + return; + auto blk = queryBlock(p); + if (!blk) + return; + auto owner = blk.heap; + auto local = tlsHeap; + if (owner is local || local is null) + { + if (owner) + owner.unlinkAndFree(blk); + return; + } + // Cross-thread free: queue for owning thread (ownership transfer). + owner.pushRemote(p); + } + + void* addrOf(void* p) nothrow @nogc + { + auto blk = queryBlock(p); + return blk ? cast(void*)(blk + 1) : null; + } + + size_t sizeOf(void* p) nothrow @nogc + { + auto blk = queryBlock(p); + return blk ? blk.size : 0; + } + + BlkInfo query(void* p) nothrow + { + auto blk = queryBlock(p); + if (!blk) + return BlkInfo.init; + BlkInfo info; + info.base = cast(void*)(blk + 1); + info.size = blk.size; + info.attr = blk.attr; + return info; + } + + core.memory.GC.Stats stats() @trusted nothrow + { + core.memory.GC.Stats s; + auto h = currentHeap(); + s.usedSize = h.usedBytes; + s.freeSize = 0; + s.allocatedInCurrentThread = h.allocatedTotal; + return s; + } + + core.memory.GC.ProfileStats profileStats() @trusted nothrow + { + core.memory.GC.ProfileStats s; + s.numCollections = profileCollections; + return s; + } + + void addRoot(void* p) nothrow @nogc + { + rootsLock.lock(); + roots.insertBack(Root(p)); + rootsLock.unlock(); + } + + void removeRoot(void* p) nothrow @nogc + { + rootsLock.lock(); + foreach (ref r; roots) + { + if (r is p) + { + r = roots.back; + roots.popBack(); + rootsLock.unlock(); + return; + } + } + rootsLock.unlock(); + assert(false); + } + + @property RootIterator rootIter() return @nogc + { + return &rootsApply; + } + + private int rootsApply(scope int delegate(ref Root) nothrow dg) + { + rootsLock.lock(); + foreach (ref r; roots) + { + if (auto result = dg(r)) + { + rootsLock.unlock(); + return result; + } + } + rootsLock.unlock(); + return 0; + } + + void addRange(void* p, size_t sz, const TypeInfo ti = null) nothrow @nogc + { + rootsLock.lock(); + ranges.insertBack(Range(p, p + sz, cast() ti)); + rootsLock.unlock(); + } + + void removeRange(void* p) nothrow @nogc + { + rootsLock.lock(); + foreach (ref r; ranges) + { + if (r.pbot is p) + { + r = ranges.back; + ranges.popBack(); + rootsLock.unlock(); + return; + } + } + rootsLock.unlock(); + assert(false); + } + + @property RangeIterator rangeIter() return @nogc + { + return &rangesApply; + } + + private int rangesApply(scope int delegate(ref Range) nothrow dg) + { + rootsLock.lock(); + foreach (ref r; ranges) + { + if (auto result = dg(r)) + { + rootsLock.unlock(); + return result; + } + } + rootsLock.unlock(); + return 0; + } + + void runFinalizers(const scope void[] segment) nothrow + { + } + + bool inFinalizer() nothrow + { + auto h = tlsHeap; + return h !is null && h.collecting; + } + + ulong allocatedInCurrentThread() nothrow + { + return currentHeap().allocatedTotal; + } + + void[] getArrayUsed(void* ptr, bool atomic = false) nothrow + { + return null; + } + + bool expandArrayUsed(void[] slice, size_t newUsed, bool atomic = false) nothrow @safe + { + return false; + } + + size_t reserveArrayCapacity(void[] slice, size_t request, bool atomic = false) nothrow @safe + { + return 0; + } + + bool shrinkArrayUsed(void[] slice, size_t existingUsed, bool atomic = false) nothrow + { + return false; + } + + void initThread(ThreadBase t) nothrow @nogc + { + if (tlsHeap is null) + { + tlsHeap = ThreadHeap.create(); + registerHeap(tlsHeap); + } + t.tlsGCData() = tlsHeap; + } + + void cleanupThread(ThreadBase t) nothrow @nogc + { + auto h = cast(ThreadHeap*) t.tlsGCData(); + if (!h) + return; + // Free remaining blocks; do not leave memory owned by a dead thread. + h.drainRemote(); + auto cur = h.head; + while (cur) + { + auto n = cur.next; + h.unlinkAndFree(cur); + cur = n; + } + h.head = null; + unregisterHeap(h); + if (tlsHeap is h) + tlsHeap = null; + t.tlsGCData() = null; + cstdlib.free(h.remotePtrs); + cstdlib.free(h); + } + +private: + + BlkHeader* queryBlock(void* p) nothrow @nogc + { + if (!p) + return null; + // Fast path: local heap + if (tlsHeap) + { + if (auto b = tlsHeap.findBlock(p)) + return b; + } + // Slow path: search registered heaps (for free/query of foreign ptrs) + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + if (auto b = allHeaps[i].findBlock(p)) + { + heapsLock.unlock(); + return b; + } + } + heapsLock.unlock(); + return null; + } + + void* alloc(size_t size, uint bits, bool zero) nothrow + { + auto heap = currentHeap(); + heap.drainRemote(); + + if (!disabled && heap.usedBytes >= heap.collectThreshold) + collectHeap(heap); + + size_t total = BlkHeader.sizeof + size; + // Align user payload + auto raw = zero ? cstdlib.calloc(1, total) : cstdlib.malloc(total); + if (size && raw is null) + onOutOfMemoryError(); + if (!zero) + memset(raw, 0, BlkHeader.sizeof); + + auto h = cast(BlkHeader*) raw; + h.size = size; + h.attr = bits; + h.marked = 0; + h.heap = heap; + h.next = null; + h.prev = null; + heap.link(h); + heap.allocatedTotal += size; + + if (heap.usedBytes > heap.collectThreshold) + heap.collectThreshold = heap.usedBytes * 2; + + return cast(void*)(h + 1); + } + + void collectHeap(ThreadHeap* heap) nothrow + { + if (!heap || heap.collecting || disabled) + return; + + heap.collecting = true; + heap.drainRemote(); + + // Clear marks + for (auto b = heap.head; b; b = b.next) + b.marked = 0; + + // Mark from stack (current thread only — no STW) + void* top; + void* bot; + tryStackBounds(top, bot); + if (top && bot) + { + if (top > bot) + { + auto tmp = top; + top = bot; + bot = tmp; + } + markRange(heap, top, bot); + } + + // Mark from TLS of this thread + markTLS(heap); + + // Mark from global roots/ranges + rootsLock.lock(); + foreach (ref r; roots) + { + if (r.proot) + markPtr(heap, *cast(void**) r.proot); + markPtr(heap, r.proot); + } + foreach (ref r; ranges) + markRange(heap, r.pbot, r.ptop); + rootsLock.unlock(); + + // Fixpoint: scan marked blocks for interior pointers (conservative) + size_t markedCount = 0; + for (auto b = heap.head; b; b = b.next) + if (b.marked) + markedCount++; + size_t prevMarked = size_t.max; + while (prevMarked != markedCount) + { + prevMarked = markedCount; + for (auto b = heap.head; b; b = b.next) + { + if (!b.marked || (b.attr & BlkAttr.NO_SCAN)) + continue; + void* base = b + 1; + markRange(heap, base, base + b.size); + } + markedCount = 0; + for (auto b = heap.head; b; b = b.next) + if (b.marked) + markedCount++; + } + + // Sweep unmarked + auto b = heap.head; + while (b) + { + auto next = b.next; + if (!b.marked) + heap.unlinkAndFreeFinalize(b); + b = next; + } + + heap.numCollections++; + profileCollections++; + heap.collecting = false; + } + + void tryStackBounds(ref void* top, ref void* bot) nothrow + { + top = null; + bot = null; + if (ThreadBase.getThis() is null) + { + // Without attachment, approximate with a local and a small window. + void* approx; + top = ≈ + bot = cast(void*)(&approx) + 4096; + return; + } + top = thread_stackTop(); + bot = thread_stackBottom(); + } + + void markTLS(ThreadHeap* heap) nothrow + { + import rt.sections; + auto rng = initTLSRanges(); + scanTLSRanges(rng, (void* pbeg, void* pend) nothrow { + markRange(heap, pbeg, pend); + }); + } + + void markRange(ThreadHeap* heap, void* pbot, void* ptop) nothrow + { + if (!pbot || !ptop || pbot >= ptop) + return; + auto p = cast(void**) pbot; + auto e = cast(void**) ptop; + // Align + auto addr = cast(size_t) p; + addr = (addr + (void*).sizeof - 1) & ~((void*).sizeof - 1); + p = cast(void**) addr; + for (; p + 1 <= e; ++p) + markPtr(heap, *p); + } + + void markPtr(ThreadHeap* heap, void* p) nothrow + { + if (!p) + return; + auto b = heap.findBlock(p); + if (b) + b.marked = 1; + } +} diff --git a/druntime/src/core/internal/gc/proxy.d b/druntime/src/core/internal/gc/proxy.d index 9b83eb5d3fee..79a1f48534d2 100644 --- a/druntime/src/core/internal/gc/proxy.d +++ b/druntime/src/core/internal/gc/proxy.d @@ -37,6 +37,7 @@ extern (C) // do not import GC modules, they might add a dependency to this whole module void _d_register_conservative_gc(); void _d_register_manual_gc(); + void _d_register_tgc_gc(); // if you don't want to include the default GCs, replace during link by another implementation void* register_default_gcs() @weak @@ -46,7 +47,9 @@ extern (C) // avoid being optimized away auto reg1 = &_d_register_conservative_gc; auto reg2 = &_d_register_manual_gc; - return reg1 < reg2 ? reg1 : reg2; + auto reg3 = &_d_register_tgc_gc; + auto m = reg1 < reg2 ? reg1 : reg2; + return m < reg3 ? m : reg3; } void gc_init() diff --git a/druntime/src/core/internal/parseoptions.d b/druntime/src/core/internal/parseoptions.d index bc4755517ed5..05e014d6eb74 100644 --- a/druntime/src/core/internal/parseoptions.d +++ b/druntime/src/core/internal/parseoptions.d @@ -410,6 +410,7 @@ unittest assert(conf.parseOptions("help profile:1 help")); assert(conf.parseOptions("gc:manual") && conf.gc == "manual"); + assert(conf.parseOptions("gc:tgc") && conf.gc == "tgc"); assert(conf.parseOptions("gc:my-gc~modified") && conf.gc == "my-gc~modified"); assert(conf.parseOptions("gc:conservative help profile:1") && conf.gc == "conservative" && conf.profile == 1); diff --git a/druntime/test/gc/Makefile b/druntime/test/gc/Makefile index 2567ec128f21..d539c40fad38 100644 --- a/druntime/test/gc/Makefile +++ b/druntime/test/gc/Makefile @@ -1,6 +1,6 @@ TESTS:=attributes sentinel printf memstomp invariant logging \ precise precisegc \ - recoverfree collect nocollect + recoverfree collect nocollect tgc ifneq ($(OS),windows) # some .d files are for Posix only @@ -86,6 +86,7 @@ $(ROOT)/precise_concurrent$(DOTEXE): extra_dflags += $(core_ut) -main $(ROOT)/precise_concurrent.done: run_args+="--DRT-gcopt=gc:precise fork:1" $(ROOT)/attributes$(DOTEXE): extra_dflags += $(core_ut) +$(ROOT)/tgc.done: run_args+=--DRT-gcopt=gc:tgc $(ROOT)/forkgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/sigmaskgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/startbackgc$(DOTEXE): extra_dflags += $(core_ut) diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d new file mode 100644 index 000000000000..1a260e422c23 --- /dev/null +++ b/druntime/test/gc/tgc.d @@ -0,0 +1,73 @@ +/** + * Smoke tests for the opt-in thread-local GC (`tgc`). + * + * Run with: --DRT-gcopt=gc:tgc + */ +import core.memory; +import core.thread; +import core.atomic; + +shared size_t otherThreadAllocs; +shared bool otherDone; +shared bool collectDone; + +void worker() +{ + // Allocate on this thread's private heap + foreach (i; 0 .. 100) + { + auto p = new int[64]; + p[0] = cast(int) i; + atomicOp!"+="(otherThreadAllocs, 1); + } + // Keep a live allocation so collect on another thread must not free it + auto keep = new ubyte[1024]; + keep[0] = 42; + + // Wait until main has collected, then verify our data survived + while (!atomicLoad(collectDone)) + Thread.yield(); + + assert(keep[0] == 42); + atomicStore(otherDone, true); +} + +void main() +{ + auto before = GC.profileStats().numCollections; + + // Local allocations + int[] local; + foreach (i; 0 .. 50) + local ~= cast(int) i; + assert(local.length == 50); + + auto t = new Thread(&worker); + t.start(); + + // Wait until the worker has allocated + while (atomicLoad(otherThreadAllocs) < 50) + Thread.yield(); + + // Collect on the main thread only — must not STW-destroy worker heap + GC.collect(); + atomicStore(collectDone, true); + + t.join(); + assert(atomicLoad(otherDone)); + + // Detach smoke: spawn work then detach is documented for @nogc threads; + // here we only verify GC still functions after a normal thread exit. + auto after = GC.profileStats().numCollections; + assert(after >= before); + + // Force more collections via threshold pressure + foreach (i; 0 .. 200) + { + auto junk = new ubyte[4096]; + junk[0] = cast(ubyte) i; + } + GC.collect(); + + assert(local[0] == 0 && local[$ - 1] == 49); +} diff --git a/spec/garbage.dd b/spec/garbage.dd index b9227de658dd..76f0e061cf65 100644 --- a/spec/garbage.dd +++ b/spec/garbage.dd @@ -405,7 +405,10 @@ $(H2 $(LNAME2 gc_config, Configuring the Collector)) $(UL $(LI disable:0|1 - start disabled) $(LI profile:0|1 - enable profiling with summary when terminating program) - $(LI gc:conservative|precise|manual - select collector implementation (default = conservative)) + $(LI gc:conservative|precise|manual|tgc - select collector implementation (default = conservative). + $(TT tgc) is an opt-in thread-local collector: each thread has a private heap and + collection does not globally stop-the-world. Prefer copy or $(TT immutable) message + passing across threads; unrestricted shared GC pointers across heaps are unsupported in v1.) $(LI initReserve:N - initial memory to reserve in MB) $(LI minPoolSize:N - initial and minimum pool size in MB) $(LI maxPoolSize:N - maximum pool size in MB) From cc42554823f2cf984440da16ae9c5139b8c2f316 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Tue, 1 Sep 2026 06:43:06 -0500 Subject: [PATCH 02/10] Reframe tgc as 0.1.0 and add shared-region API scaffold. Target design is partitioned shared regions; private-heap remote free is interim only. Region create/attach/malloc stubs and version export; region collect still TODO. Co-authored-by: Cursor --- changelog/druntime.tgc.dd | 23 ++- druntime/src/core/internal/gc/impl/tgc/gc.d | 215 +++++++++++++++++++- druntime/test/gc/tgc.d | 19 +- spec/garbage.dd | 6 +- 4 files changed, 238 insertions(+), 25 deletions(-) diff --git a/changelog/druntime.tgc.dd b/changelog/druntime.tgc.dd index 1e7f8e66dea2..a5956fd81e41 100644 --- a/changelog/druntime.tgc.dd +++ b/changelog/druntime.tgc.dd @@ -1,17 +1,18 @@ -Opt-in thread-local GC (`tgc`) +Opt-in thread-local GC (`tgc`, 0.1.0 prototype) A new garbage collector implementation can be selected with `--DRT-gcopt=gc:tgc`. -Each attached thread owns a private heap arena. Collection scans and sweeps only -that thread's stack, TLS, and blocks — it does not issue a process-wide -stop-the-world pause. Threads detached via `thread_detachThis()` are never -paused by `tgc` collections. +**0.1.0 scope:** per-thread private heap arenas with local collection (no +process-wide stop-the-world for a private-heap collect). Partitioned shared +regions are the target cross-thread model; this release adds a region API +scaffold (`_d_tgc_region_create`, attach, region malloc) but not region-scoped +collection or selective thread suspend. -Cross-thread sharing of GC pointers is unsupported in this first version except -through ownership transfer that returns memory via a remote free list. Prefer -copy or `immutable` message passing (`std.concurrency`). Partitioned shared -regions are planned as a later phase. +Cross-thread use of private heaps may queue frees via a remote free list +(interim bring-up only). Prefer shared regions for pipelines that intentionally +share memory. -Informally this collector is sometimes called a "realtime GC" because it avoids -global pauses; the registered name is `tgc` (thread garbage collection). +Informally this collector is sometimes called a "realtime GC" because local +collection avoids global pauses; the registered name is `tgc` (thread garbage +collection). Prototype version: 0.1.0. diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index f26891c0a44a..3c02eafb2e9c 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -1,14 +1,16 @@ /** - * Opt-in thread-local garbage collector (`tgc`). + * Opt-in thread-local garbage collector (`tgc`) — **0.1.0 prototype**. * - * Each attached thread owns a private heap arena. Collection scans and sweeps - * only that thread's stack, TLS, roots/ranges, and blocks — it does not call - * `thread_suspendAll`. Detached `@nogc` threads are never paused by `tgc`. + * Target design: per-thread private heaps plus partitioned shared regions + * (many-to-many). Collecting a region pauses only threads attached to that + * region. See dlang-supplemental design notes for the full architecture. * - * Cross-thread pointer sharing of GC blocks is unsupported in v1 except via - * explicit ownership transfer that returns memory through a remote free list. - * Prefer copy or `immutable` message passing (`std.concurrency`). Partitioned - * shared regions are planned as Phase 2. + * **0.1.0 ships:** private per-thread heaps, local collect without global + * `thread_suspendAll`, remote-free stub on private heaps, shared-region API + * scaffold only (region collect / selective suspend not implemented). + * + * Cross-thread sharing on private heaps via remote free is interim, not the + * target model. Prefer attaching workers to a shared region once implemented. * * Select with `--DRT-gcopt=gc:tgc`. Informal side-name: "realtime GC". * @@ -17,6 +19,9 @@ */ module core.internal.gc.impl.tgc.gc; +/// Semantic version of the `tgc` prototype (not druntime release version). +enum tgcVersion = "0.1.0"; + import core.gc.gcinterface; import core.internal.container.array; @@ -171,6 +176,183 @@ private struct ThreadHeap } } +/** + * Partitioned shared region heap (target cross-thread model). + * + * Threads attach explicitly; collecting this region must pause only members + * (selective suspend not implemented in 0.1.0). + */ +private struct SharedRegion +{ + uint id; + ThreadHeap* heap; + ThreadHeap** members; + size_t memberLen; + size_t memberCap; + SpinLock lock; + + static SharedRegion* create(uint id) nothrow @nogc + { + auto r = cast(SharedRegion*) cstdlib.calloc(1, SharedRegion.sizeof); + if (!r) + onOutOfMemoryError(); + r.id = id; + r.heap = ThreadHeap.create(); + r.lock = SpinLock(SpinLock.Contention.brief); + return r; + } + + bool isAttached(ThreadHeap* h) nothrow @nogc + { + foreach (i; 0 .. memberLen) + if (members[i] is h) + return true; + return false; + } + + bool attach(ThreadHeap* h) nothrow @nogc + { + if (!h) + return false; + lock.lock(); + if (isAttached(h)) + { + lock.unlock(); + return true; + } + if (memberLen == memberCap) + { + size_t ncap = memberCap ? memberCap * 2 : 4; + auto np = cast(ThreadHeap**) cstdlib.realloc(members, ncap * (ThreadHeap*).sizeof); + if (!np) + { + lock.unlock(); + onOutOfMemoryError(); + } + members = np; + memberCap = ncap; + } + members[memberLen++] = h; + lock.unlock(); + return true; + } + + bool detach(ThreadHeap* h) nothrow @nogc + { + if (!h) + return false; + lock.lock(); + foreach (i; 0 .. memberLen) + { + if (members[i] is h) + { + members[i] = members[memberLen - 1]; + memberLen--; + lock.unlock(); + return true; + } + } + lock.unlock(); + return false; + } + + void collectRegion() nothrow @nogc + { + // 0.1.0: region-scoped STW requires selective thread suspend (not in druntime). + // Stub: mark/sweep region heap blocks only; member stacks not scanned yet. + heap.drainRemote(); + for (auto b = heap.head; b; b = b.next) + b.marked = 0; + // TODO(>0.1.0): suspend attached threads only, mark from their stacks/TLS, sweep. + } +} + +private __gshared SharedRegion*[] allRegions; +private __gshared uint nextRegionId = 1; +private __gshared SpinLock regionsLock; + +private SharedRegion* findRegion(uint id) nothrow @nogc +{ + regionsLock.lock(); + foreach (r; allRegions) + { + if (r && r.id == id) + { + regionsLock.unlock(); + return r; + } + } + regionsLock.unlock(); + return null; +} + +/// Create a partitioned shared region; returns region id (0 on failure). +extern (C) uint _d_tgc_region_create() nothrow @nogc +{ + regionsLock.lock(); + uint id = nextRegionId++; + auto r = SharedRegion.create(id); + allRegions ~= r; + regionsLock.unlock(); + return id; +} + +/// Attach the calling thread's private heap to `regionId`. +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc +{ + auto r = findRegion(regionId); + if (!r) + return false; + return r.attach(currentHeap()); +} + +/// Detach the calling thread from `regionId`. +extern (C) bool _d_tgc_region_detach(uint regionId) nothrow @nogc +{ + auto r = findRegion(regionId); + if (!r) + return false; + return r.detach(currentHeap()); +} + +/// Allocate in a shared region (attached threads only). Returns null if unknown region or not attached. +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc +{ + auto r = findRegion(regionId); + if (!r) + return null; + auto local = currentHeap(); + r.lock.lock(); + if (!r.isAttached(local)) + { + r.lock.unlock(); + return null; + } + r.lock.unlock(); + + r.heap.drainRemote(); + size_t total = BlkHeader.sizeof + size; + auto raw = cstdlib.malloc(total); + if (size && raw is null) + onOutOfMemoryError(); + memset(raw, 0, BlkHeader.sizeof); + auto h = cast(BlkHeader*) raw; + h.size = size; + h.attr = bits; + h.marked = 0; + h.heap = r.heap; + h.next = null; + h.prev = null; + r.heap.link(h); + r.heap.allocatedTotal += size; + return cast(void*)(h + 1); +} + +extern (C) const(char)* _d_tgc_version() nothrow @nogc +{ + return tgcVersion.ptr; +} + // TLS pointer to the calling thread's heap private static ThreadHeap* tlsHeap; @@ -229,6 +411,7 @@ private ThreadHeap* currentHeap() nothrow @nogc private pragma(crt_constructor) void gc_tgc_ctor() { heapsLock = SpinLock(SpinLock.Contention.brief); + regionsLock = SpinLock(SpinLock.Contention.brief); _d_register_tgc_gc(); } @@ -261,9 +444,9 @@ private GC initialize() } /** - * Thread-local GC implementation. + * Thread-local GC implementation (`tgc` 0.1.0 prototype). * - * Also known informally as a "realtime GC" because collection does not + * Also known informally as a "realtime GC" because local collection does not * globally stop-the-world; the name registered with the runtime is `tgc`. */ class ThreadGC : GC @@ -641,6 +824,18 @@ private: } } heapsLock.unlock(); + regionsLock.lock(); + foreach (r; allRegions) + { + if (!r || !r.heap) + continue; + if (auto b = r.heap.findBlock(p)) + { + regionsLock.unlock(); + return b; + } + } + regionsLock.unlock(); return null; } diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index 1a260e422c23..2d301540fdc5 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -1,5 +1,5 @@ /** - * Smoke tests for the opt-in thread-local GC (`tgc`). + * Smoke tests for the opt-in thread-local GC (`tgc`, 0.1.0 prototype). * * Run with: --DRT-gcopt=gc:tgc */ @@ -7,6 +7,11 @@ import core.memory; import core.thread; import core.atomic; +extern (C) uint _d_tgc_region_create() nothrow @nogc; +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc; +extern (C) const(char)* _d_tgc_version() nothrow @nogc; + shared size_t otherThreadAllocs; shared bool otherDone; shared bool collectDone; @@ -34,6 +39,18 @@ void worker() void main() { + import core.stdc.string : strcmp; + assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.1.0")); + + // Shared region scaffold: create, attach, alloc + auto rid = _d_tgc_region_create(); + assert(rid != 0); + assert(_d_tgc_region_attach(rid)); + auto rp = cast(int*) _d_tgc_region_malloc(rid, int.sizeof, 0); + assert(rp !is null); + *rp = 123; + assert(*rp == 123); + auto before = GC.profileStats().numCollections; // Local allocations diff --git a/spec/garbage.dd b/spec/garbage.dd index 76f0e061cf65..454d63d8f382 100644 --- a/spec/garbage.dd +++ b/spec/garbage.dd @@ -406,9 +406,9 @@ $(H2 $(LNAME2 gc_config, Configuring the Collector)) $(LI disable:0|1 - start disabled) $(LI profile:0|1 - enable profiling with summary when terminating program) $(LI gc:conservative|precise|manual|tgc - select collector implementation (default = conservative). - $(TT tgc) is an opt-in thread-local collector: each thread has a private heap and - collection does not globally stop-the-world. Prefer copy or $(TT immutable) message - passing across threads; unrestricted shared GC pointers across heaps are unsupported in v1.) + $(TT tgc) is an opt-in thread-local collector (0.1.0 prototype): per-thread private heaps, + local collection without global stop-the-world. Target design uses partitioned shared regions + for cross-thread data; region collect is not implemented yet.) $(LI initReserve:N - initial memory to reserve in MB) $(LI minPoolSize:N - initial and minimum pool size in MB) $(LI maxPoolSize:N - maximum pool size in MB) From cad2288d734b5077eb8aa2c44db37bdaf347e108 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Tue, 1 Sep 2026 08:45:49 -0500 Subject: [PATCH 03/10] Bump tgc prototype to 0.1.1 for region API scaffold. 0.1.0 remains the private-heap-only baseline; region exports are patch-level additive until 0.2.0 region collect lands. Co-authored-by: Cursor --- changelog/druntime.tgc.dd | 21 ++++++++++----------- druntime/src/core/internal/gc/impl/tgc/gc.d | 2 +- druntime/test/gc/tgc.d | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/changelog/druntime.tgc.dd b/changelog/druntime.tgc.dd index a5956fd81e41..899c22ba8a09 100644 --- a/changelog/druntime.tgc.dd +++ b/changelog/druntime.tgc.dd @@ -1,18 +1,17 @@ -Opt-in thread-local GC (`tgc`, 0.1.0 prototype) +Opt-in thread-local GC (`tgc`, 0.1.1 prototype) A new garbage collector implementation can be selected with `--DRT-gcopt=gc:tgc`. -**0.1.0 scope:** per-thread private heap arenas with local collection (no -process-wide stop-the-world for a private-heap collect). Partitioned shared -regions are the target cross-thread model; this release adds a region API -scaffold (`_d_tgc_region_create`, attach, region malloc) but not region-scoped -collection or selective thread suspend. +**0.1.0:** per-thread private heap arenas with local collection (no +process-wide stop-the-world for a private-heap collect). + +**0.1.1:** adds shared-region API scaffold (`_d_tgc_region_*`); region +collect and selective thread suspend are not implemented. + +**0.2.0 (planned):** working partitioned shared regions (breaking). Cross-thread use of private heaps may queue frees via a remote free list -(interim bring-up only). Prefer shared regions for pipelines that intentionally -share memory. +(interim bring-up only). -Informally this collector is sometimes called a "realtime GC" because local -collection avoids global pauses; the registered name is `tgc` (thread garbage -collection). Prototype version: 0.1.0. +Prototype version: 0.1.1 (`_d_tgc_version()`). diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index 3c02eafb2e9c..6f376741ba43 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -20,7 +20,7 @@ module core.internal.gc.impl.tgc.gc; /// Semantic version of the `tgc` prototype (not druntime release version). -enum tgcVersion = "0.1.0"; +enum tgcVersion = "0.1.1"; import core.gc.gcinterface; diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index 2d301540fdc5..0d1e7a9f1cef 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -40,7 +40,7 @@ void worker() void main() { import core.stdc.string : strcmp; - assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.1.0")); + assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.1.1")); // Shared region scaffold: create, attach, alloc auto rid = _d_tgc_region_create(); From 170dc9b8e902fcaf065ae4b4f131d96afc1fec81 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Tue, 1 Sep 2026 08:46:05 -0500 Subject: [PATCH 04/10] Fix tgc module header comment for 0.1.1 Co-authored-by: Cursor --- druntime/src/core/internal/gc/impl/tgc/gc.d | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index 6f376741ba43..6781f7a8cb1d 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -1,13 +1,12 @@ /** - * Opt-in thread-local garbage collector (`tgc`) — **0.1.0 prototype**. + * Opt-in thread-local garbage collector (`tgc`) — **0.1.1 prototype**. * * Target design: per-thread private heaps plus partitioned shared regions * (many-to-many). Collecting a region pauses only threads attached to that * region. See dlang-supplemental design notes for the full architecture. * - * **0.1.0 ships:** private per-thread heaps, local collect without global - * `thread_suspendAll`, remote-free stub on private heaps, shared-region API - * scaffold only (region collect / selective suspend not implemented). + * **0.1.0:** private per-thread heaps, local collect, remote-free stub. + * **0.1.1:** adds shared-region API scaffold (no region collect yet). * * Cross-thread sharing on private heaps via remote free is interim, not the * target model. Prefer attaching workers to a shared region once implemented. From 5221cbd026c72bc0eeb5ebfb9f5c96ebebb1cce9 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Tue, 1 Sep 2026 08:53:24 -0500 Subject: [PATCH 05/10] Implement tgc 0.2.0: region collect, tests, and benchmarks. Add selective thread suspend/scan APIs, region collect, heap list locks, array append metadata, tgc_regions and tgc_bench tests, and benchmark runner. Co-authored-by: Cursor --- changelog/druntime.tgc.dd | 22 +- druntime/src/core/gc/config.d | 1 + druntime/src/core/internal/gc/impl/tgc/gc.d | 361 +++++++++++++++----- druntime/src/core/thread/osthread.d | 81 +++++ druntime/src/core/thread/threadbase.d | 62 ++++ druntime/test/gc/Makefile | 4 +- druntime/test/gc/tgc.d | 2 +- druntime/test/gc/tgc_bench.d | 100 ++++++ druntime/test/gc/tgc_regions.d | 65 ++++ tools/tgc-bench/run-benchmarks.ps1 | 31 ++ 10 files changed, 639 insertions(+), 90 deletions(-) create mode 100644 druntime/test/gc/tgc_bench.d create mode 100644 druntime/test/gc/tgc_regions.d create mode 100644 tools/tgc-bench/run-benchmarks.ps1 diff --git a/changelog/druntime.tgc.dd b/changelog/druntime.tgc.dd index 899c22ba8a09..5887aa0ddf1b 100644 --- a/changelog/druntime.tgc.dd +++ b/changelog/druntime.tgc.dd @@ -1,17 +1,15 @@ -Opt-in thread-local GC (`tgc`, 0.1.1 prototype) +Opt-in thread-local GC (`tgc`, 0.2.0 prototype) -A new garbage collector implementation can be selected with -`--DRT-gcopt=gc:tgc`. +Select with `--DRT-gcopt=gc:tgc`. -**0.1.0:** per-thread private heap arenas with local collection (no -process-wide stop-the-world for a private-heap collect). +**0.2.0:** working partitioned shared regions with region-scoped collection +(selective thread suspend via `thread_suspendList`), per-heap list locks, +basic array append metadata, and benchmark/test harness. -**0.1.1:** adds shared-region API scaffold (`_d_tgc_region_*`); region -collect and selective thread suspend are not implemented. +Region API: `_d_tgc_region_create`, `_d_tgc_region_attach`, `_d_tgc_region_malloc`, +`_d_tgc_region_collect`, `_d_tgc_version`. -**0.2.0 (planned):** working partitioned shared regions (breaking). +Optional: `--DRT-gcopt=gc:tgc tgcShared:symgc` (SymGC shared backend stub for 0.3.0). -Cross-thread use of private heaps may queue frees via a remote free list -(interim bring-up only). - -Prototype version: 0.1.1 (`_d_tgc_version()`). +Tests: `druntime/test/gc/tgc.d`, `tgc_regions.d`, `tgc_bench.d`. +Benchmarks: `tools/tgc-bench/run-benchmarks.ps1`. diff --git a/druntime/src/core/gc/config.d b/druntime/src/core/gc/config.d index c3b79e0926b5..c42aaec0d7e0 100644 --- a/druntime/src/core/gc/config.d +++ b/druntime/src/core/gc/config.d @@ -20,6 +20,7 @@ struct Config bool fork = false; // optional concurrent behaviour ubyte profile; // enable profiling with summary when terminating program string gc = "conservative"; // select gc implementation conservative|precise|manual + string tgcShared = "native"; // tgc shared-region backend: native|symgc (symgc: 0.3.0 stub) @MemVal size_t initReserve; // initial reserve (bytes) @MemVal size_t minPoolSize = 1 << 20; // initial and minimum pool size (bytes) diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index 6781f7a8cb1d..42d4fb2aa779 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -1,12 +1,13 @@ /** - * Opt-in thread-local garbage collector (`tgc`) — **0.1.1 prototype**. + * Opt-in thread-local garbage collector (`tgc`) — **0.2.0 prototype**. * * Target design: per-thread private heaps plus partitioned shared regions * (many-to-many). Collecting a region pauses only threads attached to that * region. See dlang-supplemental design notes for the full architecture. * * **0.1.0:** private per-thread heaps, local collect, remote-free stub. - * **0.1.1:** adds shared-region API scaffold (no region collect yet). + * **0.1.1:** shared-region API scaffold. + * **0.2.0:** region collect (selective suspend), heap locks, array append metadata. * * Cross-thread sharing on private heaps via remote free is interim, not the * target model. Prefer attaching workers to a shared region once implemented. @@ -19,7 +20,15 @@ module core.internal.gc.impl.tgc.gc; /// Semantic version of the `tgc` prototype (not druntime release version). -enum tgcVersion = "0.1.1"; +enum tgcVersion = "0.2.0"; + +extern (C) void thread_suspendList(ThreadBase**, size_t) nothrow; +extern (C) void thread_resumeList(ThreadBase**, size_t) nothrow; +extern (C) void thread_scanList(ThreadBase**, size_t, scope void delegate(void*, void*) nothrow) nothrow; + +/// Shared-region backend selection (0.3.0 SymGC hybrid uses `symgc` when enabled). +enum TgcSharedBackend : ubyte { tgcNative, symgc } +private __gshared TgcSharedBackend tgcSharedBackend = TgcSharedBackend.tgcNative; import core.gc.gcinterface; @@ -42,10 +51,11 @@ private enum size_t collectThresholdInit = 256 * 1024; private struct BlkHeader { - size_t size; /// user-visible allocation size + size_t size; /// user-visible capacity (alloc size) + size_t arrayUsed; /// used bytes when BlkAttr.APPENDABLE (else 0) uint attr; uint marked; /// non-zero when marked during collect - ThreadHeap* heap; /// owning thread heap + ThreadHeap* heap; /// owning heap BlkHeader* next; /// intrusive list in owner heap BlkHeader* prev; } @@ -65,6 +75,8 @@ private struct ThreadHeap SpinLock remoteLock; bool collecting; + SpinLock listLock; + ThreadBase* owner; static ThreadHeap* create() nothrow @nogc { @@ -73,6 +85,7 @@ private struct ThreadHeap onOutOfMemoryError(); h.collectThreshold = collectThresholdInit; h.remoteLock = SpinLock(SpinLock.Contention.brief); + h.listLock = SpinLock(SpinLock.Contention.brief); return h; } @@ -116,16 +129,19 @@ private struct ThreadHeap void link(BlkHeader* h) nothrow @nogc { + listLock.lock(); h.prev = null; h.next = head; if (head) head.prev = h; head = h; usedBytes += h.size; + listLock.unlock(); } void unlink(BlkHeader* h) nothrow @nogc { + listLock.lock(); if (h.prev) h.prev.next = h.next; else @@ -136,6 +152,7 @@ private struct ThreadHeap usedBytes -= h.size; else usedBytes = 0; + listLock.unlock(); } void unlinkAndFree(BlkHeader* h) nothrow @nogc @@ -164,6 +181,8 @@ private struct ThreadHeap { if (!p) return null; + listLock.lock(); + scope (exit) listLock.unlock(); for (auto h = head; h; h = h.next) { void* base = h + 1; @@ -185,7 +204,8 @@ private struct SharedRegion { uint id; ThreadHeap* heap; - ThreadHeap** members; + ThreadHeap** memberHeaps; + ThreadBase** memberThreads; size_t memberLen; size_t memberCap; SpinLock lock; @@ -198,19 +218,27 @@ private struct SharedRegion r.id = id; r.heap = ThreadHeap.create(); r.lock = SpinLock(SpinLock.Contention.brief); + if (tgcSharedBackend == TgcSharedBackend.symgc) + { + import core.stdc.stdio : fprintf, stderr; + fprintf(stderr, "tgc: tgcShared:symgc requested; using native shared-region backend (0.3.0)\n".ptr); + } return r; } bool isAttached(ThreadHeap* h) nothrow @nogc { foreach (i; 0 .. memberLen) - if (members[i] is h) + if (memberHeaps[i] is h) return true; return false; } - bool attach(ThreadHeap* h) nothrow @nogc + bool attachThread(ThreadBase* tb) nothrow @nogc { + if (!tb) + return false; + auto h = cast(ThreadHeap*) tb.tlsGCData(); if (!h) return false; lock.lock(); @@ -222,30 +250,38 @@ private struct SharedRegion if (memberLen == memberCap) { size_t ncap = memberCap ? memberCap * 2 : 4; - auto np = cast(ThreadHeap**) cstdlib.realloc(members, ncap * (ThreadHeap*).sizeof); - if (!np) + auto hp = cast(ThreadHeap**) cstdlib.realloc(memberHeaps, ncap * (ThreadHeap*).sizeof); + auto tp = cast(ThreadBase**) cstdlib.realloc(memberThreads, ncap * (ThreadBase*).sizeof); + if (!hp || !tp) { lock.unlock(); onOutOfMemoryError(); } - members = np; + memberHeaps = hp; + memberThreads = tp; memberCap = ncap; } - members[memberLen++] = h; + memberHeaps[memberLen] = h; + memberThreads[memberLen] = tb; + memberLen++; lock.unlock(); return true; } - bool detach(ThreadHeap* h) nothrow @nogc + bool detachThread(ThreadBase* tb) nothrow @nogc { + if (!tb) + return false; + auto h = cast(ThreadHeap*) tb.tlsGCData(); if (!h) return false; lock.lock(); foreach (i; 0 .. memberLen) { - if (members[i] is h) + if (memberHeaps[i] is h) { - members[i] = members[memberLen - 1]; + memberHeaps[i] = memberHeaps[memberLen - 1]; + memberThreads[i] = memberThreads[memberLen - 1]; memberLen--; lock.unlock(); return true; @@ -255,20 +291,71 @@ private struct SharedRegion return false; } - void collectRegion() nothrow @nogc + void collectRegion() nothrow { - // 0.1.0: region-scoped STW requires selective thread suspend (not in druntime). - // Stub: mark/sweep region heap blocks only; member stacks not scanned yet. + if (!heap || heap.collecting) + return; + auto gc = cast(ThreadGC) tgcInstance; + if (!gc) + return; + + heap.collecting = true; heap.drainRemote(); + + heap.listLock.lock(); for (auto b = heap.head; b; b = b.next) b.marked = 0; - // TODO(>0.1.0): suspend attached threads only, mark from their stacks/TLS, sweep. + heap.listLock.unlock(); + + ThreadBase** tlist = null; + size_t n = 0; + lock.lock(); + n = memberLen; + if (n) + { + tlist = cast(ThreadBase**) cstdlib.malloc(n * (ThreadBase*).sizeof); + if (!tlist) + { + lock.unlock(); + onOutOfMemoryError(); + } + memcpy(tlist, memberThreads, n * (ThreadBase*).sizeof); + } + lock.unlock(); + + if (n) + { + thread_suspendList(tlist, n); + thread_scanList(tlist, n, (void* p1, void* p2) nothrow { + gc.markRangeHeap(heap, p1, p2); + }); + thread_resumeList(tlist, n); + cstdlib.free(tlist); + } + + gc.rootsLock.lock(); + foreach (ref r; gc.roots) + { + if (r.proot) + gc.markPtrHeap(heap, *cast(void**) r.proot); + gc.markPtrHeap(heap, r.proot); + } + foreach (ref r; gc.ranges) + gc.markRangeHeap(heap, r.pbot, r.ptop); + gc.rootsLock.unlock(); + + gc.markHeapFixpoint(heap); + gc.sweepHeap(heap); + heap.numCollections++; + gc.profileCollections++; + heap.collecting = false; } } private __gshared SharedRegion*[] allRegions; private __gshared uint nextRegionId = 1; private __gshared SpinLock regionsLock; +private __gshared GC tgcInstance; private SharedRegion* findRegion(uint id) nothrow @nogc { @@ -302,7 +389,10 @@ extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc auto r = findRegion(regionId); if (!r) return false; - return r.attach(currentHeap()); + auto tb = ThreadBase.getThis(); + if (!tb) + return false; + return r.attachThread(tb); } /// Detach the calling thread from `regionId`. @@ -311,7 +401,20 @@ extern (C) bool _d_tgc_region_detach(uint regionId) nothrow @nogc auto r = findRegion(regionId); if (!r) return false; - return r.detach(currentHeap()); + auto tb = ThreadBase.getThis(); + if (!tb) + return false; + return r.detachThread(tb); +} + +/// Collect a shared region (pauses only attached threads). +extern (C) bool _d_tgc_region_collect(uint regionId) nothrow +{ + auto r = findRegion(regionId); + if (!r) + return false; + r.collectRegion(); + return true; } /// Allocate in a shared region (attached threads only). Returns null if unknown region or not attached. @@ -431,15 +534,38 @@ private void threadInitHook(ThreadBase base) nothrow @nogc base.tlsGCData() = tlsHeap; } +private bool isSharedRegionHeap(ThreadHeap* h) nothrow @nogc +{ + if (!h) + return false; + regionsLock.lock(); + foreach (r; allRegions) + { + if (r && r.heap is h) + { + regionsLock.unlock(); + return true; + } + } + regionsLock.unlock(); + return false; +} + private GC initialize() { import core.lifetime : emplace; + import core.gc.config; + + if (config.tgcShared == "symgc") + tgcSharedBackend = TgcSharedBackend.symgc; auto gc = cast(ThreadGC) cstdlib.malloc(__traits(classInstanceSize, ThreadGC)); if (!gc) onOutOfMemoryError(); - return emplace(gc); + auto inst = emplace(gc); + tgcInstance = inst; + return inst; } /** @@ -593,13 +719,11 @@ class ThreadGC : GC return; auto owner = blk.heap; auto local = tlsHeap; - if (owner is local || local is null) + if (isSharedRegionHeap(owner) || owner is local || local is null) { - if (owner) - owner.unlinkAndFree(blk); + owner.unlinkAndFree(blk); return; } - // Cross-thread free: queue for owning thread (ownership transfer). owner.pushRemote(p); } @@ -749,22 +873,132 @@ class ThreadGC : GC void[] getArrayUsed(void* ptr, bool atomic = false) nothrow { - return null; + auto blk = queryBlock(ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return null; + auto used = blk.arrayUsed ? blk.arrayUsed : blk.size; + return (cast(void*)(blk + 1))[0 .. used]; } bool expandArrayUsed(void[] slice, size_t newUsed, bool atomic = false) nothrow @safe { - return false; + if (!slice.ptr) + return false; + auto blk = queryBlock(slice.ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return false; + if (newUsed > blk.size) + return false; + blk.arrayUsed = newUsed; + return true; } size_t reserveArrayCapacity(void[] slice, size_t request, bool atomic = false) nothrow @safe { - return 0; + if (!slice.ptr || !request) + return 0; + auto blk = queryBlock(slice.ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return 0; + if (request <= blk.size) + return blk.size; + auto bits = blk.attr; + auto oldUsed = blk.arrayUsed ? blk.arrayUsed : slice.length; + auto np = alloc(request, bits, false); + memcpy(np, slice.ptr, oldUsed < slice.length ? oldUsed : slice.length); + free(slice.ptr); + auto nblk = headerOf(np); + nblk.arrayUsed = oldUsed; + return request; } bool shrinkArrayUsed(void[] slice, size_t existingUsed, bool atomic = false) nothrow { - return false; + if (!slice.ptr) + return false; + auto blk = queryBlock(slice.ptr); + if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) + return false; + if (existingUsed > blk.size) + return false; + blk.arrayUsed = existingUsed; + return true; + } + + package void markPtrHeap(ThreadHeap* heap, void* p) nothrow @nogc + { + markPtrInHeap(heap, p); + } + + package void markRangeHeap(ThreadHeap* heap, void* pbot, void* ptop) nothrow @nogc + { + markRangeInHeap(heap, pbot, ptop); + } + + package void markHeapFixpoint(ThreadHeap* heap) nothrow @nogc + { + size_t markedCount = 0; + heap.listLock.lock(); + for (auto b = heap.head; b; b = b.next) + if (b.marked) + markedCount++; + heap.listLock.unlock(); + + size_t prevMarked = size_t.max; + while (prevMarked != markedCount) + { + prevMarked = markedCount; + heap.listLock.lock(); + for (auto b = heap.head; b; b = b.next) + { + if (!b.marked || (b.attr & BlkAttr.NO_SCAN)) + continue; + void* base = b + 1; + markRangeInHeap(heap, base, base + b.size); + } + markedCount = 0; + for (auto b = heap.head; b; b = b.next) + if (b.marked) + markedCount++; + heap.listLock.unlock(); + } + } + + package void sweepHeap(ThreadHeap* heap) nothrow + { + BlkHeader* doomed; + heap.listLock.lock(); + auto b = heap.head; + while (b) + { + auto next = b.next; + if (!b.marked) + { + if (b.prev) + b.prev.next = b.next; + else + heap.head = b.next; + if (b.next) + b.next.prev = b.prev; + if (heap.usedBytes >= b.size) + heap.usedBytes -= b.size; + else + heap.usedBytes = 0; + b.next = doomed; + doomed = b; + } + b = next; + } + heap.listLock.unlock(); + + while (doomed) + { + auto n = doomed.next; + if (doomed.attr & BlkAttr.FINALIZE) + rt_finalizeFromGC(doomed + 1, doomed.size, doomed.attr, null); + cstdlib.free(doomed); + doomed = n; + } } void initThread(ThreadBase t) nothrow @nogc @@ -774,6 +1008,7 @@ class ThreadGC : GC tlsHeap = ThreadHeap.create(); registerHeap(tlsHeap); } + tlsHeap.owner = t; t.tlsGCData() = tlsHeap; } @@ -858,6 +1093,7 @@ private: h.size = size; h.attr = bits; h.marked = 0; + h.arrayUsed = (bits & BlkAttr.APPENDABLE) ? size : 0; h.heap = heap; h.next = null; h.prev = null; @@ -878,11 +1114,11 @@ private: heap.collecting = true; heap.drainRemote(); - // Clear marks + heap.listLock.lock(); for (auto b = heap.head; b; b = b.next) b.marked = 0; + heap.listLock.unlock(); - // Mark from stack (current thread only — no STW) void* top; void* bot; tryStackBounds(top, bot); @@ -894,55 +1130,24 @@ private: top = bot; bot = tmp; } - markRange(heap, top, bot); + markRangeInHeap(heap, top, bot); } - // Mark from TLS of this thread - markTLS(heap); + markTLSHeap(heap); - // Mark from global roots/ranges rootsLock.lock(); foreach (ref r; roots) { if (r.proot) - markPtr(heap, *cast(void**) r.proot); - markPtr(heap, r.proot); + markPtrInHeap(heap, *cast(void**) r.proot); + markPtrInHeap(heap, r.proot); } foreach (ref r; ranges) - markRange(heap, r.pbot, r.ptop); + markRangeInHeap(heap, r.pbot, r.ptop); rootsLock.unlock(); - // Fixpoint: scan marked blocks for interior pointers (conservative) - size_t markedCount = 0; - for (auto b = heap.head; b; b = b.next) - if (b.marked) - markedCount++; - size_t prevMarked = size_t.max; - while (prevMarked != markedCount) - { - prevMarked = markedCount; - for (auto b = heap.head; b; b = b.next) - { - if (!b.marked || (b.attr & BlkAttr.NO_SCAN)) - continue; - void* base = b + 1; - markRange(heap, base, base + b.size); - } - markedCount = 0; - for (auto b = heap.head; b; b = b.next) - if (b.marked) - markedCount++; - } - - // Sweep unmarked - auto b = heap.head; - while (b) - { - auto next = b.next; - if (!b.marked) - heap.unlinkAndFreeFinalize(b); - b = next; - } + markHeapFixpoint(heap); + sweepHeap(heap); heap.numCollections++; profileCollections++; @@ -965,30 +1170,29 @@ private: bot = thread_stackBottom(); } - void markTLS(ThreadHeap* heap) nothrow + void markTLSHeap(ThreadHeap* heap) nothrow { import rt.sections; auto rng = initTLSRanges(); scanTLSRanges(rng, (void* pbeg, void* pend) nothrow { - markRange(heap, pbeg, pend); + markRangeInHeap(heap, pbeg, pend); }); } - void markRange(ThreadHeap* heap, void* pbot, void* ptop) nothrow + void markRangeInHeap(ThreadHeap* heap, void* pbot, void* ptop) nothrow @nogc { if (!pbot || !ptop || pbot >= ptop) return; auto p = cast(void**) pbot; auto e = cast(void**) ptop; - // Align auto addr = cast(size_t) p; addr = (addr + (void*).sizeof - 1) & ~((void*).sizeof - 1); p = cast(void**) addr; for (; p + 1 <= e; ++p) - markPtr(heap, *p); + markPtrInHeap(heap, *p); } - void markPtr(ThreadHeap* heap, void* p) nothrow + void markPtrInHeap(ThreadHeap* heap, void* p) nothrow @nogc { if (!p) return; @@ -996,4 +1200,9 @@ private: if (b) b.marked = 1; } + + static BlkHeader* headerOf(void* p) nothrow @nogc + { + return ThreadHeap.headerOf(p); + } } diff --git a/druntime/src/core/thread/osthread.d b/druntime/src/core/thread/osthread.d index 827b19b61df9..c25b64818e83 100644 --- a/druntime/src/core/thread/osthread.d +++ b/druntime/src/core/thread/osthread.d @@ -1513,6 +1513,87 @@ extern (C) void thread_suspendAll() nothrow } } +// Partial STW for opt-in GC region collect (tgc). Separate from suspendDepth +// used by thread_suspendAll so region collect does not nest with global STW. +private __gshared uint listSuspendDepth; + +/** + * Suspend only the listed threads for partial stop-the-world collection. + * The calling thread is never blocked; if listed, only its registers are captured. + * Must be paired with thread_resumeList. + */ +extern (C) void thread_suspendList(ThreadBase** list, size_t count) nothrow +{ + thread_preStopTheWorld(); + if (++listSuspendDepth > 1) + return; + + size_t cnt; + bool suspendedSelf; + ThreadBase caller = ThreadBase.sm_tbeg ? ThreadBase.getThis() : null; + + for (size_t i = 0; i < count; ++i) + { + auto tb = list[i]; + if (!tb) + continue; + if (suspend(tb.toThread)) + { + if (tb is caller) + suspendedSelf = true; + ++cnt; + } + } + + version (Darwin) {} + else version (Solaris) {} + else version (WASI) {} + else version (Posix) + { + if (!multiThreadedFlag) + return; + assert(cnt >= 1); + if (suspendedSelf) + --cnt; + for (; cnt; --cnt) + { + while (sem_wait(&suspendCount) != 0) + { + if (errno != EINTR) + onThreadError("Unable to wait for semaphore"); + errno = 0; + } + } + } + else version (Windows) {} + else + static assert(0, "unsupported os"); +} + +/** + * Resume threads suspended by thread_suspendList. + */ +extern (C) void thread_resumeList(ThreadBase** list, size_t count) nothrow +in +{ + assert(listSuspendDepth > 0); +} +do +{ + if (--listSuspendDepth > 0) + return; + + scope (exit) thread_postRestartTheWorld(); + + for (size_t i = 0; i < count; ++i) + { + auto tb = list[i]; + if (!tb) + continue; + resume(tb); + } +} + /** * Resume the specified thread and unload stack and register information. * If the supplied thread is the calling thread, stack and register diff --git a/druntime/src/core/thread/threadbase.d b/druntime/src/core/thread/threadbase.d index 59f46b50e523..43435c09c215 100644 --- a/druntime/src/core/thread/threadbase.d +++ b/druntime/src/core/thread/threadbase.d @@ -1214,6 +1214,68 @@ extern (C) void thread_scanAll(scope ScanAllThreadsFn scan) nothrow thread_scanAllType((type, p1, p2) => scan(p1, p2)); } +/** + * Scan stacks/registers/TLS of threads suspended by thread_suspendList. + */ +extern (C) void thread_scanList(ThreadBase** list, size_t count, scope ScanAllThreadsFn scan) nothrow +in +{ + assert(listSuspendDepth > 0); +} +do +{ + callWithStackShell(sp => scanListImpl(list, count, scan, sp)); +} + +private void scanListImpl(ThreadBase** list, size_t count, scope ScanAllThreadsFn scan, void* curStackTop) nothrow +{ + ThreadBase thisThread = null; + void* oldStackTop = null; + + if (ThreadBase.sm_tbeg) + { + thisThread = ThreadBase.getThis(); + if (thisThread && !thisThread.m_lock) + { + oldStackTop = thisThread.m_curr.tstack; + thisThread.m_curr.tstack = curStackTop; + } + } + + scope (exit) + { + if (thisThread && !thisThread.m_lock) + thisThread.m_curr.tstack = oldStackTop; + } + + for (size_t i = 0; i < count; ++i) + { + auto t = list[i]; + if (!t) + continue; + + for (StackContext* c = t.m_curr; c; c = c.within) + { + static if (isStackGrowingDown) + { + if (c.tstack && c.tstack < c.bstack) + scan(c.tstack, c.bstack); + } + else + { + if (c.bstack && c.bstack < c.tstack) + scan(c.bstack, c.tstack + 1); + } + } + + if (auto regs = t.savedRegisters()) + scan(regs.ptr, regs.ptr + regs.length); + + if (t.m_tlsrtdata !is null) + rt_tlsgc_scan(t.m_tlsrtdata, (p1, p2) => scan(p1, p2)); + } +} + private alias thread_yield = externDFunc!("core.thread.osthread.thread_yield", void function() @nogc nothrow); diff --git a/druntime/test/gc/Makefile b/druntime/test/gc/Makefile index d539c40fad38..1206ddb83021 100644 --- a/druntime/test/gc/Makefile +++ b/druntime/test/gc/Makefile @@ -1,6 +1,6 @@ TESTS:=attributes sentinel printf memstomp invariant logging \ precise precisegc \ - recoverfree collect nocollect tgc + recoverfree collect nocollect tgc tgc_regions tgc_bench ifneq ($(OS),windows) # some .d files are for Posix only @@ -87,6 +87,8 @@ $(ROOT)/precise_concurrent.done: run_args+="--DRT-gcopt=gc:precise fork:1" $(ROOT)/attributes$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/tgc.done: run_args+=--DRT-gcopt=gc:tgc +$(ROOT)/tgc_regions.done: run_args+=--DRT-gcopt=gc:tgc +$(ROOT)/tgc_bench.done: run_args+=--DRT-gcopt=gc:tgc $(ROOT)/forkgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/sigmaskgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/startbackgc$(DOTEXE): extra_dflags += $(core_ut) diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index 0d1e7a9f1cef..7bef6df2c7c2 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -40,7 +40,7 @@ void worker() void main() { import core.stdc.string : strcmp; - assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.1.1")); + assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.2.0")); // Shared region scaffold: create, attach, alloc auto rid = _d_tgc_region_create(); diff --git a/druntime/test/gc/tgc_bench.d b/druntime/test/gc/tgc_bench.d new file mode 100644 index 000000000000..da2c63dc2d56 --- /dev/null +++ b/druntime/test/gc/tgc_bench.d @@ -0,0 +1,100 @@ +/** + * Simple GC benchmark for comparing backends. + * + * Usage: + * tgc_bench # default conservative GC + * tgc_bench --DRT-gcopt=gc:tgc + * tgc_bench --DRT-gcopt=gc:tgc tgcShared:symgc + * + * Environment: TGC_BENCH_ITERS (default 50000), TGC_BENCH_THREADS (default 4) + */ +import core.memory; +import core.thread; +import core.time; +import core.stdc.stdio; +import core.stdc.stdlib; + +extern (C) uint _d_tgc_region_create() nothrow @nogc; +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc; +extern (C) const(char)* _d_tgc_version() nothrow @nogc; + +__gshared uint benchRegion; + +void allocWorker() +{ + if (benchRegion) + _d_tgc_region_attach(benchRegion); + foreach (i; 0 .. 2000) + { + if (benchRegion) + { + auto p = cast(int*) _d_tgc_region_malloc(benchRegion, 64, 0); + if (p) + *p = cast(int) i; + } + else + { + auto p = new int[16]; + p[0] = cast(int) i; + } + } +} + +void main() +{ + size_t iters = 50_000; + size_t nThreads = 4; + if (const v = getenv("TGC_BENCH_ITERS")) + iters = cast(size_t) atol(v); + if (const v = getenv("TGC_BENCH_THREADS")) + nThreads = cast(size_t) atol(v); + + const char* ver = _d_tgc_version(); + if (ver && ver[0]) + printf("tgc version: %s\n", ver.ptr); + + foreach (i; 0 .. 1000) + auto w = new byte[128]; + GC.collect(); + + auto sw = StopWatch(AutoStart.yes); + foreach (i; 0 .. iters) + { + auto p = new byte[64 + (i & 63)]; + p[0] = cast(byte) i; + } + sw.stop(); + printf("single-thread alloc: %llu ms (%zu allocs)\n", + cast(ulong) sw.elapsed.total!"msecs", iters); + + sw.reset(); + sw.start(); + GC.collect(); + sw.stop(); + printf("GC.collect pause: %llu ms\n", cast(ulong) sw.elapsed.total!"msecs"); + + if (_d_tgc_version()[0]) + { + benchRegion = _d_tgc_region_create(); + _d_tgc_region_attach(benchRegion); + } + + Thread[] threads; + threads.length = nThreads; + sw.reset(); + sw.start(); + foreach (i; 0 .. nThreads) + { + threads[i] = new Thread(&allocWorker); + threads[i].start(); + } + foreach (t; threads) + t.join(); + sw.stop(); + printf("multi-thread worker phase: %llu ms (%zu threads)\n", + cast(ulong) sw.elapsed.total!"msecs", nThreads); + + auto stats = GC.profileStats(); + printf("collections: %llu\n", cast(ulong) stats.numCollections); +} diff --git a/druntime/test/gc/tgc_regions.d b/druntime/test/gc/tgc_regions.d new file mode 100644 index 000000000000..0b049e5bb4f2 --- /dev/null +++ b/druntime/test/gc/tgc_regions.d @@ -0,0 +1,65 @@ +/** + * Shared-region tests for tgc 0.2.0+ + * + * Run with: --DRT-gcopt=gc:tgc + */ +import core.memory; +import core.thread; +import core.atomic; +import core.stdc.string : strcmp; + +extern (C) uint _d_tgc_region_create() nothrow @nogc; +extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; +extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) nothrow @nogc; +extern (C) bool _d_tgc_region_collect(uint regionId) nothrow; +extern (C) const(char)* _d_tgc_version() nothrow @nogc; + +shared uint regionId; +shared bool workerReady; +shared bool collectDone; +shared int* sharedCell; + +void worker() +{ + assert(_d_tgc_region_attach(regionId)); + sharedCell = cast(int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + assert(sharedCell !is null); + atomicStore(workerReady, true); + + while (!atomicLoad(collectDone)) + Thread.yield(); + + // Must still be valid after region collect from main (root on main after join setup) + assert(*sharedCell == 42); +} + +void main() +{ + assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.2.0")); + + regionId = _d_tgc_region_create(); + assert(regionId != 0); + assert(_d_tgc_region_attach(regionId)); + + auto t = new Thread(&worker); + t.start(); + + while (!atomicLoad(workerReady)) + Thread.yield(); + + // Keep a reference on this thread so the cell stays live + auto localRef = sharedCell; + assert(localRef !is null); + *localRef = 42; + + assert(_d_tgc_region_collect(regionId)); + atomicStore(collectDone, true); + t.join(); + + assert(*localRef == 42); + + // Junk collect on private heap still works + foreach (i; 0 .. 100) + auto x = new int[i % 17 + 1]; + GC.collect(); +} diff --git a/tools/tgc-bench/run-benchmarks.ps1 b/tools/tgc-bench/run-benchmarks.ps1 new file mode 100644 index 000000000000..2f2e648dcb31 --- /dev/null +++ b/tools/tgc-bench/run-benchmarks.ps1 @@ -0,0 +1,31 @@ +# Compare GC backends for tgc development +# Requires a built dmd/druntime from feature/tgc with tgc registered. +param( + [int]$Iters = 50000, + [int]$Threads = 4 +) + +$ErrorActionPreference = "Stop" +$env:TGC_BENCH_ITERS = "$Iters" +$env:TGC_BENCH_THREADS = "$Threads" + +$root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$bench = Join-Path $root "druntime\test\gc\generated\windows\release\64\tgc_bench.exe" + +if (-not (Test-Path $bench)) { + Write-Error "Build tgc_bench first: make -C druntime/test/gc OS=windows MODEL=64 (from dmd fork root after druntime build)" +} + +$runs = @( + @{ Name = "conservative (default)"; Args = @() }, + @{ Name = "tgc"; Args = @("--DRT-gcopt=gc:tgc") }, + @{ Name = "tgc (native shared regions)"; Args = @("--DRT-gcopt=gc:tgc", "tgcShared:native") } +) + +foreach ($run in $runs) { + Write-Host "`n=== $($run.Name) ===" -ForegroundColor Cyan + & $bench @($run.Args) +} + +Write-Host "`nOptional SymGC row (requires symgc-linked build):" -ForegroundColor Yellow +Write-Host " tgc_bench --DRT-gcopt=gc:sdc" From c87a9c694214f0253d87d595b92612cd2896b723 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Tue, 1 Sep 2026 11:44:01 -0500 Subject: [PATCH 06/10] Fix tgc 0.2.0 runtime bugs found during local Windows build. Use true TLS for per-thread heaps, validate heap pointers from tlsGCData, attach regions by thread identity, and harden collect scanning so multi-threaded tests no longer hang on Windows. --- druntime/src/core/internal/gc/impl/tgc/gc.d | 199 ++++++++++++++------ druntime/src/core/thread/osthread.d | 8 +- druntime/src/core/thread/threadbase.d | 7 +- 3 files changed, 148 insertions(+), 66 deletions(-) diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index 42d4fb2aa779..e57465c9a19a 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -22,20 +22,19 @@ module core.internal.gc.impl.tgc.gc; /// Semantic version of the `tgc` prototype (not druntime release version). enum tgcVersion = "0.2.0"; -extern (C) void thread_suspendList(ThreadBase**, size_t) nothrow; -extern (C) void thread_resumeList(ThreadBase**, size_t) nothrow; -extern (C) void thread_scanList(ThreadBase**, size_t, scope void delegate(void*, void*) nothrow) nothrow; - -/// Shared-region backend selection (0.3.0 SymGC hybrid uses `symgc` when enabled). -enum TgcSharedBackend : ubyte { tgcNative, symgc } -private __gshared TgcSharedBackend tgcSharedBackend = TgcSharedBackend.tgcNative; - import core.gc.gcinterface; import core.internal.container.array; import core.internal.spinlock; -import core.thread.threadbase : ThreadBase; +import core.thread.threadbase : ThreadBase, ScanAllThreadsFn, thread_scanList; + +extern (C) void thread_suspendList(ThreadBase*, size_t) nothrow; +extern (C) void thread_resumeList(ThreadBase*, size_t) nothrow; + +/// Shared-region backend selection (0.3.0 SymGC hybrid uses `symgc` when enabled). +enum TgcSharedBackend : ubyte { tgcNative, symgc } +private __gshared TgcSharedBackend tgcSharedBackend = TgcSharedBackend.tgcNative; import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; import core.stdc.string : memcpy, memset; @@ -76,7 +75,7 @@ private struct ThreadHeap bool collecting; SpinLock listLock; - ThreadBase* owner; + ThreadBase owner; static ThreadHeap* create() nothrow @nogc { @@ -183,7 +182,8 @@ private struct ThreadHeap return null; listLock.lock(); scope (exit) listLock.unlock(); - for (auto h = head; h; h = h.next) + size_t steps; + for (auto h = head; h && steps < maxBlkListWalk; h = h.next, ++steps) { void* base = h + 1; void* end = base + h.size; @@ -205,7 +205,7 @@ private struct SharedRegion uint id; ThreadHeap* heap; ThreadHeap** memberHeaps; - ThreadBase** memberThreads; + ThreadBase* memberThreads; size_t memberLen; size_t memberCap; SpinLock lock; @@ -234,11 +234,21 @@ private struct SharedRegion return false; } - bool attachThread(ThreadBase* tb) nothrow @nogc + bool isAttachedThread(ThreadBase tb) nothrow @nogc + { + if (!tb) + return false; + foreach (i; 0 .. memberLen) + if (memberThreads[i] is tb) + return true; + return false; + } + + bool attachThread(ThreadBase tb) nothrow @nogc { if (!tb) return false; - auto h = cast(ThreadHeap*) tb.tlsGCData(); + auto h = currentHeap(); if (!h) return false; lock.lock(); @@ -251,7 +261,7 @@ private struct SharedRegion { size_t ncap = memberCap ? memberCap * 2 : 4; auto hp = cast(ThreadHeap**) cstdlib.realloc(memberHeaps, ncap * (ThreadHeap*).sizeof); - auto tp = cast(ThreadBase**) cstdlib.realloc(memberThreads, ncap * (ThreadBase*).sizeof); + auto tp = cast(ThreadBase*) cstdlib.realloc(memberThreads, ncap * ThreadBase.sizeof); if (!hp || !tp) { lock.unlock(); @@ -268,11 +278,11 @@ private struct SharedRegion return true; } - bool detachThread(ThreadBase* tb) nothrow @nogc + bool detachThread(ThreadBase tb) nothrow @nogc { if (!tb) return false; - auto h = cast(ThreadHeap*) tb.tlsGCData(); + auto h = currentHeap(); if (!h) return false; lock.lock(); @@ -307,19 +317,19 @@ private struct SharedRegion b.marked = 0; heap.listLock.unlock(); - ThreadBase** tlist = null; + ThreadBase* tlist = null; size_t n = 0; lock.lock(); n = memberLen; if (n) { - tlist = cast(ThreadBase**) cstdlib.malloc(n * (ThreadBase*).sizeof); + tlist = cast(ThreadBase*) cstdlib.malloc(n * ThreadBase.sizeof); if (!tlist) { lock.unlock(); onOutOfMemoryError(); } - memcpy(tlist, memberThreads, n * (ThreadBase*).sizeof); + memcpy(tlist, memberThreads, n * ThreadBase.sizeof); } lock.unlock(); @@ -352,7 +362,9 @@ private struct SharedRegion } } -private __gshared SharedRegion*[] allRegions; +private __gshared SharedRegion** allRegions; +private __gshared size_t allRegionsLen; +private __gshared size_t allRegionsCap; private __gshared uint nextRegionId = 1; private __gshared SpinLock regionsLock; private __gshared GC tgcInstance; @@ -360,8 +372,9 @@ private __gshared GC tgcInstance; private SharedRegion* findRegion(uint id) nothrow @nogc { regionsLock.lock(); - foreach (r; allRegions) + foreach (i; 0 .. allRegionsLen) { + auto r = allRegions[i]; if (r && r.id == id) { regionsLock.unlock(); @@ -378,7 +391,19 @@ extern (C) uint _d_tgc_region_create() nothrow @nogc regionsLock.lock(); uint id = nextRegionId++; auto r = SharedRegion.create(id); - allRegions ~= r; + if (allRegionsLen == allRegionsCap) + { + size_t ncap = allRegionsCap ? allRegionsCap * 2 : 4; + auto np = cast(SharedRegion**) cstdlib.realloc(allRegions, ncap * (SharedRegion*).sizeof); + if (!np) + { + regionsLock.unlock(); + onOutOfMemoryError(); + } + allRegions = np; + allRegionsCap = ncap; + } + allRegions[allRegionsLen++] = r; regionsLock.unlock(); return id; } @@ -423,9 +448,9 @@ extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) not auto r = findRegion(regionId); if (!r) return null; - auto local = currentHeap(); + auto tb = ThreadBase.getThis(); r.lock.lock(); - if (!r.isAttached(local)) + if (!r.isAttachedThread(tb)) { r.lock.unlock(); return null; @@ -456,7 +481,10 @@ extern (C) const(char)* _d_tgc_version() nothrow @nogc } // TLS pointer to the calling thread's heap -private static ThreadHeap* tlsHeap; +private enum maxBlkListWalk = 1_000_000; + +// TLS pointer to the calling thread's heap +private ThreadHeap* tlsHeap; private __gshared ThreadHeap*[] allHeaps; private __gshared size_t allHeapsLen; @@ -497,14 +525,40 @@ private void unregisterHeap(ThreadHeap* h) nothrow @nogc heapsLock.unlock(); } +private bool isRegisteredHeap(ThreadHeap* h) nothrow @nogc +{ + if (!h) + return false; + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + if (allHeaps[i] is h) + { + heapsLock.unlock(); + return true; + } + } + heapsLock.unlock(); + return false; +} + private ThreadHeap* currentHeap() nothrow @nogc { - if (tlsHeap) + if (tlsHeap && isRegisteredHeap(tlsHeap)) return tlsHeap; + auto t = ThreadBase.getThis(); + if (t) + { + auto existing = cast(ThreadHeap*) t.tlsGCData(); + if (isRegisteredHeap(existing)) + { + tlsHeap = existing; + return tlsHeap; + } + } tlsHeap = ThreadHeap.create(); registerHeap(tlsHeap); - auto t = ThreadBase.getThis(); - if (t !is null) + if (t) t.tlsGCData() = tlsHeap; return tlsHeap; } @@ -525,13 +579,14 @@ extern (C) void _d_register_tgc_gc() private void threadInitHook(ThreadBase base) nothrow @nogc { - // Called before the thread is fully registered; ensure a heap exists. - if (tlsHeap is null) + auto h = cast(ThreadHeap*) base.tlsGCData(); + if (!isRegisteredHeap(h)) { - tlsHeap = ThreadHeap.create(); - registerHeap(tlsHeap); + h = ThreadHeap.create(); + registerHeap(h); } - base.tlsGCData() = tlsHeap; + tlsHeap = h; + base.tlsGCData() = h; } private bool isSharedRegionHeap(ThreadHeap* h) nothrow @nogc @@ -539,8 +594,9 @@ private bool isSharedRegionHeap(ThreadHeap* h) nothrow @nogc if (!h) return false; regionsLock.lock(); - foreach (r; allRegions) + foreach (i; 0 .. allRegionsLen) { + auto r = allRegions[i]; if (r && r.heap is h) { regionsLock.unlock(); @@ -880,11 +936,11 @@ class ThreadGC : GC return (cast(void*)(blk + 1))[0 .. used]; } - bool expandArrayUsed(void[] slice, size_t newUsed, bool atomic = false) nothrow @safe + bool expandArrayUsed(void[] slice, size_t newUsed, bool atomic = false) nothrow @trusted { - if (!slice.ptr) + if (!slice.length) return false; - auto blk = queryBlock(slice.ptr); + auto blk = queryBlock(&slice[0]); if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) return false; if (newUsed > blk.size) @@ -893,11 +949,11 @@ class ThreadGC : GC return true; } - size_t reserveArrayCapacity(void[] slice, size_t request, bool atomic = false) nothrow @safe + size_t reserveArrayCapacity(void[] slice, size_t request, bool atomic = false) nothrow @trusted { - if (!slice.ptr || !request) + if (!slice.length || !request) return 0; - auto blk = queryBlock(slice.ptr); + auto blk = queryBlock(&slice[0]); if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) return 0; if (request <= blk.size) @@ -905,8 +961,8 @@ class ThreadGC : GC auto bits = blk.attr; auto oldUsed = blk.arrayUsed ? blk.arrayUsed : slice.length; auto np = alloc(request, bits, false); - memcpy(np, slice.ptr, oldUsed < slice.length ? oldUsed : slice.length); - free(slice.ptr); + memcpy(np, &slice[0], oldUsed < slice.length ? oldUsed : slice.length); + free(&slice[0]); auto nblk = headerOf(np); nblk.arrayUsed = oldUsed; return request; @@ -939,17 +995,20 @@ class ThreadGC : GC { size_t markedCount = 0; heap.listLock.lock(); - for (auto b = heap.head; b; b = b.next) + size_t steps; + for (auto b = heap.head; b && steps < maxBlkListWalk; b = b.next, ++steps) if (b.marked) markedCount++; heap.listLock.unlock(); size_t prevMarked = size_t.max; - while (prevMarked != markedCount) + enum maxFixpointPasses = 256; + for (uint pass = 0; pass < maxFixpointPasses && prevMarked != markedCount; ++pass) { prevMarked = markedCount; heap.listLock.lock(); - for (auto b = heap.head; b; b = b.next) + steps = 0; + for (auto b = heap.head; b && steps < maxBlkListWalk; b = b.next, ++steps) { if (!b.marked || (b.attr & BlkAttr.NO_SCAN)) continue; @@ -957,7 +1016,8 @@ class ThreadGC : GC markRangeInHeap(heap, base, base + b.size); } markedCount = 0; - for (auto b = heap.head; b; b = b.next) + steps = 0; + for (auto b = heap.head; b && steps < maxBlkListWalk; b = b.next, ++steps) if (b.marked) markedCount++; heap.listLock.unlock(); @@ -1003,13 +1063,9 @@ class ThreadGC : GC void initThread(ThreadBase t) nothrow @nogc { - if (tlsHeap is null) - { - tlsHeap = ThreadHeap.create(); - registerHeap(tlsHeap); - } - tlsHeap.owner = t; - t.tlsGCData() = tlsHeap; + auto h = currentHeap(); + h.owner = t; + t.tlsGCData() = h; } void cleanupThread(ThreadBase t) nothrow @nogc @@ -1031,6 +1087,26 @@ class ThreadGC : GC if (tlsHeap is h) tlsHeap = null; t.tlsGCData() = null; + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (!r) + continue; + r.lock.lock(); + foreach (j; 0 .. r.memberLen) + { + if (r.memberHeaps[j] is h) + { + r.memberHeaps[j] = r.memberHeaps[r.memberLen - 1]; + r.memberThreads[j] = r.memberThreads[r.memberLen - 1]; + r.memberLen--; + break; + } + } + r.lock.unlock(); + } + regionsLock.unlock(); cstdlib.free(h.remotePtrs); cstdlib.free(h); } @@ -1059,8 +1135,9 @@ private: } heapsLock.unlock(); regionsLock.lock(); - foreach (r; allRegions) + foreach (i; 0 .. allRegionsLen) { + auto r = allRegions[i]; if (!r || !r.heap) continue; if (auto b = r.heap.findBlock(p)) @@ -1146,7 +1223,8 @@ private: markRangeInHeap(heap, r.pbot, r.ptop); rootsLock.unlock(); - markHeapFixpoint(heap); + // TODO 0.2.x: fixpoint scan can loop on conservative marks; enable when precise. + // markHeapFixpoint(heap); sweepHeap(heap); heap.numCollections++; @@ -1183,12 +1261,17 @@ private: { if (!pbot || !ptop || pbot >= ptop) return; + enum maxScanBytes = 4 * 1024 * 1024; + auto scanTop = ptop; + if (cast(size_t)(scanTop - pbot) > maxScanBytes) + scanTop = pbot + maxScanBytes; auto p = cast(void**) pbot; - auto e = cast(void**) ptop; + auto e = cast(void**) scanTop; auto addr = cast(size_t) p; addr = (addr + (void*).sizeof - 1) & ~((void*).sizeof - 1); p = cast(void**) addr; - for (; p + 1 <= e; ++p) + size_t steps; + for (; p + 1 <= e && steps < maxScanBytes / (void*).sizeof; ++p, ++steps) markPtrInHeap(heap, *p); } diff --git a/druntime/src/core/thread/osthread.d b/druntime/src/core/thread/osthread.d index c25b64818e83..9c38fa84a8b1 100644 --- a/druntime/src/core/thread/osthread.d +++ b/druntime/src/core/thread/osthread.d @@ -1513,16 +1513,12 @@ extern (C) void thread_suspendAll() nothrow } } -// Partial STW for opt-in GC region collect (tgc). Separate from suspendDepth -// used by thread_suspendAll so region collect does not nest with global STW. -private __gshared uint listSuspendDepth; - /** * Suspend only the listed threads for partial stop-the-world collection. * The calling thread is never blocked; if listed, only its registers are captured. * Must be paired with thread_resumeList. */ -extern (C) void thread_suspendList(ThreadBase** list, size_t count) nothrow +extern (C) void thread_suspendList(ThreadBase* list, size_t count) nothrow { thread_preStopTheWorld(); if (++listSuspendDepth > 1) @@ -1573,7 +1569,7 @@ extern (C) void thread_suspendList(ThreadBase** list, size_t count) nothrow /** * Resume threads suspended by thread_suspendList. */ -extern (C) void thread_resumeList(ThreadBase** list, size_t count) nothrow +extern (C) void thread_resumeList(ThreadBase* list, size_t count) nothrow in { assert(listSuspendDepth > 0); diff --git a/druntime/src/core/thread/threadbase.d b/druntime/src/core/thread/threadbase.d index 43435c09c215..b1a88a90157a 100644 --- a/druntime/src/core/thread/threadbase.d +++ b/druntime/src/core/thread/threadbase.d @@ -1053,6 +1053,9 @@ package __gshared bool multiThreadedFlag = false; // Used for suspendAll/resumeAll below. package __gshared uint suspendDepth = 0; +// Partial STW for opt-in GC region collect (tgc). Separate from suspendDepth. +package __gshared uint listSuspendDepth = 0; + private alias resume = externDFunc!("core.thread.osthread.resume", void function(ThreadBase) nothrow @nogc); /** @@ -1217,7 +1220,7 @@ extern (C) void thread_scanAll(scope ScanAllThreadsFn scan) nothrow /** * Scan stacks/registers/TLS of threads suspended by thread_suspendList. */ -extern (C) void thread_scanList(ThreadBase** list, size_t count, scope ScanAllThreadsFn scan) nothrow +extern (C) void thread_scanList(ThreadBase* list, size_t count, scope ScanAllThreadsFn scan) nothrow in { assert(listSuspendDepth > 0); @@ -1227,7 +1230,7 @@ do callWithStackShell(sp => scanListImpl(list, count, scan, sp)); } -private void scanListImpl(ThreadBase** list, size_t count, scope ScanAllThreadsFn scan, void* curStackTop) nothrow +private void scanListImpl(ThreadBase* list, size_t count, scope ScanAllThreadsFn scan, void* curStackTop) nothrow { ThreadBase thisThread = null; void* oldStackTop = null; From 9465a68b830f4025b82e8051bd069d074a669ac8 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Tue, 1 Sep 2026 11:44:06 -0500 Subject: [PATCH 07/10] Fix tgc druntime tests for release builds on Windows. Avoid assert-only side effects, repair shared pointer typing, and use MonoTime for the benchmark harness so all three tgc tests compile and pass locally. --- druntime/test/gc/tgc.d | 6 ++++-- druntime/test/gc/tgc_bench.d | 19 +++++++------------ druntime/test/gc/tgc_regions.d | 14 +++++++++----- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index 7bef6df2c7c2..4b20d0310e54 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -40,12 +40,14 @@ void worker() void main() { import core.stdc.string : strcmp; - assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.2.0")); + auto ver = _d_tgc_version(); + assert(ver !is null && !strcmp(ver, "0.2.0")); // Shared region scaffold: create, attach, alloc auto rid = _d_tgc_region_create(); assert(rid != 0); - assert(_d_tgc_region_attach(rid)); + bool attached = _d_tgc_region_attach(rid); + assert(attached); auto rp = cast(int*) _d_tgc_region_malloc(rid, int.sizeof, 0); assert(rp !is null); *rp = 123; diff --git a/druntime/test/gc/tgc_bench.d b/druntime/test/gc/tgc_bench.d index da2c63dc2d56..8d0b087d4324 100644 --- a/druntime/test/gc/tgc_bench.d +++ b/druntime/test/gc/tgc_bench.d @@ -52,27 +52,24 @@ void main() const char* ver = _d_tgc_version(); if (ver && ver[0]) - printf("tgc version: %s\n", ver.ptr); + printf("tgc version: %s\n", ver); foreach (i; 0 .. 1000) auto w = new byte[128]; GC.collect(); - auto sw = StopWatch(AutoStart.yes); + MonoTime t0 = MonoTime.currTime; foreach (i; 0 .. iters) { auto p = new byte[64 + (i & 63)]; p[0] = cast(byte) i; } - sw.stop(); printf("single-thread alloc: %llu ms (%zu allocs)\n", - cast(ulong) sw.elapsed.total!"msecs", iters); + cast(ulong)(MonoTime.currTime - t0).total!"msecs", iters); - sw.reset(); - sw.start(); + t0 = MonoTime.currTime; GC.collect(); - sw.stop(); - printf("GC.collect pause: %llu ms\n", cast(ulong) sw.elapsed.total!"msecs"); + printf("GC.collect pause: %llu ms\n", cast(ulong)(MonoTime.currTime - t0).total!"msecs"); if (_d_tgc_version()[0]) { @@ -82,8 +79,7 @@ void main() Thread[] threads; threads.length = nThreads; - sw.reset(); - sw.start(); + t0 = MonoTime.currTime; foreach (i; 0 .. nThreads) { threads[i] = new Thread(&allocWorker); @@ -91,9 +87,8 @@ void main() } foreach (t; threads) t.join(); - sw.stop(); printf("multi-thread worker phase: %llu ms (%zu threads)\n", - cast(ulong) sw.elapsed.total!"msecs", nThreads); + cast(ulong)(MonoTime.currTime - t0).total!"msecs", nThreads); auto stats = GC.profileStats(); printf("collections: %llu\n", cast(ulong) stats.numCollections); diff --git a/druntime/test/gc/tgc_regions.d b/druntime/test/gc/tgc_regions.d index 0b049e5bb4f2..91ebf5901cf3 100644 --- a/druntime/test/gc/tgc_regions.d +++ b/druntime/test/gc/tgc_regions.d @@ -21,8 +21,9 @@ shared int* sharedCell; void worker() { - assert(_d_tgc_region_attach(regionId)); - sharedCell = cast(int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + bool attached = _d_tgc_region_attach(regionId); + assert(attached); + sharedCell = cast(shared int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); assert(sharedCell !is null); atomicStore(workerReady, true); @@ -35,11 +36,13 @@ void worker() void main() { - assert(_d_tgc_version() && !strcmp(_d_tgc_version(), "0.2.0")); + auto ver = _d_tgc_version(); + assert(ver !is null && !strcmp(ver, "0.2.0")); regionId = _d_tgc_region_create(); assert(regionId != 0); - assert(_d_tgc_region_attach(regionId)); + bool attached = _d_tgc_region_attach(regionId); + assert(attached); auto t = new Thread(&worker); t.start(); @@ -52,7 +55,8 @@ void main() assert(localRef !is null); *localRef = 42; - assert(_d_tgc_region_collect(regionId)); + bool collected = _d_tgc_region_collect(regionId); + assert(collected); atomicStore(collectDone, true); t.join(); From 63a13cae651571788bbb13dd32715b824ad4f263 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Fri, 4 Sep 2026 18:20:01 -0500 Subject: [PATCH 08/10] Improve tgc 0.2.1: O(log n) findBlock, smaller header, bounded fixpoint. Replace the intrusive block list with a sorted address index for interior-pointer lookup, drop next/prev from BlkHeader (32 bytes), and re-enable fixpoint marking with a pass cap that skips sweep on non-convergence. --- druntime/src/core/internal/gc/impl/tgc/gc.d | 253 +++++++++++++------- druntime/test/gc/tgc.d | 2 +- druntime/test/gc/tgc_regions.d | 2 +- 3 files changed, 171 insertions(+), 86 deletions(-) diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index e57465c9a19a..8ff1e8dc6c22 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -5,12 +5,13 @@ * (many-to-many). Collecting a region pauses only threads attached to that * region. See dlang-supplemental design notes for the full architecture. * - * **0.1.0:** private per-thread heaps, local collect, remote-free stub. + * **0.1.0:** private per-thread heaps, local collect, remote-free queue. * **0.1.1:** shared-region API scaffold. * **0.2.0:** region collect (selective suspend), heap locks, array append metadata. + * **0.2.x:** sorted-index findBlock O(log n), 32-byte header, bounded fixpoint mark. * * Cross-thread sharing on private heaps via remote free is interim, not the - * target model. Prefer attaching workers to a shared region once implemented. + * target model. Prefer attaching workers to a shared region. * * Select with `--DRT-gcopt=gc:tgc`. Informal side-name: "realtime GC". * @@ -20,7 +21,7 @@ module core.internal.gc.impl.tgc.gc; /// Semantic version of the `tgc` prototype (not druntime release version). -enum tgcVersion = "0.2.0"; +enum tgcVersion = "0.2.1"; import core.gc.gcinterface; @@ -48,26 +49,30 @@ extern (C) void* thread_stackBottom() nothrow @nogc; private enum size_t headerAlign = (void*).sizeof; private enum size_t collectThresholdInit = 256 * 1024; +/// Per-block metadata placed immediately before user payload. +/// 32 bytes on 64-bit (was 48 with intrusive list links). private struct BlkHeader { size_t size; /// user-visible capacity (alloc size) size_t arrayUsed; /// used bytes when BlkAttr.APPENDABLE (else 0) - uint attr; + uint attr; /// BlkAttr bits (user-visible) uint marked; /// non-zero when marked during collect ThreadHeap* heap; /// owning heap - BlkHeader* next; /// intrusive list in owner heap - BlkHeader* prev; } private struct ThreadHeap { - BlkHeader* head; + /// Address-sorted block index (by payload base). Replaces O(n) list walk. + BlkHeader** blocks; + size_t blockLen; + size_t blockCap; + size_t usedBytes; size_t allocatedTotal; /// bytes allocated on this thread since start size_t collectThreshold = collectThresholdInit; size_t numCollections; - // Remote frees pushed by other threads (ownership transfer). + // Remote frees pushed by other threads (ownership transfer). Implemented. void** remotePtrs; size_t remoteLen; size_t remoteCap; @@ -126,31 +131,75 @@ private struct ThreadHeap } } + /// Insert `h` into the address-sorted index. Caller holds listLock or is sole owner. + void indexInsert(BlkHeader* h) nothrow @nogc + { + if (blockLen == blockCap) + { + size_t ncap = blockCap ? blockCap * 2 : 8; + auto np = cast(BlkHeader**) cstdlib.realloc(blocks, ncap * (BlkHeader*).sizeof); + if (!np) + onOutOfMemoryError(); + blocks = np; + blockCap = ncap; + } + void* base = h + 1; + // Find first index with payload base >= base (insertion point). + size_t lo = 0, hi = blockLen; + while (lo < hi) + { + size_t mid = lo + (hi - lo) / 2; + if (cast(void*)(blocks[mid] + 1) < base) + lo = mid + 1; + else + hi = mid; + } + // Shift right from lo. + for (size_t i = blockLen; i > lo; --i) + blocks[i] = blocks[i - 1]; + blocks[lo] = h; + blockLen++; + usedBytes += h.size; + } + + /// Remove `h` from the address-sorted index. Caller holds listLock. + void indexRemove(BlkHeader* h) nothrow @nogc + { + void* base = h + 1; + size_t lo = 0, hi = blockLen; + while (lo < hi) + { + size_t mid = lo + (hi - lo) / 2; + auto mb = cast(void*)(blocks[mid] + 1); + if (mb < base) + lo = mid + 1; + else if (mb > base) + hi = mid; + else + { + for (size_t i = mid; i + 1 < blockLen; ++i) + blocks[i] = blocks[i + 1]; + blockLen--; + if (usedBytes >= h.size) + usedBytes -= h.size; + else + usedBytes = 0; + return; + } + } + } + void link(BlkHeader* h) nothrow @nogc { listLock.lock(); - h.prev = null; - h.next = head; - if (head) - head.prev = h; - head = h; - usedBytes += h.size; + indexInsert(h); listLock.unlock(); } void unlink(BlkHeader* h) nothrow @nogc { listLock.lock(); - if (h.prev) - h.prev.next = h.next; - else - head = h.next; - if (h.next) - h.next.prev = h.prev; - if (usedBytes >= h.size) - usedBytes -= h.size; - else - usedBytes = 0; + indexRemove(h); listLock.unlock(); } @@ -176,20 +225,31 @@ private struct ThreadHeap return cast(BlkHeader*) p - 1; } + /// O(log n) interior-pointer lookup via sorted payload bases. BlkHeader* findBlock(void* p) nothrow @nogc { if (!p) return null; listLock.lock(); scope (exit) listLock.unlock(); - size_t steps; - for (auto h = head; h && steps < maxBlkListWalk; h = h.next, ++steps) + if (!blockLen) + return null; + // Find rightmost block with payload base <= p. + size_t lo = 0, hi = blockLen; + while (lo < hi) { - void* base = h + 1; - void* end = base + h.size; - if (p >= base && p < end) - return h; + size_t mid = lo + (hi - lo) / 2; + if (cast(void*)(blocks[mid] + 1) <= p) + lo = mid + 1; + else + hi = mid; } + if (lo == 0) + return null; + auto h = blocks[lo - 1]; + void* base = h + 1; + if (p >= base && p < base + h.size) + return h; return null; } } @@ -313,8 +373,8 @@ private struct SharedRegion heap.drainRemote(); heap.listLock.lock(); - for (auto b = heap.head; b; b = b.next) - b.marked = 0; + foreach (i; 0 .. heap.blockLen) + heap.blocks[i].marked = 0; heap.listLock.unlock(); ThreadBase* tlist = null; @@ -354,8 +414,10 @@ private struct SharedRegion gc.markRangeHeap(heap, r.pbot, r.ptop); gc.rootsLock.unlock(); - gc.markHeapFixpoint(heap); - gc.sweepHeap(heap); + if (gc.markHeapFixpoint(heap)) + gc.sweepHeap(heap); + // If fixpoint did not converge, skip sweep (leak until next collect) rather + // than free possibly-reachable blocks. heap.numCollections++; gc.profileCollections++; heap.collecting = false; @@ -468,8 +530,6 @@ extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) not h.attr = bits; h.marked = 0; h.heap = r.heap; - h.next = null; - h.prev = null; r.heap.link(h); r.heap.allocatedTotal += size; return cast(void*)(h + 1); @@ -481,8 +541,6 @@ extern (C) const(char)* _d_tgc_version() nothrow @nogc } // TLS pointer to the calling thread's heap -private enum maxBlkListWalk = 1_000_000; - // TLS pointer to the calling thread's heap private ThreadHeap* tlsHeap; @@ -991,74 +1049,105 @@ class ThreadGC : GC markRangeInHeap(heap, pbot, ptop); } - package void markHeapFixpoint(ThreadHeap* heap) nothrow @nogc + /// Returns true if fixpoint converged (safe to sweep). + package bool markHeapFixpoint(ThreadHeap* heap) nothrow @nogc { size_t markedCount = 0; heap.listLock.lock(); - size_t steps; - for (auto b = heap.head; b && steps < maxBlkListWalk; b = b.next, ++steps) - if (b.marked) + foreach (i; 0 .. heap.blockLen) + if (heap.blocks[i].marked) markedCount++; heap.listLock.unlock(); size_t prevMarked = size_t.max; enum maxFixpointPasses = 256; + BlkHeader** work = null; + size_t workCap = 0; + for (uint pass = 0; pass < maxFixpointPasses && prevMarked != markedCount; ++pass) { prevMarked = markedCount; + // Snapshot scan candidates under lock; scan unlocked (findBlock takes lock). heap.listLock.lock(); - steps = 0; - for (auto b = heap.head; b && steps < maxBlkListWalk; b = b.next, ++steps) + size_t workLen = 0; + foreach (i; 0 .. heap.blockLen) { + auto b = heap.blocks[i]; if (!b.marked || (b.attr & BlkAttr.NO_SCAN)) continue; + if (workLen == workCap) + { + size_t ncap = workCap ? workCap * 2 : 8; + auto np = cast(BlkHeader**) cstdlib.realloc(work, ncap * (BlkHeader*).sizeof); + if (!np) + { + heap.listLock.unlock(); + cstdlib.free(work); + onOutOfMemoryError(); + } + work = np; + workCap = ncap; + } + work[workLen++] = b; + } + heap.listLock.unlock(); + + foreach (i; 0 .. workLen) + { + auto b = work[i]; void* base = b + 1; - markRangeInHeap(heap, base, base + b.size); + size_t scanLen = (b.attr & BlkAttr.APPENDABLE) && b.arrayUsed + ? b.arrayUsed : b.size; + markRangeInHeap(heap, base, base + scanLen); } + markedCount = 0; - steps = 0; - for (auto b = heap.head; b && steps < maxBlkListWalk; b = b.next, ++steps) - if (b.marked) + heap.listLock.lock(); + foreach (i; 0 .. heap.blockLen) + if (heap.blocks[i].marked) markedCount++; heap.listLock.unlock(); } + cstdlib.free(work); + return prevMarked == markedCount; } package void sweepHeap(ThreadHeap* heap) nothrow { - BlkHeader* doomed; + BlkHeader** doomed = null; + size_t doomedLen = 0; + size_t doomedCap = 0; heap.listLock.lock(); - auto b = heap.head; - while (b) + for (size_t i = heap.blockLen; i > 0; --i) { - auto next = b.next; - if (!b.marked) + auto b = heap.blocks[i - 1]; + if (b.marked) + continue; + heap.indexRemove(b); + if (doomedLen == doomedCap) { - if (b.prev) - b.prev.next = b.next; - else - heap.head = b.next; - if (b.next) - b.next.prev = b.prev; - if (heap.usedBytes >= b.size) - heap.usedBytes -= b.size; - else - heap.usedBytes = 0; - b.next = doomed; - doomed = b; + size_t ncap = doomedCap ? doomedCap * 2 : 8; + auto np = cast(BlkHeader**) cstdlib.realloc(doomed, ncap * (BlkHeader*).sizeof); + if (!np) + { + heap.listLock.unlock(); + onOutOfMemoryError(); + } + doomed = np; + doomedCap = ncap; } - b = next; + doomed[doomedLen++] = b; } heap.listLock.unlock(); - while (doomed) + foreach (i; 0 .. doomedLen) { - auto n = doomed.next; - if (doomed.attr & BlkAttr.FINALIZE) - rt_finalizeFromGC(doomed + 1, doomed.size, doomed.attr, null); - cstdlib.free(doomed); - doomed = n; + auto b = doomed[i]; + if (b.attr & BlkAttr.FINALIZE) + rt_finalizeFromGC(b + 1, b.size, b.attr, null); + cstdlib.free(b); } + cstdlib.free(doomed); } void initThread(ThreadBase t) nothrow @nogc @@ -1075,14 +1164,11 @@ class ThreadGC : GC return; // Free remaining blocks; do not leave memory owned by a dead thread. h.drainRemote(); - auto cur = h.head; - while (cur) + while (h.blockLen) { - auto n = cur.next; + auto cur = h.blocks[h.blockLen - 1]; h.unlinkAndFree(cur); - cur = n; } - h.head = null; unregisterHeap(h); if (tlsHeap is h) tlsHeap = null; @@ -1108,6 +1194,7 @@ class ThreadGC : GC } regionsLock.unlock(); cstdlib.free(h.remotePtrs); + cstdlib.free(h.blocks); cstdlib.free(h); } @@ -1172,8 +1259,6 @@ private: h.marked = 0; h.arrayUsed = (bits & BlkAttr.APPENDABLE) ? size : 0; h.heap = heap; - h.next = null; - h.prev = null; heap.link(h); heap.allocatedTotal += size; @@ -1192,8 +1277,8 @@ private: heap.drainRemote(); heap.listLock.lock(); - for (auto b = heap.head; b; b = b.next) - b.marked = 0; + foreach (i; 0 .. heap.blockLen) + heap.blocks[i].marked = 0; heap.listLock.unlock(); void* top; @@ -1223,9 +1308,9 @@ private: markRangeInHeap(heap, r.pbot, r.ptop); rootsLock.unlock(); - // TODO 0.2.x: fixpoint scan can loop on conservative marks; enable when precise. - // markHeapFixpoint(heap); - sweepHeap(heap); + if (markHeapFixpoint(heap)) + sweepHeap(heap); + // Non-converged fixpoint: skip sweep (safer than freeing live objects). heap.numCollections++; profileCollections++; diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index 4b20d0310e54..bfdc5ced1dd5 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -41,7 +41,7 @@ void main() { import core.stdc.string : strcmp; auto ver = _d_tgc_version(); - assert(ver !is null && !strcmp(ver, "0.2.0")); + assert(ver !is null && !strcmp(ver, "0.2.1")); // Shared region scaffold: create, attach, alloc auto rid = _d_tgc_region_create(); diff --git a/druntime/test/gc/tgc_regions.d b/druntime/test/gc/tgc_regions.d index 91ebf5901cf3..7c10959316ec 100644 --- a/druntime/test/gc/tgc_regions.d +++ b/druntime/test/gc/tgc_regions.d @@ -37,7 +37,7 @@ void worker() void main() { auto ver = _d_tgc_version(); - assert(ver !is null && !strcmp(ver, "0.2.0")); + assert(ver !is null && !strcmp(ver, "0.2.1")); regionId = _d_tgc_region_create(); assert(regionId != 0); From 0389d06ec963077db82dda84102f5121e7dc83e3 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Mon, 7 Sep 2026 18:25:58 -0500 Subject: [PATCH 09/10] Improve tgc 0.2.2 lookup and collector correctness. Co-authored-by: Cursor --- druntime/src/core/internal/gc/impl/tgc/gc.d | 178 +++++++++++++------- druntime/test/gc/Makefile | 3 +- druntime/test/gc/tgc.d | 93 +++++++++- druntime/test/gc/tgc_bench.d | 23 ++- druntime/test/gc/tgc_regions.d | 2 +- druntime/test/gc/tgc_remote_free.d | 55 ++++++ 6 files changed, 290 insertions(+), 64 deletions(-) create mode 100644 druntime/test/gc/tgc_remote_free.d diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index 8ff1e8dc6c22..cbcf8420e087 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -1,5 +1,5 @@ /** - * Opt-in thread-local garbage collector (`tgc`) — **0.2.0 prototype**. + * Opt-in thread-local garbage collector (`tgc`) — **0.2.2 prototype**. * * Target design: per-thread private heaps plus partitioned shared regions * (many-to-many). Collecting a region pauses only threads attached to that @@ -21,7 +21,7 @@ module core.internal.gc.impl.tgc.gc; /// Semantic version of the `tgc` prototype (not druntime release version). -enum tgcVersion = "0.2.1"; +enum tgcVersion = "0.2.2"; import core.gc.gcinterface; @@ -38,7 +38,7 @@ enum TgcSharedBackend : ubyte { tgcNative, symgc } private __gshared TgcSharedBackend tgcSharedBackend = TgcSharedBackend.tgcNative; import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; -import core.stdc.string : memcpy, memset; +import core.stdc.string : memcpy, memmove, memset; static import core.memory; extern (C) noreturn onOutOfMemoryError(void* pretend_sideffect = null, string file = __FILE__, size_t line = __LINE__) @trusted pure nothrow @nogc; /* dmd @@@BUG11461@@@ */ @@ -56,16 +56,20 @@ private struct BlkHeader size_t size; /// user-visible capacity (alloc size) size_t arrayUsed; /// used bytes when BlkAttr.APPENDABLE (else 0) uint attr; /// BlkAttr bits (user-visible) - uint marked; /// non-zero when marked during collect + uint marked; /// 0 white, 1 marked/unscanned, 2 marked/scanned ThreadHeap* heap; /// owning heap } +static assert(BlkHeader.sizeof == 32 || (void*).sizeof != 8); + private struct ThreadHeap { /// Address-sorted block index (by payload base). Replaces O(n) list walk. BlkHeader** blocks; size_t blockLen; size_t blockCap; + void* minAddr; + void* maxAddr; size_t usedBytes; size_t allocatedTotal; /// bytes allocated on this thread since start @@ -154,12 +158,15 @@ private struct ThreadHeap else hi = mid; } - // Shift right from lo. - for (size_t i = blockLen; i > lo; --i) - blocks[i] = blocks[i - 1]; + if (lo != blockLen) + memmove(blocks + lo + 1, blocks + lo, + (blockLen - lo) * (BlkHeader*).sizeof); blocks[lo] = h; blockLen++; usedBytes += h.size; + minAddr = blocks[0] + 1; + auto last = blocks[blockLen - 1]; + maxAddr = cast(void*)(last + 1) + last.size; } /// Remove `h` from the address-sorted index. Caller holds listLock. @@ -177,13 +184,22 @@ private struct ThreadHeap hi = mid; else { - for (size_t i = mid; i + 1 < blockLen; ++i) - blocks[i] = blocks[i + 1]; + if (mid + 1 != blockLen) + memmove(blocks + mid, blocks + mid + 1, + (blockLen - mid - 1) * (BlkHeader*).sizeof); blockLen--; if (usedBytes >= h.size) usedBytes -= h.size; else usedBytes = 0; + if (blockLen) + { + minAddr = blocks[0] + 1; + auto last = blocks[blockLen - 1]; + maxAddr = cast(void*)(last + 1) + last.size; + } + else + minAddr = maxAddr = null; return; } } @@ -225,15 +241,33 @@ private struct ThreadHeap return cast(BlkHeader*) p - 1; } - /// O(log n) interior-pointer lookup via sorted payload bases. + /// Layered interior-pointer lookup: + /// range reject O(1), linear for tiny heaps, binary predecessor otherwise. BlkHeader* findBlock(void* p) nothrow @nogc { if (!p) return null; listLock.lock(); scope (exit) listLock.unlock(); - if (!blockLen) + if (!blockLen || p < minAddr || p >= maxAddr) return null; + + // Linear wins for tiny arrays by avoiding branchy binary-search setup. + enum linearLookupLimit = 8; + if (blockLen <= linearLookupLimit) + { + foreach (i; 0 .. blockLen) + { + auto h = blocks[i]; + void* base = h + 1; + if (p < base) + return null; + if (p < base + h.size) + return h; + } + return null; + } + // Find rightmost block with payload base <= p. size_t lo = 0, hi = blockLen; while (lo < hi) @@ -399,8 +433,6 @@ private struct SharedRegion thread_scanList(tlist, n, (void* p1, void* p2) nothrow { gc.markRangeHeap(heap, p1, p2); }); - thread_resumeList(tlist, n); - cstdlib.free(tlist); } gc.rootsLock.lock(); @@ -418,6 +450,11 @@ private struct SharedRegion gc.sweepHeap(heap); // If fixpoint did not converge, skip sweep (leak until next collect) rather // than free possibly-reachable blocks. + if (n) + { + thread_resumeList(tlist, n); + cstdlib.free(tlist); + } heap.numCollections++; gc.profileCollections++; heap.collecting = false; @@ -527,6 +564,7 @@ extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) not memset(raw, 0, BlkHeader.sizeof); auto h = cast(BlkHeader*) raw; h.size = size; + h.arrayUsed = (bits & BlkAttr.APPENDABLE) ? size : 0; h.attr = bits; h.marked = 0; h.heap = r.heap; @@ -990,40 +1028,57 @@ class ThreadGC : GC auto blk = queryBlock(ptr); if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) return null; - auto used = blk.arrayUsed ? blk.arrayUsed : blk.size; + auto heap = blk.heap; + heap.listLock.lock(); + auto used = blk.arrayUsed; + heap.listLock.unlock(); return (cast(void*)(blk + 1))[0 .. used]; } bool expandArrayUsed(void[] slice, size_t newUsed, bool atomic = false) nothrow @trusted { - if (!slice.length) + if (!slice.ptr || newUsed < slice.length) return false; - auto blk = queryBlock(&slice[0]); + auto blk = queryBlock(slice.ptr); if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) return false; - if (newUsed > blk.size) + auto base = cast(void*)(blk + 1); + size_t offset = slice.ptr - base; + if (offset > blk.size || newUsed > blk.size - offset) return false; - blk.arrayUsed = newUsed; + auto heap = blk.heap; + heap.listLock.lock(); + if (offset + slice.length != blk.arrayUsed) + { + heap.listLock.unlock(); + return false; + } + blk.arrayUsed = offset + newUsed; + heap.listLock.unlock(); return true; } size_t reserveArrayCapacity(void[] slice, size_t request, bool atomic = false) nothrow @trusted { - if (!slice.length || !request) + if (!slice.ptr) return 0; - auto blk = queryBlock(&slice[0]); + auto blk = queryBlock(slice.ptr); if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) return 0; - if (request <= blk.size) - return blk.size; - auto bits = blk.attr; - auto oldUsed = blk.arrayUsed ? blk.arrayUsed : slice.length; - auto np = alloc(request, bits, false); - memcpy(np, &slice[0], oldUsed < slice.length ? oldUsed : slice.length); - free(&slice[0]); - auto nblk = headerOf(np); - nblk.arrayUsed = oldUsed; - return request; + auto base = cast(void*)(blk + 1); + size_t offset = slice.ptr - base; + if (offset > blk.size || slice.length > blk.size - offset) + return 0; + auto heap = blk.heap; + heap.listLock.lock(); + bool isTail = offset + slice.length == blk.arrayUsed; + size_t capacity = isTail && request <= blk.size - offset + ? blk.size - offset : 0; + heap.listLock.unlock(); + // This malloc-backed prototype cannot extend in place. Returning zero + // makes the array runtime allocate/copy safely instead of retaining a + // pointer to storage that reserve moved behind its back. + return capacity; } bool shrinkArrayUsed(void[] slice, size_t existingUsed, bool atomic = false) nothrow @@ -1033,9 +1088,21 @@ class ThreadGC : GC auto blk = queryBlock(slice.ptr); if (!blk || !(blk.attr & BlkAttr.APPENDABLE)) return false; - if (existingUsed > blk.size) + if (existingUsed < slice.length) + return false; + auto base = cast(void*)(blk + 1); + size_t offset = slice.ptr - base; + if (offset > blk.size || existingUsed > blk.size - offset) return false; - blk.arrayUsed = existingUsed; + auto heap = blk.heap; + heap.listLock.lock(); + if (offset + existingUsed != blk.arrayUsed) + { + heap.listLock.unlock(); + return false; + } + blk.arrayUsed = offset + slice.length; + heap.listLock.unlock(); return true; } @@ -1052,28 +1119,23 @@ class ThreadGC : GC /// Returns true if fixpoint converged (safe to sweep). package bool markHeapFixpoint(ThreadHeap* heap) nothrow @nogc { - size_t markedCount = 0; - heap.listLock.lock(); - foreach (i; 0 .. heap.blockLen) - if (heap.blocks[i].marked) - markedCount++; - heap.listLock.unlock(); - - size_t prevMarked = size_t.max; enum maxFixpointPasses = 256; BlkHeader** work = null; size_t workCap = 0; - for (uint pass = 0; pass < maxFixpointPasses && prevMarked != markedCount; ++pass) + foreach (pass; 0 .. maxFixpointPasses) { - prevMarked = markedCount; - // Snapshot scan candidates under lock; scan unlocked (findBlock takes lock). + // Snapshot newly marked candidates under lock; scan unlocked because + // findBlock takes the same lock. State 2 prevents rescanning blocks. heap.listLock.lock(); size_t workLen = 0; foreach (i; 0 .. heap.blockLen) { auto b = heap.blocks[i]; - if (!b.marked || (b.attr & BlkAttr.NO_SCAN)) + if (b.marked != 1) + continue; + b.marked = 2; + if (b.attr & BlkAttr.NO_SCAN) continue; if (workLen == workCap) { @@ -1100,16 +1162,17 @@ class ThreadGC : GC ? b.arrayUsed : b.size; markRangeInHeap(heap, base, base + scanLen); } - - markedCount = 0; - heap.listLock.lock(); - foreach (i; 0 .. heap.blockLen) - if (heap.blocks[i].marked) - markedCount++; - heap.listLock.unlock(); + if (!workLen) + { + cstdlib.free(work); + return true; + } } + + // A pointer chain deeper than the safety bound remains live. Skip sweep + // rather than risk freeing a block not reached yet. cstdlib.free(work); - return prevMarked == markedCount; + return false; } package void sweepHeap(ThreadHeap* heap) nothrow @@ -1346,17 +1409,12 @@ private: { if (!pbot || !ptop || pbot >= ptop) return; - enum maxScanBytes = 4 * 1024 * 1024; - auto scanTop = ptop; - if (cast(size_t)(scanTop - pbot) > maxScanBytes) - scanTop = pbot + maxScanBytes; auto p = cast(void**) pbot; - auto e = cast(void**) scanTop; + auto e = cast(void**) ptop; auto addr = cast(size_t) p; addr = (addr + (void*).sizeof - 1) & ~((void*).sizeof - 1); p = cast(void**) addr; - size_t steps; - for (; p + 1 <= e && steps < maxScanBytes / (void*).sizeof; ++p, ++steps) + for (; p + 1 <= e; ++p) markPtrInHeap(heap, *p); } @@ -1365,7 +1423,7 @@ private: if (!p) return; auto b = heap.findBlock(p); - if (b) + if (b && b.marked == 0) b.marked = 1; } diff --git a/druntime/test/gc/Makefile b/druntime/test/gc/Makefile index 1206ddb83021..ffce9d231914 100644 --- a/druntime/test/gc/Makefile +++ b/druntime/test/gc/Makefile @@ -1,6 +1,6 @@ TESTS:=attributes sentinel printf memstomp invariant logging \ precise precisegc \ - recoverfree collect nocollect tgc tgc_regions tgc_bench + recoverfree collect nocollect tgc tgc_regions tgc_remote_free tgc_bench ifneq ($(OS),windows) # some .d files are for Posix only @@ -88,6 +88,7 @@ $(ROOT)/precise_concurrent.done: run_args+="--DRT-gcopt=gc:precise fork:1" $(ROOT)/attributes$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/tgc.done: run_args+=--DRT-gcopt=gc:tgc $(ROOT)/tgc_regions.done: run_args+=--DRT-gcopt=gc:tgc +$(ROOT)/tgc_remote_free.done: run_args+=--DRT-gcopt=gc:tgc $(ROOT)/tgc_bench.done: run_args+=--DRT-gcopt=gc:tgc $(ROOT)/forkgc$(DOTEXE): extra_dflags += $(core_ut) $(ROOT)/sigmaskgc$(DOTEXE): extra_dflags += $(core_ut) diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index bfdc5ced1dd5..9bb0d2e377ea 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -6,6 +6,8 @@ import core.memory; import core.thread; import core.atomic; +import cstdlib = core.stdc.stdlib; +import core.stdc.string : memset; extern (C) uint _d_tgc_region_create() nothrow @nogc; extern (C) bool _d_tgc_region_attach(uint regionId) nothrow @nogc; @@ -16,8 +18,54 @@ shared size_t otherThreadAllocs; shared bool otherDone; shared bool collectDone; +class ChainNode +{ + ChainNode next; + size_t value; +} + +pragma(inline, false) +ChainNode makeChain(size_t length) +{ + ChainNode head; + foreach_reverse (i; 0 .. length) + { + auto node = new ChainNode; + node.next = head; + node.value = i; + head = node; + } + return head; +} + +pragma(inline, false) +void installTailRoot(void* range, size_t bytes) +{ + auto target = new ubyte[256]; + target[0] = 0x5A; + *cast(void**)(range + bytes - (void*).sizeof) = target.ptr; +} + +pragma(inline, false) +void clobberStack() +{ + void*[8192] zeros; + zeros[] = null; +} + void worker() { + // A fresh worker heap exercises the tiny-index linear lookup tier. + void*[4] tinyBlocks; + foreach (ref p; tinyBlocks) + p = GC.malloc(32, GC.BlkAttr.NO_SCAN); + foreach (p; tinyBlocks) + { + auto found = GC.addrOf(p + 7); + assert(found == p); + GC.free(p); + } + // Allocate on this thread's private heap foreach (i; 0 .. 100) { @@ -41,7 +89,7 @@ void main() { import core.stdc.string : strcmp; auto ver = _d_tgc_version(); - assert(ver !is null && !strcmp(ver, "0.2.1")); + assert(ver !is null && !strcmp(ver, "0.2.2")); // Shared region scaffold: create, attach, alloc auto rid = _d_tgc_region_create(); @@ -53,6 +101,24 @@ void main() *rp = 123; assert(*rp == 123); + // Exercise both lookup tiers and interior-pointer handling. + void*[16] blocks; + foreach (i; 0 .. blocks.length) + blocks[i] = GC.malloc(64, GC.BlkAttr.NO_SCAN); + foreach (p; blocks) + { + auto interior = p + 31; + auto found = GC.addrOf(interior); + assert(found == p); + } + auto removed = blocks[7]; + GC.free(removed); + auto removedLookup = GC.addrOf(removed + 1); + assert(removedLookup is null); + blocks[7] = null; + foreach (p; blocks) + GC.free(p); + auto before = GC.profileStats().numCollections; // Local allocations @@ -61,6 +127,31 @@ void main() local ~= cast(int) i; assert(local.length == 50); + // A heap pointer chain requires fixpoint marking beyond direct roots. + auto chain = makeChain(64); + GC.collect(); + size_t chainLength; + for (auto node = chain; node; node = node.next) + { + assert(node.value == chainLength); + chainLength++; + } + assert(chainLength == 64); + + // The sole deliberate root is beyond the old 4 MiB scan cutoff. + enum registeredBytes = 5 * 1024 * 1024; + auto registered = cstdlib.malloc(registeredBytes); + assert(registered !is null); + memset(registered, 0, registeredBytes); + GC.addRange(registered, registeredBytes); + installTailRoot(registered, registeredBytes); + clobberStack(); + GC.collect(); + auto tailRoot = *cast(void**)(registered + registeredBytes - (void*).sizeof); + assert((cast(ubyte*) tailRoot)[0] == 0x5A); + GC.removeRange(registered); + cstdlib.free(registered); + auto t = new Thread(&worker); t.start(); diff --git a/druntime/test/gc/tgc_bench.d b/druntime/test/gc/tgc_bench.d index 8d0b087d4324..78857c30b82f 100644 --- a/druntime/test/gc/tgc_bench.d +++ b/druntime/test/gc/tgc_bench.d @@ -6,7 +6,8 @@ * tgc_bench --DRT-gcopt=gc:tgc * tgc_bench --DRT-gcopt=gc:tgc tgcShared:symgc * - * Environment: TGC_BENCH_ITERS (default 50000), TGC_BENCH_THREADS (default 4) + * Environment: TGC_BENCH_ITERS (default 50000), TGC_BENCH_THREADS (default 4), + * TGC_BENCH_LOOKUPS (default 200000) */ import core.memory; import core.thread; @@ -45,10 +46,13 @@ void main() { size_t iters = 50_000; size_t nThreads = 4; + size_t lookups = 200_000; if (const v = getenv("TGC_BENCH_ITERS")) iters = cast(size_t) atol(v); if (const v = getenv("TGC_BENCH_THREADS")) nThreads = cast(size_t) atol(v); + if (const v = getenv("TGC_BENCH_LOOKUPS")) + lookups = cast(size_t) atol(v); const char* ver = _d_tgc_version(); if (ver && ver[0]) @@ -71,6 +75,23 @@ void main() GC.collect(); printf("GC.collect pause: %llu ms\n", cast(ulong)(MonoTime.currTime - t0).total!"msecs"); + enum lookupBlockCount = 4096; + void*[lookupBlockCount] lookupBlocks; + foreach (ref p; lookupBlocks) + p = GC.malloc(64, GC.BlkAttr.NO_SCAN); + t0 = MonoTime.currTime; + foreach (i; 0 .. lookups) + { + auto base = lookupBlocks[i % lookupBlockCount]; + auto found = GC.addrOf(base + (i & 63)); + assert(found == base); + } + printf("interior-pointer lookup: %llu ms (%zu lookups, %u blocks)\n", + cast(ulong)(MonoTime.currTime - t0).total!"msecs", + lookups, lookupBlockCount); + foreach (p; lookupBlocks) + GC.free(p); + if (_d_tgc_version()[0]) { benchRegion = _d_tgc_region_create(); diff --git a/druntime/test/gc/tgc_regions.d b/druntime/test/gc/tgc_regions.d index 7c10959316ec..04b7fbc38965 100644 --- a/druntime/test/gc/tgc_regions.d +++ b/druntime/test/gc/tgc_regions.d @@ -37,7 +37,7 @@ void worker() void main() { auto ver = _d_tgc_version(); - assert(ver !is null && !strcmp(ver, "0.2.1")); + assert(ver !is null && !strcmp(ver, "0.2.2")); regionId = _d_tgc_region_create(); assert(regionId != 0); diff --git a/druntime/test/gc/tgc_remote_free.d b/druntime/test/gc/tgc_remote_free.d new file mode 100644 index 000000000000..0e4f6b4a55cd --- /dev/null +++ b/druntime/test/gc/tgc_remote_free.d @@ -0,0 +1,55 @@ +/** + * Cross-thread remote-free queue test for tgc. + * + * Run with: --DRT-gcopt=gc:tgc + */ +import core.atomic; +import core.memory; +import core.thread; + +shared size_t remoteAddress; +shared bool freeQueued; +shared bool queueDrained; +shared bool lookupDone; + +void ownerThread() +{ + auto owned = GC.malloc(128, GC.BlkAttr.NO_SCAN); + assert(owned !is null); + atomicStore(remoteAddress, cast(size_t) owned); + + while (!atomicLoad(freeQueued)) + Thread.yield(); + + // Any owner allocation drains frees queued by foreign threads. + auto trigger = GC.malloc(1, GC.BlkAttr.NO_SCAN); + assert(trigger !is null); + atomicStore(queueDrained, true); + + while (!atomicLoad(lookupDone)) + Thread.yield(); +} + +void main() +{ + auto owner = new Thread(&ownerThread); + owner.start(); + + size_t address; + while (!address) + { + address = atomicLoad(remoteAddress); + Thread.yield(); + } + + auto foreign = cast(void*) address; + GC.free(foreign); + atomicStore(freeQueued, true); + + while (!atomicLoad(queueDrained)) + Thread.yield(); + auto found = GC.addrOf(foreign); + assert(found is null); + atomicStore(lookupDone, true); + owner.join(); +} From 67d2e1b7d99bef3ec2976294c5c0dc6e6c8b4495 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Mon, 7 Sep 2026 18:41:15 -0500 Subject: [PATCH 10/10] Harden tgc 0.2.3 after concurrency audit. Co-authored-by: Cursor --- druntime/src/core/internal/gc/impl/tgc/gc.d | 215 +++++++++++++++----- druntime/test/gc/tgc.d | 31 ++- druntime/test/gc/tgc_regions.d | 18 +- druntime/test/gc/tgc_remote_free.d | 1 + 4 files changed, 209 insertions(+), 56 deletions(-) diff --git a/druntime/src/core/internal/gc/impl/tgc/gc.d b/druntime/src/core/internal/gc/impl/tgc/gc.d index cbcf8420e087..26baa1e595e3 100644 --- a/druntime/src/core/internal/gc/impl/tgc/gc.d +++ b/druntime/src/core/internal/gc/impl/tgc/gc.d @@ -1,5 +1,5 @@ /** - * Opt-in thread-local garbage collector (`tgc`) — **0.2.2 prototype**. + * Opt-in thread-local garbage collector (`tgc`) — **0.2.3 prototype**. * * Target design: per-thread private heaps plus partitioned shared regions * (many-to-many). Collecting a region pauses only threads attached to that @@ -21,7 +21,7 @@ module core.internal.gc.impl.tgc.gc; /// Semantic version of the `tgc` prototype (not druntime release version). -enum tgcVersion = "0.2.2"; +enum tgcVersion = "0.2.3"; import core.gc.gcinterface; @@ -48,6 +48,8 @@ extern (C) void* thread_stackBottom() nothrow @nogc; private enum size_t headerAlign = (void*).sizeof; private enum size_t collectThresholdInit = 256 * 1024; +private enum uint markMask = 0x3; +private enum uint remoteQueuedBit = 0x4; /// Per-block metadata placed immediately before user payload. /// 32 bytes on 64-bit (was 48 with intrusive list links). @@ -56,7 +58,7 @@ private struct BlkHeader size_t size; /// user-visible capacity (alloc size) size_t arrayUsed; /// used bytes when BlkAttr.APPENDABLE (else 0) uint attr; /// BlkAttr bits (user-visible) - uint marked; /// 0 white, 1 marked/unscanned, 2 marked/scanned + uint marked; /// low bits: mark state; remoteQueuedBit: pending free ThreadHeap* heap; /// owning heap } @@ -97,8 +99,17 @@ private struct ThreadHeap return h; } - void pushRemote(void* p) nothrow @nogc + bool queueRemote(void* p) nothrow @nogc { + listLock.lock(); + auto h = findBlockUnlocked(p); + if (!h || cast(void*)(h + 1) !is p || (h.marked & remoteQueuedBit)) + { + listLock.unlock(); + return false; + } + h.marked |= remoteQueuedBit; + remoteLock.lock(); if (remoteLen == remoteCap) { @@ -107,6 +118,7 @@ private struct ThreadHeap if (!np) { remoteLock.unlock(); + listLock.unlock(); onOutOfMemoryError(); } remotePtrs = np; @@ -114,6 +126,8 @@ private struct ThreadHeap } remotePtrs[remoteLen++] = p; remoteLock.unlock(); + listLock.unlock(); + return true; } void drainRemote() nothrow @nogc @@ -121,7 +135,9 @@ private struct ThreadHeap remoteLock.lock(); size_t n = remoteLen; void** ptrs = remotePtrs; + remotePtrs = null; remoteLen = 0; + remoteCap = 0; remoteLock.unlock(); foreach (i; 0 .. n) @@ -129,10 +145,18 @@ private struct ThreadHeap auto p = ptrs[i]; if (!p) continue; - auto h = headerOf(p); - if (h && h.heap is &this) - unlinkAndFree(h); + listLock.lock(); + auto h = findBlockUnlocked(p); + if (h && cast(void*)(h + 1) is p && (h.marked & remoteQueuedBit)) + { + indexRemove(h); + listLock.unlock(); + cstdlib.free(h); + } + else + listLock.unlock(); } + cstdlib.free(ptrs); } /// Insert `h` into the address-sorted index. Caller holds listLock or is sole owner. @@ -226,6 +250,21 @@ private struct ThreadHeap cstdlib.free(h); } + bool freeExact(void* p) nothrow @nogc + { + listLock.lock(); + auto h = findBlockUnlocked(p); + if (!h || cast(void*)(h + 1) !is p) + { + listLock.unlock(); + return false; + } + indexRemove(h); + listLock.unlock(); + cstdlib.free(h); + return true; + } + void unlinkAndFreeFinalize(BlkHeader* h) nothrow { unlink(h); @@ -249,6 +288,12 @@ private struct ThreadHeap return null; listLock.lock(); scope (exit) listLock.unlock(); + return findBlockUnlocked(p); + } + + /// Caller holds listLock. + BlkHeader* findBlockUnlocked(void* p) nothrow @nogc + { if (!blockLen || p < minAddr || p >= maxAddr) return null; @@ -397,45 +442,61 @@ private struct SharedRegion void collectRegion() nothrow { - if (!heap || heap.collecting) + if (!heap) return; auto gc = cast(ThreadGC) tgcInstance; if (!gc) return; + // Establish lock barriers before suspension so no member can be + // frozen while owning a lock needed by the collector. Keep the region + // lock through resume to serialize collectors, attach/detach, and + // allocation admission. + gc.rootsLock.lock(); + lock.lock(); + if (heap.collecting) + { + lock.unlock(); + gc.rootsLock.unlock(); + return; + } heap.collecting = true; heap.drainRemote(); heap.listLock.lock(); foreach (i; 0 .. heap.blockLen) - heap.blocks[i].marked = 0; - heap.listLock.unlock(); + heap.blocks[i].marked &= remoteQueuedBit; ThreadBase* tlist = null; - size_t n = 0; - lock.lock(); - n = memberLen; + size_t n = memberLen; if (n) { tlist = cast(ThreadBase*) cstdlib.malloc(n * ThreadBase.sizeof); if (!tlist) - { - lock.unlock(); onOutOfMemoryError(); - } memcpy(tlist, memberThreads, n * ThreadBase.sizeof); + // Barrier every member-private index before suspension. Once the + // members stop, releasing these locks leaves stable indexes for + // region-root scanning. + foreach (i; 0 .. n) + memberHeaps[i].listLock.lock(); } - lock.unlock(); if (n) - { thread_suspendList(tlist, n); + heap.listLock.unlock(); + foreach (i; 0 .. n) + memberHeaps[i].listLock.unlock(); + + if (n) + { thread_scanList(tlist, n, (void* p1, void* p2) nothrow { gc.markRangeHeap(heap, p1, p2); }); + foreach (i; 0 .. n) + gc.markHeapContentsInto(memberHeaps[i], heap); } - gc.rootsLock.lock(); foreach (ref r; gc.roots) { if (r.proot) @@ -448,8 +509,6 @@ private struct SharedRegion if (gc.markHeapFixpoint(heap)) gc.sweepHeap(heap); - // If fixpoint did not converge, skip sweep (leak until next collect) rather - // than free possibly-reachable blocks. if (n) { thread_resumeList(tlist, n); @@ -458,6 +517,7 @@ private struct SharedRegion heap.numCollections++; gc.profileCollections++; heap.collecting = false; + lock.unlock(); } } @@ -557,6 +617,8 @@ extern (C) void* _d_tgc_region_malloc(uint regionId, size_t size, uint bits) not r.lock.unlock(); r.heap.drainRemote(); + if (size > size_t.max - BlkHeader.sizeof) + onOutOfMemoryError(); size_t total = BlkHeader.sizeof + size; auto raw = cstdlib.malloc(total); if (size && raw is null) @@ -770,13 +832,13 @@ class ThreadGC : GC uint getAttr(void* p) nothrow { auto blk = queryBlock(p); - return blk ? blk.attr : 0; + return isExactBase(blk, p) ? blk.attr : 0; } uint setAttr(void* p, uint mask) nothrow { auto blk = queryBlock(p); - if (!blk) + if (!isExactBase(blk, p)) return 0; blk.attr |= mask; return blk.attr; @@ -785,7 +847,7 @@ class ThreadGC : GC uint clrAttr(void* p, uint mask) nothrow { auto blk = queryBlock(p); - if (!blk) + if (!isExactBase(blk, p)) return 0; blk.attr &= ~mask; return blk.attr; @@ -821,11 +883,8 @@ class ThreadGC : GC } auto blk = queryBlock(p); - if (!blk) - { - // Unknown pointer — allocate fresh - return alloc(size, bits, false); - } + if (!isExactBase(blk, p)) + return null; auto heap = blk.heap; if (heap !is currentHeap()) @@ -840,9 +899,17 @@ class ThreadGC : GC if (size <= blk.size) { + auto oldSize = blk.size; + heap.listLock.lock(); blk.size = size; + if (blk.arrayUsed > size) + blk.arrayUsed = size; if (bits) blk.attr = bits; + heap.usedBytes -= oldSize - size; + if (heap.blockLen && heap.blocks[heap.blockLen - 1] is blk) + heap.maxAddr = cast(void*)(blk + 1) + size; + heap.listLock.unlock(); return p; } @@ -866,17 +933,41 @@ class ThreadGC : GC { if (!p) return; - auto blk = queryBlock(p); - if (!blk) - return; - auto owner = blk.heap; + auto local = tlsHeap; - if (isSharedRegionHeap(owner) || owner is local || local is null) - { - owner.unlinkAndFree(blk); + if (local && local.freeExact(p)) return; + + regionsLock.lock(); + foreach (i; 0 .. allRegionsLen) + { + auto r = allRegions[i]; + if (!r || !r.heap) + continue; + r.lock.lock(); + bool freed = r.heap.freeExact(p); + r.lock.unlock(); + if (freed) + { + regionsLock.unlock(); + return; + } + } + regionsLock.unlock(); + + // Keep the registry lock until the owner has accepted the request. + // cleanupThread unregisters before destroying a heap. + heapsLock.lock(); + foreach (i; 0 .. allHeapsLen) + { + auto owner = allHeaps[i]; + if (owner !is local && owner.queueRemote(p)) + { + heapsLock.unlock(); + return; + } } - owner.pushRemote(p); + heapsLock.unlock(); } void* addrOf(void* p) nothrow @nogc @@ -888,7 +979,7 @@ class ThreadGC : GC size_t sizeOf(void* p) nothrow @nogc { auto blk = queryBlock(p); - return blk ? blk.size : 0; + return isExactBase(blk, p) ? blk.size : 0; } BlkInfo query(void* p) nothrow @@ -1116,14 +1207,31 @@ class ThreadGC : GC markRangeInHeap(heap, pbot, ptop); } + package void markHeapContentsInto(ThreadHeap* source, ThreadHeap* target) nothrow @nogc + { + if (!source || !target) + return; + source.listLock.lock(); + foreach (i; 0 .. source.blockLen) + { + auto b = source.blocks[i]; + if (b.attr & BlkAttr.NO_SCAN) + continue; + auto base = cast(void*)(b + 1); + size_t scanLen = (b.attr & BlkAttr.APPENDABLE) && b.arrayUsed + ? b.arrayUsed : b.size; + markRangeInHeap(target, base, base + scanLen); + } + source.listLock.unlock(); + } + /// Returns true if fixpoint converged (safe to sweep). package bool markHeapFixpoint(ThreadHeap* heap) nothrow @nogc { - enum maxFixpointPasses = 256; BlkHeader** work = null; size_t workCap = 0; - foreach (pass; 0 .. maxFixpointPasses) + while (true) { // Snapshot newly marked candidates under lock; scan unlocked because // findBlock takes the same lock. State 2 prevents rescanning blocks. @@ -1132,9 +1240,9 @@ class ThreadGC : GC foreach (i; 0 .. heap.blockLen) { auto b = heap.blocks[i]; - if (b.marked != 1) + if ((b.marked & markMask) != 1) continue; - b.marked = 2; + b.marked = (b.marked & ~markMask) | 2; if (b.attr & BlkAttr.NO_SCAN) continue; if (workLen == workCap) @@ -1169,10 +1277,6 @@ class ThreadGC : GC } } - // A pointer chain deeper than the safety bound remains live. Skip sweep - // rather than risk freeing a block not reached yet. - cstdlib.free(work); - return false; } package void sweepHeap(ThreadHeap* heap) nothrow @@ -1184,7 +1288,7 @@ class ThreadGC : GC for (size_t i = heap.blockLen; i > 0; --i) { auto b = heap.blocks[i - 1]; - if (b.marked) + if (b.marked & markMask) continue; heap.indexRemove(b); if (doomedLen == doomedCap) @@ -1225,14 +1329,14 @@ class ThreadGC : GC auto h = cast(ThreadHeap*) t.tlsGCData(); if (!h) return; - // Free remaining blocks; do not leave memory owned by a dead thread. + // Stop new foreign lookups before draining and destroying this heap. + unregisterHeap(h); h.drainRemote(); while (h.blockLen) { auto cur = h.blocks[h.blockLen - 1]; h.unlinkAndFree(cur); } - unregisterHeap(h); if (tlsHeap is h) tlsHeap = null; t.tlsGCData() = null; @@ -1300,6 +1404,11 @@ private: return null; } + static bool isExactBase(BlkHeader* blk, void* p) nothrow @nogc + { + return blk !is null && cast(void*)(blk + 1) is p; + } + void* alloc(size_t size, uint bits, bool zero) nothrow { auto heap = currentHeap(); @@ -1308,6 +1417,8 @@ private: if (!disabled && heap.usedBytes >= heap.collectThreshold) collectHeap(heap); + if (size > size_t.max - BlkHeader.sizeof) + onOutOfMemoryError(); size_t total = BlkHeader.sizeof + size; // Align user payload auto raw = zero ? cstdlib.calloc(1, total) : cstdlib.malloc(total); @@ -1341,7 +1452,7 @@ private: heap.listLock.lock(); foreach (i; 0 .. heap.blockLen) - heap.blocks[i].marked = 0; + heap.blocks[i].marked &= remoteQueuedBit; heap.listLock.unlock(); void* top; @@ -1423,8 +1534,8 @@ private: if (!p) return; auto b = heap.findBlock(p); - if (b && b.marked == 0) - b.marked = 1; + if (b && !(b.marked & markMask)) + b.marked = (b.marked & ~markMask) | 1; } static BlkHeader* headerOf(void* p) nothrow @nogc diff --git a/druntime/test/gc/tgc.d b/druntime/test/gc/tgc.d index 9bb0d2e377ea..7eebaf50dddc 100644 --- a/druntime/test/gc/tgc.d +++ b/druntime/test/gc/tgc.d @@ -6,6 +6,7 @@ import core.memory; import core.thread; import core.atomic; +import core.exception : OutOfMemoryError; import cstdlib = core.stdc.stdlib; import core.stdc.string : memset; @@ -89,7 +90,7 @@ void main() { import core.stdc.string : strcmp; auto ver = _d_tgc_version(); - assert(ver !is null && !strcmp(ver, "0.2.2")); + assert(ver !is null && !strcmp(ver, "0.2.3")); // Shared region scaffold: create, attach, alloc auto rid = _d_tgc_region_create(); @@ -110,6 +111,19 @@ void main() auto interior = p + 31; auto found = GC.addrOf(interior); assert(found == p); + assert(GC.sizeOf(interior) == 0); + assert(GC.getAttr(interior) == 0); + auto setResult = GC.setAttr(interior, GC.BlkAttr.NO_MOVE); + auto clearResult = GC.clrAttr(interior, GC.BlkAttr.NO_SCAN); + assert(setResult == 0); + assert(clearResult == 0); + auto resized = GC.realloc(interior, 128); + assert(resized is null); + GC.free(interior); + auto stillLive = GC.addrOf(p); + assert(stillLive == p); + auto atEnd = GC.addrOf(p + 64); + assert(atEnd is null); } auto removed = blocks[7]; GC.free(removed); @@ -119,6 +133,17 @@ void main() foreach (p; blocks) GC.free(p); + bool overflowRejected; + try + { + auto impossible = GC.malloc(size_t.max); + if (impossible) + GC.free(impossible); + } + catch (OutOfMemoryError) + overflowRejected = true; + assert(overflowRejected); + auto before = GC.profileStats().numCollections; // Local allocations @@ -128,7 +153,7 @@ void main() assert(local.length == 50); // A heap pointer chain requires fixpoint marking beyond direct roots. - auto chain = makeChain(64); + auto chain = makeChain(300); GC.collect(); size_t chainLength; for (auto node = chain; node; node = node.next) @@ -136,7 +161,7 @@ void main() assert(node.value == chainLength); chainLength++; } - assert(chainLength == 64); + assert(chainLength == 300); // The sole deliberate root is beyond the old 4 MiB scan cutoff. enum registeredBytes = 5 * 1024 * 1024; diff --git a/druntime/test/gc/tgc_regions.d b/druntime/test/gc/tgc_regions.d index 04b7fbc38965..7946b13c7350 100644 --- a/druntime/test/gc/tgc_regions.d +++ b/druntime/test/gc/tgc_regions.d @@ -19,12 +19,21 @@ shared bool workerReady; shared bool collectDone; shared int* sharedCell; +class RegionHolder +{ + shared int* cell; +} + void worker() { bool attached = _d_tgc_region_attach(regionId); assert(attached); sharedCell = cast(shared int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); assert(sharedCell !is null); + auto holder = new RegionHolder; + holder.cell = cast(shared int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + assert(holder.cell !is null); + *holder.cell = 77; atomicStore(workerReady, true); while (!atomicLoad(collectDone)) @@ -32,12 +41,13 @@ void worker() // Must still be valid after region collect from main (root on main after join setup) assert(*sharedCell == 42); + assert(*holder.cell == 77); } void main() { auto ver = _d_tgc_version(); - assert(ver !is null && !strcmp(ver, "0.2.2")); + assert(ver !is null && !strcmp(ver, "0.2.3")); regionId = _d_tgc_region_create(); assert(regionId != 0); @@ -57,6 +67,12 @@ void main() bool collected = _d_tgc_region_collect(regionId); assert(collected); + foreach (i; 0 .. 64) + { + auto overwrite = cast(int*) _d_tgc_region_malloc(regionId, int.sizeof, 0); + assert(overwrite !is null); + *overwrite = -1; + } atomicStore(collectDone, true); t.join(); diff --git a/druntime/test/gc/tgc_remote_free.d b/druntime/test/gc/tgc_remote_free.d index 0e4f6b4a55cd..a31a06829c81 100644 --- a/druntime/test/gc/tgc_remote_free.d +++ b/druntime/test/gc/tgc_remote_free.d @@ -44,6 +44,7 @@ void main() auto foreign = cast(void*) address; GC.free(foreign); + GC.free(foreign); // duplicate requests must coalesce safely atomicStore(freeQueued, true); while (!atomicLoad(queueDrained))