runtime (gc_blocks.go): use a linked stack to scan marked objects

builder/sizes_test.go

@@ -42,9 +42,9 @@ func TestBinarySize(t *testing.T) {
 	// This is a small number of very diverse targets that we want to test.
 	tests := []sizeTest{
 		// microcontrollers
-		{"hifive1b", "examples/echo", 3896, 280, 0, 2268},
-		{"microbit", "examples/serial", 2860, 360, 8, 2272},
-		{"wioterminal", "examples/pininterrupt", 7361, 1491, 116, 6912},
+		{"hifive1b", "examples/echo", 3568, 280, 0, 2268},
+		{"microbit", "examples/serial", 2630, 342, 8, 2272},
+		{"wioterminal", "examples/pininterrupt", 7175, 1493, 116, 6912},
 
 		// TODO: also check wasm. Right now this is difficult, because
 		// wasm binaries are run through wasm-opt and therefore the

src/runtime/gc_blocks.go

@@ -46,11 +46,11 @@ const (
 	bytesPerBlock      = wordsPerBlock * unsafe.Sizeof(heapStart)
 	stateBits          = 2 // how many bits a block state takes (see blockState type)
 	blocksPerStateByte = 8 / stateBits
-	markStackSize      = 8 * unsafe.Sizeof((*int)(nil)) // number of to-be-marked blocks to queue before forcing a rescan
 )
 
 var (
 	metadataStart unsafe.Pointer // pointer to the start of the heap metadata
+	scanList      *objHeader     // scanList is a singly linked list of heap objects that have been marked but not scanned
 	nextAlloc     gcBlock        // the next block that should be tried by the allocator
 	endBlock      gcBlock        // the block just past the end of the available space
 	gcTotalAlloc  uint64         // total number of bytes allocated
@@ -225,6 +225,15 @@ func (b gcBlock) unmark() {
 	}
 }
 
+// objHeader is a structure prepended to every heap object to hold metadata.
+type objHeader struct {
+	// next is the next object to scan after this.
+	next *objHeader
+
+	// layout holds the layout bitmap used to find pointers in the object.
+	layout gcLayout
+}
+
 func isOnHeap(ptr uintptr) bool {
 	return ptr >= heapStart && ptr < uintptr(metadataStart)
 }
@@ -315,13 +324,10 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 		runtimePanicAt(returnAddress(0), "heap alloc in interrupt")
 	}
 
-	// Round the size up to a multiple of blocks.
+	// Round the size up to a multiple of blocks, adding space for the header.
 	rawSize := size
+	size += align(unsafe.Sizeof(objHeader{}))
 	size += bytesPerBlock - 1
-	if preciseHeap {
-		// Add space for the layout.
-		size += align(unsafe.Sizeof(layout))
-	}
 	if size < rawSize {
 		// The size overflowed.
 		runtimePanicAt(returnAddress(0), "out of memory")
@@ -414,20 +420,18 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 				i.setState(blockStateTail)
 			}
 
+			// Create the object header.
+			pointer := thisAlloc.pointer()
+			header := (*objHeader)(pointer)
+			header.layout = parseGCLayout(layout)
+
 			// We've claimed this allocation, now we can unlock the heap.
 			gcLock.Unlock()
 
 			// Return a pointer to this allocation.
-			pointer := thisAlloc.pointer()
-			if preciseHeap {
-				// Store the object layout at the start of the object.
-				// TODO: this wastes a little bit of space on systems with
-				// larger-than-pointer alignment requirements.
-				*(*unsafe.Pointer)(pointer) = layout
-				add := align(unsafe.Sizeof(layout))
-				pointer = unsafe.Add(pointer, add)
-				size -= add
-			}
+			add := align(unsafe.Sizeof(objHeader{}))
+			pointer = unsafe.Add(pointer, add)
+			size -= add
 			memzero(pointer, size)
 			return pointer
 		}
@@ -528,8 +532,7 @@ func runGC() (freeBytes uintptr) {
 
 // markRoots reads all pointers from start to end (exclusive) and if they look
 // like a heap pointer and are unmarked, marks them and scans that object as
-// well (recursively). The start and end parameters must be valid pointers and
-// must be aligned.
+// well (recursively). The starting address must be valid and aligned.
 func markRoots(start, end uintptr) {
 	if gcDebug {
 		println("mark from", start, "to", end, int(end-start))
@@ -541,18 +544,21 @@ func markRoots(start, end uintptr) {
 		if start%unsafe.Alignof(start) != 0 {
 			runtimePanic("gc: unaligned start pointer")
 		}
-		if end%unsafe.Alignof(end) != 0 {
-			runtimePanic("gc: unaligned end pointer")
-		}
 	}
 
-	// Reduce the end bound to avoid reading too far on platforms where pointer alignment is smaller than pointer size.
-	// If the size of the range is 0, then end will be slightly below start after this.
-	end -= unsafe.Sizeof(end) - unsafe.Alignof(end)
+	// Scan the range conservatively.
+	scanConservative(start, end-start)
+}
 
-	for addr := start; addr < end; addr += unsafe.Alignof(addr) {
+// scanConservative scans all possible pointer locations in a range and marks referenced heap allocations.
+// The starting address must be valid and pointer-aligned.
+func scanConservative(addr, len uintptr) {
+	for len >= unsafe.Sizeof(addr) {
 		root := *(*uintptr)(unsafe.Pointer(addr))
 		markRoot(addr, root)
+
+		addr += unsafe.Alignof(addr)
+		len -= unsafe.Alignof(addr)
 	}
 }
 
@@ -562,124 +568,66 @@ func markCurrentGoroutineStack(sp uintptr) {
 	markRoot(0, sp)
 }
 
-// stackOverflow is a flag which is set when the GC scans too deep while marking.
-// After it is set, all marked allocations must be re-scanned.
-var stackOverflow bool
-
-// startMark starts the marking process on a root and all of its children.
-func startMark(root gcBlock) {
-	var stack [markStackSize]gcBlock
-	stack[0] = root
-	root.setState(blockStateMark)
-	stackLen := 1
-	for stackLen > 0 {
-		// Pop a block off of the stack.
-		stackLen--
-		block := stack[stackLen]
-		if gcDebug {
-			println("stack popped, remaining stack:", stackLen)
+// finishMark finishes the marking process by scanning all heap objects on scanList.
+func finishMark() {
+	for {
+		// Remove an object from the scan list.
+		obj := scanList
+		if obj == nil {
+			return
 		}
+		scanList = obj.next
 
-		// Scan all pointers inside the block.
-		scanner := newGCObjectScanner(block)
-		if scanner.pointerFree() {
+		// Check if the object may contain pointers.
+		if obj.layout.pointerFree() {
 			// This object doesn't contain any pointers.
 			// This is a fast path for objects like make([]int, 4096).
+			// It skips the length calculation.
 			continue
 		}
-		start, end := block.address(), block.findNext().address()
-		if preciseHeap {
-			// The first word of the object is just the pointer layout value.
-			// Skip it.
-			start += align(unsafe.Sizeof(uintptr(0)))
-		}
-		for addr := start; addr != end; addr += unsafe.Alignof(addr) {
-			// Load the word.
-			word := *(*uintptr)(unsafe.Pointer(addr))
-
-			if !scanner.nextIsPointer(word, root.address(), addr) {
-				// Not a heap pointer.
-				continue
-			}
-
-			// Find the corresponding memory block.
-			referencedBlock := blockFromAddr(word)
-
-			if referencedBlock.state() == blockStateFree {
-				// The to-be-marked object doesn't actually exist.
-				// This is probably a false positive.
-				if gcDebug {
-					println("found reference to free memory:", word, "at:", addr)
-				}
-				continue
-			}
-
-			// Move to the block's head.
-			referencedBlock = referencedBlock.findHead()
-
-			if referencedBlock.state() == blockStateMark {
-				// The block has already been marked by something else.
-				continue
-			}
 
-			// Mark block.
-			if gcDebug {
-				println("marking block:", referencedBlock)
-			}
-			referencedBlock.setState(blockStateMark)
-
-			if stackLen == len(stack) {
-				// The stack is full.
-				// It is necessary to rescan all marked blocks once we are done.
-				stackOverflow = true
-				if gcDebug {
-					println("gc stack overflowed")
-				}
-				continue
-			}
+		// Compute the scan bounds.
+		objAddr := uintptr(unsafe.Pointer(obj))
+		start := objAddr + align(unsafe.Sizeof(objHeader{}))
+		end := blockFromAddr(objAddr).findNext().address()
 
-			// Push the pointer onto the stack to be scanned later.
-			stack[stackLen] = referencedBlock
-			stackLen++
-		}
+		// Scan the object.
+		obj.layout.scan(start, end-start)
 	}
 }
 
-// finishMark finishes the marking process by processing all stack overflows.
-func finishMark() {
-	for stackOverflow {
-		// Re-mark all blocks.
-		stackOverflow = false
-		for block := gcBlock(0); block < endBlock; block++ {
-			if block.state() != blockStateMark {
-				// Block is not marked, so we do not need to rescan it.
-				continue
-			}
+// mark a GC root at the address addr.
+func markRoot(addr, root uintptr) {
+	// Find the heap block corresponding to the root.
+	if !isOnHeap(root) {
+		// This is not a heap pointer.
+		return
+	}
+	block := blockFromAddr(root)
 
-			// Re-mark the block.
-			startMark(block)
-		}
+	// Find the head of the corresponding object.
+	if block.state() == blockStateFree {
+		// The to-be-marked object doesn't actually exist.
+		// This could either be a dangling pointer (oops!) but most likely
+		// just a false positive.
+		return
 	}
-}
+	head := block.findHead()
 
-// mark a GC root at the address addr.
-func markRoot(addr, root uintptr) {
-	if isOnHeap(root) {
-		block := blockFromAddr(root)
-		if block.state() == blockStateFree {
-			// The to-be-marked object doesn't actually exist.
-			// This could either be a dangling pointer (oops!) but most likely
-			// just a false positive.
-			return
-		}
-		head := block.findHead()
-		if head.state() != blockStateMark {
-			if gcDebug {
-				println("found unmarked pointer", root, "at address", addr)
-			}
-			startMark(head)
-		}
+	// Mark the object.
+	if head.state() == blockStateMark {
+		// This object is already marked.
+		return
 	}
+	if gcDebug {
+		println("found unmarked pointer", root, "at address", addr)
+	}
+	head.setState(blockStateMark)
+
+	// Add the object to the scan list.
+	header := (*objHeader)(head.pointer())
+	header.next = scanList
+	scanList = header
 }
 
 // Sweep goes through all memory and frees unmarked memory.

src/runtime/gc_conservative.go

@@ -6,24 +6,25 @@
 
 package runtime
 
-const preciseHeap = false
+import "unsafe"
 
-type gcObjectScanner struct {
+// parseGCLayout stores the layout information passed to alloc into a gcLayout value.
+// The conservative GC discards this information.
+func parseGCLayout(layout unsafe.Pointer) gcLayout {
+	return gcLayout{}
 }
 
-func newGCObjectScanner(block gcBlock) gcObjectScanner {
-	return gcObjectScanner{}
+// gcLayout tracks pointer locations in a heap object.
+// The conservative GC treats all locations as potential pointers, so this doesn't need to store anything.
+type gcLayout struct {
 }
 
-func (scanner *gcObjectScanner) pointerFree() bool {
+func (l gcLayout) pointerFree() bool {
 	// We don't know whether this object contains pointers, so conservatively
 	// return false.
 	return false
 }
 
-// nextIsPointer returns whether this could be a pointer. Because the GC is
-// conservative, we can't do much more than check whether the object lies
-// somewhere in the heap.
-func (scanner gcObjectScanner) nextIsPointer(ptr, parent, addrOfWord uintptr) bool {
-	return isOnHeap(ptr)
+func (l gcLayout) scan(start, len uintptr) {
+	scanConservative(start, len)
 }