<,�tEXtComment File Manager

File Manager

Viewing File: mgcscavenge_test.go

// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"fmt"
	"internal/goos"
	"math"
	"math/rand"
	. "runtime"
	"runtime/internal/atomic"
	"testing"
	"time"
)

// makePallocData produces an initialized PallocData by setting
// the ranges of described in alloc and scavenge.
func makePallocData(alloc, scavenged []BitRange) *PallocData {
	b := new(PallocData)
	for _, v := range alloc {
		if v.N == 0 {
			// Skip N==0. It's harmless and allocRange doesn't
			// handle this case.
			continue
		}
		b.AllocRange(v.I, v.N)
	}
	for _, v := range scavenged {
		if v.N == 0 {
			// See the previous loop.
			continue
		}
		b.ScavengedSetRange(v.I, v.N)
	}
	return b
}

func TestFillAligned(t *testing.T) {
	fillAlignedSlow := func(x uint64, m uint) uint64 {
		if m == 1 {
			return x
		}
		out := uint64(0)
		for i := uint(0); i < 64; i += m {
			for j := uint(0); j < m; j++ {
				if x&(uint64(1)<<(i+j)) != 0 {
					out |= ((uint64(1) << m) - 1) << i
					break
				}
			}
		}
		return out
	}
	check := func(x uint64, m uint) {
		want := fillAlignedSlow(x, m)
		if got := FillAligned(x, m); got != want {
			t.Logf("got:  %064b", got)
			t.Logf("want: %064b", want)
			t.Errorf("bad fillAligned(%016x, %d)", x, m)
		}
	}
	for m := uint(1); m <= 64; m *= 2 {
		tests := []uint64{
			0x0000000000000000,
			0x00000000ffffffff,
			0xffffffff00000000,
			0x8000000000000001,
			0xf00000000000000f,
			0xf00000010050000f,
			0xffffffffffffffff,
			0x0000000000000001,
			0x0000000000000002,
			0x0000000000000008,
			uint64(1) << (m - 1),
			uint64(1) << m,
			// Try a few fixed arbitrary examples.
			0xb02b9effcf137016,
			0x3975a076a9fbff18,
			0x0f8c88ec3b81506e,
			0x60f14d80ef2fa0e6,
		}
		for _, test := range tests {
			check(test, m)
		}
		for i := 0; i < 1000; i++ {
			// Try a pseudo-random numbers.
			check(rand.Uint64(), m)

			if m > 1 {
				// For m != 1, let's construct a slightly more interesting
				// random test. Generate a bitmap which is either 0 or
				// randomly set bits for each m-aligned group of m bits.
				val := uint64(0)
				for n := uint(0); n < 64; n += m {
					// For each group of m bits, flip a coin:
					// * Leave them as zero.
					// * Set them randomly.
					if rand.Uint64()%2 == 0 {
						val |= (rand.Uint64() & ((1 << m) - 1)) << n
					}
				}
				check(val, m)
			}
		}
	}
}

func TestPallocDataFindScavengeCandidate(t *testing.T) {
	type test struct {
		alloc, scavenged []BitRange
		min, max         uintptr
		want             BitRange
	}
	tests := map[string]test{
		"MixedMin1": {
			alloc:     []BitRange{{0, 40}, {42, PallocChunkPages - 42}},
			scavenged: []BitRange{{0, 41}, {42, PallocChunkPages - 42}},
			min:       1,
			max:       PallocChunkPages,
			want:      BitRange{41, 1},
		},
		"MultiMin1": {
			alloc:     []BitRange{{0, 63}, {65, 20}, {87, PallocChunkPages - 87}},
			scavenged: []BitRange{{86, 1}},
			min:       1,
			max:       PallocChunkPages,
			want:      BitRange{85, 1},
		},
	}
	// Try out different page minimums.
	for m := uintptr(1); m <= 64; m *= 2 {
		suffix := fmt.Sprintf("Min%d", m)
		tests["AllFree"+suffix] = test{
			min:  m,
			max:  PallocChunkPages,
			want: BitRange{0, PallocChunkPages},
		}
		tests["AllScavenged"+suffix] = test{
			scavenged: []BitRange{{0, PallocChunkPages}},
			min:       m,
			max:       PallocChunkPages,
			want:      BitRange{0, 0},
		}
		tests["NoneFree"+suffix] = test{
			alloc:     []BitRange{{0, PallocChunkPages}},
			scavenged: []BitRange{{PallocChunkPages / 2, PallocChunkPages / 2}},
			min:       m,
			max:       PallocChunkPages,
			want:      BitRange{0, 0},
		}
		tests["StartFree"+suffix] = test{
			alloc: []BitRange{{uint(m), PallocChunkPages - uint(m)}},
			min:   m,
			max:   PallocChunkPages,
			want:  BitRange{0, uint(m)},
		}
		tests["EndFree"+suffix] = test{
			alloc: []BitRange{{0, PallocChunkPages - uint(m)}},
			min:   m,
			max:   PallocChunkPages,
			want:  BitRange{PallocChunkPages - uint(m), uint(m)},
		}
		tests["Straddle64"+suffix] = test{
			alloc: []BitRange{{0, 64 - uint(m)}, {64 + uint(m), PallocChunkPages - (64 + uint(m))}},
			min:   m,
			max:   2 * m,
			want:  BitRange{64 - uint(m), 2 * uint(m)},
		}
		tests["BottomEdge64WithFull"+suffix] = test{
			alloc:     []BitRange{{64, 64}, {128 + 3*uint(m), PallocChunkPages - (128 + 3*uint(m))}},
			scavenged: []BitRange{{1, 10}},
			min:       m,
			max:       3 * m,
			want:      BitRange{128, 3 * uint(m)},
		}
		tests["BottomEdge64WithPocket"+suffix] = test{
			alloc:     []BitRange{{64, 62}, {127, 1}, {128 + 3*uint(m), PallocChunkPages - (128 + 3*uint(m))}},
			scavenged: []BitRange{{1, 10}},
			min:       m,
			max:       3 * m,
			want:      BitRange{128, 3 * uint(m)},
		}
		tests["Max0"+suffix] = test{
			scavenged: []BitRange{{0, PallocChunkPages - uint(m)}},
			min:       m,
			max:       0,
			want:      BitRange{PallocChunkPages - uint(m), uint(m)},
		}
		if m <= 8 {
			tests["OneFree"] = test{
				alloc: []BitRange{{0, 40}, {40 + uint(m), PallocChunkPages - (40 + uint(m))}},
				min:   m,
				max:   PallocChunkPages,
				want:  BitRange{40, uint(m)},
			}
			tests["OneScavenged"] = test{
				alloc:     []BitRange{{0, 40}, {40 + uint(m), PallocChunkPages - (40 + uint(m))}},
				scavenged: []BitRange{{40, 1}},
				min:       m,
				max:       PallocChunkPages,
				want:      BitRange{0, 0},
			}
		}
		if m > 1 {
			tests["MaxUnaligned"+suffix] = test{
				scavenged: []BitRange{{0, PallocChunkPages - uint(m*2-1)}},
				min:       m,
				max:       m - 2,
				want:      BitRange{PallocChunkPages - uint(m), uint(m)},
			}
			tests["SkipSmall"+suffix] = test{
				alloc: []BitRange{{0, 64 - uint(m)}, {64, 5}, {70, 11}, {82, PallocChunkPages - 82}},
				min:   m,
				max:   m,
				want:  BitRange{64 - uint(m), uint(m)},
			}
			tests["SkipMisaligned"+suffix] = test{
				alloc: []BitRange{{0, 64 - uint(m)}, {64, 63}, {127 + uint(m), PallocChunkPages - (127 + uint(m))}},
				min:   m,
				max:   m,
				want:  BitRange{64 - uint(m), uint(m)},
			}
			tests["MaxLessThan"+suffix] = test{
				scavenged: []BitRange{{0, PallocChunkPages - uint(m)}},
				min:       m,
				max:       1,
				want:      BitRange{PallocChunkPages - uint(m), uint(m)},
			}
		}
	}
	if PhysHugePageSize > uintptr(PageSize) {
		// Check hugepage preserving behavior.
		bits := uint(PhysHugePageSize / uintptr(PageSize))
		if bits < PallocChunkPages {
			tests["PreserveHugePageBottom"] = test{
				alloc: []BitRange{{bits + 2, PallocChunkPages - (bits + 2)}},
				min:   1,
				max:   3, // Make it so that max would have us try to break the huge page.
				want:  BitRange{0, bits + 2},
			}
			if 3*bits < PallocChunkPages {
				// We need at least 3 huge pages in a chunk for this test to make sense.
				tests["PreserveHugePageMiddle"] = test{
					alloc: []BitRange{{0, bits - 10}, {2*bits + 10, PallocChunkPages - (2*bits + 10)}},
					min:   1,
					max:   12, // Make it so that max would have us try to break the huge page.
					want:  BitRange{bits, bits + 10},
				}
			}
			tests["PreserveHugePageTop"] = test{
				alloc: []BitRange{{0, PallocChunkPages - bits}},
				min:   1,
				max:   1, // Even one page would break a huge page in this case.
				want:  BitRange{PallocChunkPages - bits, bits},
			}
		} else if bits == PallocChunkPages {
			tests["PreserveHugePageAll"] = test{
				min:  1,
				max:  1, // Even one page would break a huge page in this case.
				want: BitRange{0, PallocChunkPages},
			}
		} else {
			// The huge page size is greater than pallocChunkPages, so it should
			// be effectively disabled. There's no way we can possible scavenge
			// a huge page out of this bitmap chunk.
			tests["PreserveHugePageNone"] = test{
				min:  1,
				max:  1,
				want: BitRange{PallocChunkPages - 1, 1},
			}
		}
	}
	for name, v := range tests {
		v := v
		t.Run(name, func(t *testing.T) {
			b := makePallocData(v.alloc, v.scavenged)
			start, size := b.FindScavengeCandidate(PallocChunkPages-1, v.min, v.max)
			got := BitRange{start, size}
			if !(got.N == 0 && v.want.N == 0) && got != v.want {
				t.Fatalf("candidate mismatch: got %v, want %v", got, v.want)
			}
		})
	}
}

// Tests end-to-end scavenging on a pageAlloc.
func TestPageAllocScavenge(t *testing.T) {
	if GOOS == "openbsd" && testing.Short() {
		t.Skip("skipping because virtual memory is limited; see #36210")
	}
	type test struct {
		request, expect uintptr
	}
	minPages := PhysPageSize / PageSize
	if minPages < 1 {
		minPages = 1
	}
	type setup struct {
		beforeAlloc map[ChunkIdx][]BitRange
		beforeScav  map[ChunkIdx][]BitRange
		expect      []test
		afterScav   map[ChunkIdx][]BitRange
	}
	tests := map[string]setup{
		"AllFreeUnscavExhaust": {
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {},
				BaseChunkIdx + 1: {},
				BaseChunkIdx + 2: {},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {},
				BaseChunkIdx + 1: {},
				BaseChunkIdx + 2: {},
			},
			expect: []test{
				{^uintptr(0), 3 * PallocChunkPages * PageSize},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {{0, PallocChunkPages}},
				BaseChunkIdx + 1: {{0, PallocChunkPages}},
				BaseChunkIdx + 2: {{0, PallocChunkPages}},
			},
		},
		"NoneFreeUnscavExhaust": {
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {{0, PallocChunkPages}},
				BaseChunkIdx + 1: {},
				BaseChunkIdx + 2: {{0, PallocChunkPages}},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {},
				BaseChunkIdx + 1: {{0, PallocChunkPages}},
				BaseChunkIdx + 2: {},
			},
			expect: []test{
				{^uintptr(0), 0},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {},
				BaseChunkIdx + 1: {{0, PallocChunkPages}},
				BaseChunkIdx + 2: {},
			},
		},
		"ScavHighestPageFirst": {
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx: {},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx: {{uint(minPages), PallocChunkPages - uint(2*minPages)}},
			},
			expect: []test{
				{1, minPages * PageSize},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx: {{uint(minPages), PallocChunkPages - uint(minPages)}},
			},
		},
		"ScavMultiple": {
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx: {},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx: {{uint(minPages), PallocChunkPages - uint(2*minPages)}},
			},
			expect: []test{
				{minPages * PageSize, minPages * PageSize},
				{minPages * PageSize, minPages * PageSize},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx: {{0, PallocChunkPages}},
			},
		},
		"ScavMultiple2": {
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {},
				BaseChunkIdx + 1: {},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {{uint(minPages), PallocChunkPages - uint(2*minPages)}},
				BaseChunkIdx + 1: {{0, PallocChunkPages - uint(2*minPages)}},
			},
			expect: []test{
				{2 * minPages * PageSize, 2 * minPages * PageSize},
				{minPages * PageSize, minPages * PageSize},
				{minPages * PageSize, minPages * PageSize},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:     {{0, PallocChunkPages}},
				BaseChunkIdx + 1: {{0, PallocChunkPages}},
			},
		},
		"ScavDiscontiguous": {
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx:       {},
				BaseChunkIdx + 0xe: {},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:       {{uint(minPages), PallocChunkPages - uint(2*minPages)}},
				BaseChunkIdx + 0xe: {{uint(2 * minPages), PallocChunkPages - uint(2*minPages)}},
			},
			expect: []test{
				{2 * minPages * PageSize, 2 * minPages * PageSize},
				{^uintptr(0), 2 * minPages * PageSize},
				{^uintptr(0), 0},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:       {{0, PallocChunkPages}},
				BaseChunkIdx + 0xe: {{0, PallocChunkPages}},
			},
		},
	}
	// Disable these tests on iOS since we have a small address space.
	// See #46860.
	if PageAlloc64Bit != 0 && goos.IsIos == 0 {
		tests["ScavAllVeryDiscontiguous"] = setup{
			beforeAlloc: map[ChunkIdx][]BitRange{
				BaseChunkIdx:          {},
				BaseChunkIdx + 0x1000: {},
			},
			beforeScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:          {},
				BaseChunkIdx + 0x1000: {},
			},
			expect: []test{
				{^uintptr(0), 2 * PallocChunkPages * PageSize},
				{^uintptr(0), 0},
			},
			afterScav: map[ChunkIdx][]BitRange{
				BaseChunkIdx:          {{0, PallocChunkPages}},
				BaseChunkIdx + 0x1000: {{0, PallocChunkPages}},
			},
		}
	}
	for name, v := range tests {
		v := v
		t.Run(name, func(t *testing.T) {
			b := NewPageAlloc(v.beforeAlloc, v.beforeScav)
			defer FreePageAlloc(b)

			for iter, h := range v.expect {
				if got := b.Scavenge(h.request); got != h.expect {
					t.Fatalf("bad scavenge #%d: want %d, got %d", iter+1, h.expect, got)
				}
			}
			want := NewPageAlloc(v.beforeAlloc, v.afterScav)
			defer FreePageAlloc(want)

			checkPageAlloc(t, want, b)
		})
	}
}

func TestScavenger(t *testing.T) {
	// workedTime is a standard conversion of bytes of scavenge
	// work to time elapsed.
	workedTime := func(bytes uintptr) int64 {
		return int64((bytes+4095)/4096) * int64(10*time.Microsecond)
	}

	// Set up a bunch of state that we're going to track and verify
	// throughout the test.
	totalWork := uint64(64<<20 - 3*PhysPageSize)
	var totalSlept, totalWorked atomic.Int64
	var availableWork atomic.Uint64
	var stopAt atomic.Uint64 // How much available work to stop at.

	// Set up the scavenger.
	var s Scavenger
	s.Sleep = func(ns int64) int64 {
		totalSlept.Add(ns)
		return ns
	}
	s.Scavenge = func(bytes uintptr) (uintptr, int64) {
		avail := availableWork.Load()
		if uint64(bytes) > avail {
			bytes = uintptr(avail)
		}
		t := workedTime(bytes)
		if bytes != 0 {
			availableWork.Add(-int64(bytes))
			totalWorked.Add(t)
		}
		return bytes, t
	}
	s.ShouldStop = func() bool {
		if availableWork.Load() <= stopAt.Load() {
			return true
		}
		return false
	}
	s.GoMaxProcs = func() int32 {
		return 1
	}

	// Define a helper for verifying that various properties hold.
	verifyScavengerState := func(t *testing.T, expWork uint64) {
		t.Helper()

		// Check to make sure it did the amount of work we expected.
		if workDone := uint64(s.Released()); workDone != expWork {
			t.Errorf("want %d bytes of work done, got %d", expWork, workDone)
		}
		// Check to make sure the scavenger is meeting its CPU target.
		idealFraction := float64(ScavengePercent) / 100.0
		cpuFraction := float64(totalWorked.Load()) / float64(totalWorked.Load()+totalSlept.Load())
		if cpuFraction < idealFraction-0.005 || cpuFraction > idealFraction+0.005 {
			t.Errorf("want %f CPU fraction, got %f", idealFraction, cpuFraction)
		}
	}

	// Start the scavenger.
	s.Start()

	// Set up some work and let the scavenger run to completion.
	availableWork.Store(totalWork)
	s.Wake()
	if !s.BlockUntilParked(2e9 /* 2 seconds */) {
		t.Fatal("timed out waiting for scavenger to run to completion")
	}
	// Run a check.
	verifyScavengerState(t, totalWork)

	// Now let's do it again and see what happens when we have no work to do.
	// It should've gone right back to sleep.
	s.Wake()
	if !s.BlockUntilParked(2e9 /* 2 seconds */) {
		t.Fatal("timed out waiting for scavenger to run to completion")
	}
	// Run another check.
	verifyScavengerState(t, totalWork)

	// One more time, this time doing the same amount of work as the first time.
	// Let's see if we can get the scavenger to continue.
	availableWork.Store(totalWork)
	s.Wake()
	if !s.BlockUntilParked(2e9 /* 2 seconds */) {
		t.Fatal("timed out waiting for scavenger to run to completion")
	}
	// Run another check.
	verifyScavengerState(t, 2*totalWork)

	// This time, let's stop after a certain amount of work.
	//
	// Pick a stopping point such that when subtracted from totalWork
	// we get a multiple of a relatively large power of 2. verifyScavengerState
	// always makes an exact check, but the scavenger might go a little over,
	// which is OK. If this breaks often or gets annoying to maintain, modify
	// verifyScavengerState.
	availableWork.Store(totalWork)
	stoppingPoint := uint64(1<<20 - 3*PhysPageSize)
	stopAt.Store(stoppingPoint)
	s.Wake()
	if !s.BlockUntilParked(2e9 /* 2 seconds */) {
		t.Fatal("timed out waiting for scavenger to run to completion")
	}
	// Run another check.
	verifyScavengerState(t, 2*totalWork+(totalWork-stoppingPoint))

	// Clean up.
	s.Stop()
}

func TestScavengeIndex(t *testing.T) {
	// This test suite tests the scavengeIndex data structure.

	// markFunc is a function that makes the address range [base, limit)
	// available for scavenging in a test index.
	type markFunc func(base, limit uintptr)

	// findFunc is a function that searches for the next available page
	// to scavenge in the index. It asserts that the page is found in
	// chunk "ci" at page "offset."
	type findFunc func(ci ChunkIdx, offset uint)

	// The structure of the tests below is as follows:
	//
	// setup creates a fake scavengeIndex that can be mutated and queried by
	// the functions it returns. Those functions capture the testing.T that
	// setup is called with, so they're bound to the subtest they're created in.
	//
	// Tests are then organized into test cases which mark some pages as
	// scavenge-able then try to find them. Tests expect that the initial
	// state of the scavengeIndex has all of the chunks as dense in the last
	// generation and empty to the scavenger.
	//
	// There are a few additional tests that interleave mark and find operations,
	// so they're defined separately, but use the same infrastructure.
	setup := func(t *testing.T, force bool) (mark markFunc, find findFunc, nextGen func()) {
		t.Helper()

		// Pick some reasonable bounds. We don't need a huge range just to test.
		si := NewScavengeIndex(BaseChunkIdx, BaseChunkIdx+64)

		// Initialize all the chunks as dense and empty.
		//
		// Also, reset search addresses so that we can get page offsets.
		si.AllocRange(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+64, 0))
		si.NextGen()
		si.FreeRange(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+64, 0))
		for ci := BaseChunkIdx; ci < BaseChunkIdx+64; ci++ {
			si.SetEmpty(ci)
		}
		si.ResetSearchAddrs()

		// Create and return test functions.
		mark = func(base, limit uintptr) {
			t.Helper()

			si.AllocRange(base, limit)
			si.FreeRange(base, limit)
		}
		find = func(want ChunkIdx, wantOffset uint) {
			t.Helper()

			got, gotOffset := si.Find(force)
			if want != got {
				t.Errorf("find: wanted chunk index %d, got %d", want, got)
			}
			if wantOffset != gotOffset {
				t.Errorf("find: wanted page offset %d, got %d", wantOffset, gotOffset)
			}
			if t.Failed() {
				t.FailNow()
			}
			si.SetEmpty(got)
		}
		nextGen = func() {
			t.Helper()

			si.NextGen()
		}
		return
	}

	// Each of these test cases calls mark and then find once.
	type testCase struct {
		name string
		mark func(markFunc)
		find func(findFunc)
	}
	for _, test := range []testCase{
		{
			name: "Uninitialized",
			mark: func(_ markFunc) {},
			find: func(_ findFunc) {},
		},
		{
			name: "OnePage",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 3), PageBase(BaseChunkIdx, 4))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx, 3)
			},
		},
		{
			name: "FirstPage",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx, 1))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx, 0)
			},
		},
		{
			name: "SeveralPages",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 9), PageBase(BaseChunkIdx, 14))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx, 13)
			},
		},
		{
			name: "WholeChunk",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+1, 0))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx, PallocChunkPages-1)
			},
		},
		{
			name: "LastPage",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, PallocChunkPages-1), PageBase(BaseChunkIdx+1, 0))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx, PallocChunkPages-1)
			},
		},
		{
			name: "TwoChunks",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 128), PageBase(BaseChunkIdx+1, 128))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx+1, 127)
				find(BaseChunkIdx, PallocChunkPages-1)
			},
		},
		{
			name: "TwoChunksOffset",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx+7, 128), PageBase(BaseChunkIdx+8, 129))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx+8, 128)
				find(BaseChunkIdx+7, PallocChunkPages-1)
			},
		},
		{
			name: "SevenChunksOffset",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx+6, 11), PageBase(BaseChunkIdx+13, 15))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx+13, 14)
				for i := BaseChunkIdx + 12; i >= BaseChunkIdx+6; i-- {
					find(i, PallocChunkPages-1)
				}
			},
		},
		{
			name: "ThirtyTwoChunks",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+32, 0))
			},
			find: func(find findFunc) {
				for i := BaseChunkIdx + 31; i >= BaseChunkIdx; i-- {
					find(i, PallocChunkPages-1)
				}
			},
		},
		{
			name: "ThirtyTwoChunksOffset",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx+3, 0), PageBase(BaseChunkIdx+35, 0))
			},
			find: func(find findFunc) {
				for i := BaseChunkIdx + 34; i >= BaseChunkIdx+3; i-- {
					find(i, PallocChunkPages-1)
				}
			},
		},
		{
			name: "Mark",
			mark: func(mark markFunc) {
				for i := BaseChunkIdx; i < BaseChunkIdx+32; i++ {
					mark(PageBase(i, 0), PageBase(i+1, 0))
				}
			},
			find: func(find findFunc) {
				for i := BaseChunkIdx + 31; i >= BaseChunkIdx; i-- {
					find(i, PallocChunkPages-1)
				}
			},
		},
		{
			name: "MarkIdempotentOneChunk",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+1, 0))
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+1, 0))
			},
			find: func(find findFunc) {
				find(BaseChunkIdx, PallocChunkPages-1)
			},
		},
		{
			name: "MarkIdempotentThirtyTwoChunks",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+32, 0))
				mark(PageBase(BaseChunkIdx, 0), PageBase(BaseChunkIdx+32, 0))
			},
			find: func(find findFunc) {
				for i := BaseChunkIdx + 31; i >= BaseChunkIdx; i-- {
					find(i, PallocChunkPages-1)
				}
			},
		},
		{
			name: "MarkIdempotentThirtyTwoChunksOffset",
			mark: func(mark markFunc) {
				mark(PageBase(BaseChunkIdx+4, 0), PageBase(BaseChunkIdx+31, 0))
				mark(PageBase(BaseChunkIdx+5, 0), PageBase(BaseChunkIdx+36, 0))
			},
			find: func(find findFunc) {
				for i := BaseChunkIdx + 35; i >= BaseChunkIdx+4; i-- {
					find(i, PallocChunkPages-1)
				}
			},
		},
	} {
		test := test
		t.Run("Bg/"+test.name, func(t *testing.T) {
			mark, find, nextGen := setup(t, false)
			test.mark(mark)
			find(0, 0)      // Make sure we find nothing at this point.
			nextGen()       // Move to the next generation.
			test.find(find) // Now we should be able to find things.
			find(0, 0)      // The test should always fully exhaust the index.
		})
		t.Run("Force/"+test.name, func(t *testing.T) {
			mark, find, _ := setup(t, true)
			test.mark(mark)
			test.find(find) // Finding should always work when forced.
			find(0, 0)      // The test should always fully exhaust the index.
		})
	}
	t.Run("Bg/MarkInterleaved", func(t *testing.T) {
		mark, find, nextGen := setup(t, false)
		for i := BaseChunkIdx; i < BaseChunkIdx+32; i++ {
			mark(PageBase(i, 0), PageBase(i+1, 0))
			nextGen()
			find(i, PallocChunkPages-1)
		}
		find(0, 0)
	})
	t.Run("Force/MarkInterleaved", func(t *testing.T) {
		mark, find, _ := setup(t, true)
		for i := BaseChunkIdx; i < BaseChunkIdx+32; i++ {
			mark(PageBase(i, 0), PageBase(i+1, 0))
			find(i, PallocChunkPages-1)
		}
		find(0, 0)
	})
}

func TestScavChunkDataPack(t *testing.T) {
	if !CheckPackScavChunkData(1918237402, 512, 512, 0b11) {
		t.Error("failed pack/unpack check for scavChunkData 1")
	}
	if !CheckPackScavChunkData(^uint32(0), 12, 0, 0b00) {
		t.Error("failed pack/unpack check for scavChunkData 2")
	}
}

func FuzzPIController(f *testing.F) {
	isNormal := func(x float64) bool {
		return !math.IsInf(x, 0) && !math.IsNaN(x)
	}
	isPositive := func(x float64) bool {
		return isNormal(x) && x > 0
	}
	// Seed with constants from controllers in the runtime.
	// It's not critical that we keep these in sync, they're just
	// reasonable seed inputs.
	f.Add(0.3375, 3.2e6, 1e9, 0.001, 1000.0, 0.01)
	f.Add(0.9, 4.0, 1000.0, -1000.0, 1000.0, 0.84)
	f.Fuzz(func(t *testing.T, kp, ti, tt, min, max, setPoint float64) {
		// Ignore uninteresting invalid parameters. These parameters
		// are constant, so in practice surprising values will be documented
		// or will be other otherwise immediately visible.
		//
		// We just want to make sure that given a non-Inf, non-NaN input,
		// we always get a non-Inf, non-NaN output.
		if !isPositive(kp) || !isPositive(ti) || !isPositive(tt) {
			return
		}
		if !isNormal(min) || !isNormal(max) || min > max {
			return
		}
		// Use a random source, but make it deterministic.
		rs := rand.New(rand.NewSource(800))
		randFloat64 := func() float64 {
			return math.Float64frombits(rs.Uint64())
		}
		p := NewPIController(kp, ti, tt, min, max)
		state := float64(0)
		for i := 0; i < 100; i++ {
			input := randFloat64()
			// Ignore the "ok" parameter. We're just trying to break it.
			// state is intentionally completely uncorrelated with the input.
			var ok bool
			state, ok = p.Next(input, setPoint, 1.0)
			if !isNormal(state) {
				t.Fatalf("got NaN or Inf result from controller: %f %v", state, ok)
			}
		}
	})
}

��b IDATx��ytVս��ϓ22 �A@�IR��:�h�CiZ[�v��*E��:�W��ũZA� ^d��QeQ ��@ !��j�Z�'�>g�s��V��仿$�|?g�)&x-E�IE�N�T ��;@x�T.i��%-��X��}S�v�S�5�.��r/UHz^�_$-��W"�w��)�Ɗ�/@Z �&IoX��P�$K��}��JzX��:�;`�� &�, ��ŋu�i��,�e��6��m��X� Ե��rKb1�ԗ��)D�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�AD�A�݀!��I*��]R;I�2$eZ#OR��Z��Sr��r�6m��teff�u��*((P��u�'��v{��DIߔ4^�pI��m�'77WEEE;v�Ǝ��4�-��$]'�RI��{��\�I�&�G�� :I��HJ�� DWBB��=�\��W�R�޽m� o$K�(�V9��ABB�.��}��jѢ��v�`^?IO�ȅ}ڶm�G}T#F��J`�5�6��$-��ھ}�F��I�&��v;0(h;��Б�3�8CӧOWf��!�;A i:��F_m��9s&�|�q%=�#��w��Z�p�rrrl�a A� &��P\\��СC�[A#!� �{��o��l��F�} `��E2��}��M��K/��vV��)i��{4Bff�V�\��|ۭX�`��b�@��k��ɶ�@��%i��$K�z��5z�h�mX�[�I�XZ`� '��b%$��r�5��M��4�º��/�l� ԃ��ߖ�xhʔ)�[@=�} K�6IM}^��5k��㏷݆��z ��Η�ÿO��:gdG��B�m��y�T/��@+��Vɶ�纽z񕏵l�.��y�޴i�t뭷zV��0[�Y^>�Ws�qs}�\/�@$�(�T7�f��.��I�nݺi��R$푔n��.�~?H))\Z�RW'M�o�~v� Ov6o��ԃ��x��z��!�S,&�xm/�y�ɞԟ?�'ua��S�ѽ��b�,8�Gל�K��b��o�i�&��3t7Y,��)JJ��c[�n��z��ӳ�d�E��&K�sZ�Lӄ��I?@��&�%ӟ�۶mSMM�њ0��i�ؐSZ�,�|J+��N�� ~�,��0A0!5%Q-��YQQa��3}$_vV��r�f9f?S8`��zD�AD�AD�AD�AD�AD�AD�AD�AD�A�d��q�P��,ت��m��M��mg��1V?�r�S��I꒟��]�u|�l ��R��CyE�f�٢9�j��URb�zt�Ѱ!m5~tG��j�2�D��h�G�*�{H9�)꒟��ר3:(+3\?�/;TU��ݭ��ʴ~S�6lڧ��U��J*�i�$�d�(#=Yݺd�{�,�p|3��B))��q�:��vN0Y.�jk��ק6��;�SɶVzHJJЀ�-u��tѹ�սk��>�QUU��\��޲��~]fF�n��K?��&��ߡ��5��b��=z9�)��^�|u_�k�-[��y%Z��NU6 ��7M�i��:�]ۦt�k��[�n� ��X��(�e6B�b�.��"�8�cۭ|��~�t�e��u��u��w�|��ή��I-5�"��~��U��k��;��Zi�cE�m��N/��:�]�M��> cQ�^�ui�ƞ��??Ң��p�c#TUU�3�U��a��k��Nw�A`��:��Y_V��-8.KKf�R��itv�޲* 9S�6ֿ��j�,Ճ��NOMߤ��]��z�^��f��O�h��|��<�>@Å5��_��/I��u?{S��Y��4h�K��/2��]��4�%i��t�5��q�]�G�G��e�2�%i�R��| ��W��&f�*^]�??��vq[��Lg��E��_��3f}��F�xu~��}q��d�-��ږF�xu~I N>\��;��͗��O��֊�:�̗��W��J@Bh�W=��y��|��Ggwܷ�H�_NY��?��)T��d��i�'?��խw�h�lm�Qi� �!SUU��sw4kӺ�e4rf��x�u�-[n�Ht��MFj}�H�_��u�~w�>)�o�V}�(�T'��e��bʒ�v��3_�[+v�n@Ȭ\S�}��o��t��}w��=��k�H��F�n�xg��S�� 0eޢm~�l��}��u��q��Z�f��F��oZuu��E��g� ��`z�t~?b�;t�%�>��WTkķ�h��[�2e�G8L��IW�x��,��^\thr�l��^��Ϊ��{��=�ǆ��<}�q��V�@ ��⠨��W��y^��L��F_��>�0��U�k��D�u�ʫu��Cs$)I��v��:�IK��;6ֲ�4{^��6��ե��m+l��3>�݆��uM �9��u��?>�Z��c��}g�~q��h��Kw��ڭ��eFMM�~p�М�uq�ǿz��6T��b@8��@Y�|��jx��]�(^]�gf�}�M�"tG ��-w��.@�vOqh~/�HII��`��S�[l��.��6�nØXL��9�v�U�cOo��B�\�x�o��Ǥ�'�T�&I��Ǎ�Qw��_w�p�v��[�k�m��O�{�w�~�>�#=P1P�ɞ�a�-w��e�:i�Ǐl��H��o�׈��꒟�f9��SzH�?��+s�h�k%F��s:��q��Vh�qY��`��j�vO�'ρ?PyX3�lх��]�˾u��V�{��ݞ]1��,�M�z�YN�W~̈́�joY�n��}��ȚF߾׮mS]��F�� z�+�E��D�xm/��d{�F��{-�W��-�4w�Y��듏:�?��?��_�g�P�f� ^3��e��cg ��ҵs��8R��2�מ�z�@T��A��N��Gj�)��}CNi��/�R~��}��c:5�{��!��Z�HӋӾ��6}T�]��G�]�7W�6^��n �9*,��Y��qOZj��:P?�Q�� DF��L�|��?��-�^��.��Ɵ�7��}f�F�h׶��xe��2P�s��c��z�1��&5\cn��[�=�V��n[��ĶE鎀uˌ��d3G�II ��k�;�l�NmشOuuRVf��BE��]ۣ�eӶu :��X�-[��(��e��r�4��~��LH�i�6:��Ѻ@ԅ��r��ST�0��trk%$Č��0ez��"� *��z�"��T�/X9|8��.��C5F�eg}��C�Q%�͞�ˣ��JvL��/��?�j�^��h��&�9x�F��`�њ�Z��(��&�y��F��&Iݻf�g��#��W��;�3^�{Wo^4�'v�V[[�K'��;+��m��Ӎִ�]AC@��W?1^{�එyh��+^]��f��m��~�i��Ե�]AB@��WTk�̏t��uR�?�l�.O�IH�i�Yy�Զ�]A�ˀ7c��:��q}ힽ��a�f�6��Z~�қm(��+sK4{^�6}T��*UUu�]��n��.��:kx{�:��2�� _m��=�sA�ߤ�U��@?��Z��-V�ކ�е��z왍��Nэ��{|5� pڶn��b� �p-@��sPg]0G7�fy��-��M�{GCF��'%�{�4`��=�$-�Ge\��eU:m�+Z�t�'�W��jO�!O�AF@��i�k&t�݆��ϥ_�� e��}��=]"��Wz��_��.��͜�E3�l�e�W��F�i��h|t��-w��Z��ۍ��-�u��w=�6�YN��{��6|��}��|��*={��Ѽ��n.�S�.��z��1z��j�ۻT��H]��흾� �D�u��D��v��mv��K��.`V]yY�~s�I��@��t?/��ϓ.� ��m�&�["�+��P��?M��z�ovV��ЫG3�-�G��RR��[(!!\�_��,��^��%?�v@��ҵ�ő�� m��`�Y)�te�m8��G��Mx.))A�]Y��i`�V��i��W�`�?�^��~!�S#��^+�ѽ��GZj��?V�ģ�0.))A�꨷��l�z�L�*��]��O��X�r��Y�`DBBL�Oj��{��-M�H'�ii�-ϰ��ok�7^�� )쭡�b��]�UX��S�ְ�mռY��|5��*��c�ֽk��0B��7镹%ڽ��P#8n�Ȏ��q}mJr�23��_>��l��E�5��$i��wu��i�+ ��H�~�F`��IjƵ@�q �\ �@#qG�0"��.�0"� l��`��.�0!� ,�AQ�HN6�q��z�k��KJ�#��o;`X�v2��>,tێJ��J��7Z/*��A��.@f�ف�jM��zk��g��@TvZ�H3Z�xu�6Ra��'%��O��?/d�Q��5�x��Yk��U]��R�ֽk�ق@��Da�S^�RS�ּ�5��|��B��e��HNN�͘p �Hvc�Y�c��C5:��y �#��`�οb��;z��2��.��!��k��r��}g��U��Wk��yZ��n�=�f�� Pv��sn��3�p��~�;4p�˚=�ē~�Nm�I] ��¾0lH[��_��L��h�sh��_��ғߤ��c_њ�e��c�)��g�7��V�IZ��5��yr�gk̞W��#��IjӪ�v�>��՞��y睝��M�8�[�|�]��\�շ��8�M��6�%�|@P��Z��ڨ�I-m��>=�k��=�'a��iRo�-x�?>Q��.��}`�Ȏ:�Ws�mu� �u�� > �.@,&��;+!!�˱�tﭧD��Q�w��RW\��vF\~Q7�>�s�p�Y�w$��%A~�;~}��6��¾��g�&if_��=��j�,�v+�U��L�1(tW��a��ke��:@Ș>�j�$�Gq�2�t7S?�vL��|��]u�/�� .�(�0�E��6M��k�6�h��iۺ��z�ښ��O��r��i�f�ޱ�xm/��G�x>�� La�l%%��~�{��l�Bs��R4�*��}{�0Z/��t��N�I��ɚ��p�V^#�L�f�:u@k#�RSu�� =��S^��Zy��uR/��.@n�&��΃z~��B=��0eg뺆��#,��Þ�[�B/?�H� uUf�7�y ��W��y}Bw��eg��ל`��Wh�(|��|��`l`.�;�Ws��?��V�@"��c:i�ɍ��L֯��PG�v�6z�c�tM��̠��'��:w��uW��;d��=�;E�v�e��D��}��9�J@B(��0�iհ�b�v�P�1{�\P��&��G�7��D��޴I��y��_��$-Q�jm�~Yrr�&]�C��Dv%b�h|�Yz�n�i_�R;�k��g�}n��JOII��w��yuL}{�Ќǋ�}�:+3Y�?:��W��J/N+Rz��d=�h��b��;d��j͒su��ݔ��@NKMԄ�j��qz��C��5@��y°�h��L��m;*5�ezᕏ��=�ep�� X��L�n?�מ:��r�`��۵�tŤ�Z�|�1�v`�V�뽧��_c��s�ج'�ߤ%o�Tuum�k%%%h�)�u�y]��N�k��[�n� ��'b�2� �l�.=��͜�E%��gf$��[c;�s:�V-�͞W��ߤW�h��-��j�7��]4��=��F-X�]�>��Z�LS�i��[�Y��*�W��e;��Z�a��n(ӇW|e(HNNP��5[= r4tP �&0�<��p�c#��`�v�TNV GFq��v�T�i�*��Ty�a�m�$��ߏWyE�*�V��JKMTfF�w��>'��$-�ؽ�.Ho��.��8��c��"@D�AD�AD�AD�AD�AD�AD�AD�AD�A~�j��*֘,N;Pi�3599�h=��g�o�ضL�g��i��J�5��փy~�}&��Z��d9p֚ e:|��h��L��`��`��b/��d9p�?�fgg+%%�hM��gXo��s�ج��, Ω��Ol0Z��h=x��d��j��L��m��h��ݻ�o��O�[�g_�l�,8a��]�٭+��ӧ��0��$��I�]��c��]:粹:Te��ꢢ"�5a^��K�g�h,&��=��=��՟^��߶�ߢE�ܹS �J}��I%�:8 �IDAT~,�9/ʃPW'M�o�}z��N�ƍ쨓z�Pb��NZ�~�^z�=4m��s��w��g;5�� Y�~��S�VM��RXUյڱ�R��f��?��s�:w �;6�H:�º��i��5��-�maM��&O��3;1I�K�eam�Z�h�͛7+##�v+�c ~u�~ca]�Gn��F'��ټL~��PPP��b��n� v�o��C�4R,ӟ��gg��%�hq}@#M�4IÇ�� O�y^�x�M��Z�x�� )� �yOw@H�k�N˖-Sǎm�b]X@n��+i��͖��!++K3g�d��\�$m�t�$^��Y�f��J��\8PR��F�)77W��א!Cl��$i��:�@@��_o��G�� I{$��#� ��8磌��ŋ9�1A��(�I�m7��֭��>}�ߴ�J�q�7ޗt��^� -[��ԩS�j�*��}��%]&�'� -��ɓ'�ꫯVzz��vB#�;a �7@G��xI��{��j޼�ƌ��.�LÇWBB�7��`O��"I�$/�@R�@eee��@�۷��>}�0��,ɒ2$53Xs��|c��S~��rpTYYY��} kH�c�%��&k��.]��, @��AD�AD�AD�AD�AD�AD�AD�AD�A��@�l��T��<%''�*��L�o�^={��رc5h� %$+CnܸQ3f��ҥK�}�vUVV��s�9G� R,�_{�x��ˇ��3��o߾��;TTTd�}��馛��]uuuG��~��i�ԩ��@4��b��n��v�m�vfϞ�/�Peeeq}}za I~,�誫��{UWW뮻��}��_~Y�ƍSMMM��Yχ֝w��aw\�ď�cxꩧt�E��ƍկ_?�۷��5��@�u�?�1�k�N�ׯWzz�/�w�y�>}z�j��3�� k�(�ٺu��q_�Z�v�f̘��:~�AB�Q&�r��|��!��%K��ҥK��g��Ԟ={<_��X-�z� !��C�y�FUU��z~�AB�QIIIjݺ��W�$UXX��D��ٳ��Z~�AB�Q�ƍ�e��c��W�$<(~<�RSS�v�Zu��jjjԧO�Z��Qu��@4� 8��m�&&&j�ԩ��g�$�ď��1h� ͟?_��{�768��@��g �=@�`)))��5o��6m��3��)��ѣ�ƌ��J�;w��ҿUTT��/��K��ZR�{~a=@��0o�<��*狔��i�F��ɶ[�ˎ;T]]��OX@��?��K�.�ۈ�xN ��pppppppppppppppppP��fl߾],�{ｧ��k۶mڿ��o�5B��TӦMӴiӴ|�r�� DB��2e�|�A��n!D��y�'t��k�Ν�[A� $***t�5��'�� "�!�駟��o�a�Dn�Ν:t�֭[g�D��ШQ��0�6q�D��;��@ ��x� M�6�v�(Pii��z�m��Z�4e��w��"@��̴i�x��f��[��~-F�ٱc��&I�Z�2�|��n��!�?$@��{�[��H�T��ɏ�#��@�h��Ȏ��I#�_�m�(F��/6Z3��z��'�\r�,��r��!��;��w2Z3j��=~�G��Y��7��"I�$��i�I.�p��_"��?�p��N`�y��DD��?: ��_�� G�ÿa��b�7J��!B��x@�0�� B��o ��c��G��@`1��C��[@�0��G ��@`0��C��_�u�V1�� a��CX��>�W�` |��`!<��S�`"<��.��`#�c�`�?c��A��C4 �?��c�� p#��~@�0��?:��0��8&��_��M�Q1��J�h#��?��/`��7��;I��q7�aw�Q��A�1Hp ��!�#��<8��/#@�1��U�l7��=�S�=��K.�4Z�?E��_$i��@��!�1�!E4�?��`��P_� ��@��Bă�1��0�#��:��"��a��U�,�x�b��F��Y1� ��[��n|��n� ��#'��v�E��H:`�x��b� ��#��v��D��4��Y ��h��i.i��&��E�Ζ��v#��O� H��4�I�Ŷ��}�:I�k�h��@t��ZR��F��#��(�tXҙ��zZ �?��I��3l7q��@õ��|ۍ�1,G��p�u��Y� ��Ꮿ@h��J��v#�x��xk$ ��v#�9��5��}��_��$��c �S�#��=+��"K�{F��*m7�`#��%�H:NRS�p�6I?��sIՖ{Ap��$I$I:QR��v�2$�Z�@�UJ*��$�]<��F��O4IEND�B`�