PNG  IHDRxsBIT|d pHYs+tEXtSoftwarewww.inkscape.org<,tEXtComment File Manager

File Manager

Path: /opt/golang/1.22.0/src/runtime/

Viewing File: mspanset.go

// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import (
	"internal/cpu"
	"internal/goarch"
	"runtime/internal/atomic"
	"unsafe"
)

// A spanSet is a set of *mspans.
//
// spanSet is safe for concurrent push and pop operations.
type spanSet struct {
	// A spanSet is a two-level data structure consisting of a
	// growable spine that points to fixed-sized blocks. The spine
	// can be accessed without locks, but adding a block or
	// growing it requires taking the spine lock.
	//
	// Because each mspan covers at least 8K of heap and takes at
	// most 8 bytes in the spanSet, the growth of the spine is
	// quite limited.
	//
	// The spine and all blocks are allocated off-heap, which
	// allows this to be used in the memory manager and avoids the
	// need for write barriers on all of these. spanSetBlocks are
	// managed in a pool, though never freed back to the operating
	// system. We never release spine memory because there could be
	// concurrent lock-free access and we're likely to reuse it
	// anyway. (In principle, we could do this during STW.)

	spineLock mutex
	spine     atomicSpanSetSpinePointer // *[N]atomic.Pointer[spanSetBlock]
	spineLen  atomic.Uintptr            // Spine array length
	spineCap  uintptr                   // Spine array cap, accessed under spineLock

	// index is the head and tail of the spanSet in a single field.
	// The head and the tail both represent an index into the logical
	// concatenation of all blocks, with the head always behind or
	// equal to the tail (indicating an empty set). This field is
	// always accessed atomically.
	//
	// The head and the tail are only 32 bits wide, which means we
	// can only support up to 2^32 pushes before a reset. If every
	// span in the heap were stored in this set, and each span were
	// the minimum size (1 runtime page, 8 KiB), then roughly the
	// smallest heap which would be unrepresentable is 32 TiB in size.
	index atomicHeadTailIndex
}

const (
	spanSetBlockEntries = 512 // 4KB on 64-bit
	spanSetInitSpineCap = 256 // Enough for 1GB heap on 64-bit
)

type spanSetBlock struct {
	// Free spanSetBlocks are managed via a lock-free stack.
	lfnode

	// popped is the number of pop operations that have occurred on
	// this block. This number is used to help determine when a block
	// may be safely recycled.
	popped atomic.Uint32

	// spans is the set of spans in this block.
	spans [spanSetBlockEntries]atomicMSpanPointer
}

// push adds span s to buffer b. push is safe to call concurrently
// with other push and pop operations.
func (b *spanSet) push(s *mspan) {
	// Obtain our slot.
	cursor := uintptr(b.index.incTail().tail() - 1)
	top, bottom := cursor/spanSetBlockEntries, cursor%spanSetBlockEntries

	// Do we need to add a block?
	spineLen := b.spineLen.Load()
	var block *spanSetBlock
retry:
	if top < spineLen {
		block = b.spine.Load().lookup(top).Load()
	} else {
		// Add a new block to the spine, potentially growing
		// the spine.
		lock(&b.spineLock)
		// spineLen cannot change until we release the lock,
		// but may have changed while we were waiting.
		spineLen = b.spineLen.Load()
		if top < spineLen {
			unlock(&b.spineLock)
			goto retry
		}

		spine := b.spine.Load()
		if spineLen == b.spineCap {
			// Grow the spine.
			newCap := b.spineCap * 2
			if newCap == 0 {
				newCap = spanSetInitSpineCap
			}
			newSpine := persistentalloc(newCap*goarch.PtrSize, cpu.CacheLineSize, &memstats.gcMiscSys)
			if b.spineCap != 0 {
				// Blocks are allocated off-heap, so
				// no write barriers.
				memmove(newSpine, spine.p, b.spineCap*goarch.PtrSize)
			}
			spine = spanSetSpinePointer{newSpine}

			// Spine is allocated off-heap, so no write barrier.
			b.spine.StoreNoWB(spine)
			b.spineCap = newCap
			// We can't immediately free the old spine
			// since a concurrent push with a lower index
			// could still be reading from it. We let it
			// leak because even a 1TB heap would waste
			// less than 2MB of memory on old spines. If
			// this is a problem, we could free old spines
			// during STW.
		}

		// Allocate a new block from the pool.
		block = spanSetBlockPool.alloc()

		// Add it to the spine.
		// Blocks are allocated off-heap, so no write barrier.
		spine.lookup(top).StoreNoWB(block)
		b.spineLen.Store(spineLen + 1)
		unlock(&b.spineLock)
	}

	// We have a block. Insert the span atomically, since there may be
	// concurrent readers via the block API.
	block.spans[bottom].StoreNoWB(s)
}

// pop removes and returns a span from buffer b, or nil if b is empty.
// pop is safe to call concurrently with other pop and push operations.
func (b *spanSet) pop() *mspan {
	var head, tail uint32
claimLoop:
	for {
		headtail := b.index.load()
		head, tail = headtail.split()
		if head >= tail {
			// The buf is empty, as far as we can tell.
			return nil
		}
		// Check if the head position we want to claim is actually
		// backed by a block.
		spineLen := b.spineLen.Load()
		if spineLen <= uintptr(head)/spanSetBlockEntries {
			// We're racing with a spine growth and the allocation of
			// a new block (and maybe a new spine!), and trying to grab
			// the span at the index which is currently being pushed.
			// Instead of spinning, let's just notify the caller that
			// there's nothing currently here. Spinning on this is
			// almost definitely not worth it.
			return nil
		}
		// Try to claim the current head by CASing in an updated head.
		// This may fail transiently due to a push which modifies the
		// tail, so keep trying while the head isn't changing.
		want := head
		for want == head {
			if b.index.cas(headtail, makeHeadTailIndex(want+1, tail)) {
				break claimLoop
			}
			headtail = b.index.load()
			head, tail = headtail.split()
		}
		// We failed to claim the spot we were after and the head changed,
		// meaning a popper got ahead of us. Try again from the top because
		// the buf may not be empty.
	}
	top, bottom := head/spanSetBlockEntries, head%spanSetBlockEntries

	// We may be reading a stale spine pointer, but because the length
	// grows monotonically and we've already verified it, we'll definitely
	// be reading from a valid block.
	blockp := b.spine.Load().lookup(uintptr(top))

	// Given that the spine length is correct, we know we will never
	// see a nil block here, since the length is always updated after
	// the block is set.
	block := blockp.Load()
	s := block.spans[bottom].Load()
	for s == nil {
		// We raced with the span actually being set, but given that we
		// know a block for this span exists, the race window here is
		// extremely small. Try again.
		s = block.spans[bottom].Load()
	}
	// Clear the pointer. This isn't strictly necessary, but defensively
	// avoids accidentally re-using blocks which could lead to memory
	// corruption. This way, we'll get a nil pointer access instead.
	block.spans[bottom].StoreNoWB(nil)

	// Increase the popped count. If we are the last possible popper
	// in the block (note that bottom need not equal spanSetBlockEntries-1
	// due to races) then it's our responsibility to free the block.
	//
	// If we increment popped to spanSetBlockEntries, we can be sure that
	// we're the last popper for this block, and it's thus safe to free it.
	// Every other popper must have crossed this barrier (and thus finished
	// popping its corresponding mspan) by the time we get here. Because
	// we're the last popper, we also don't have to worry about concurrent
	// pushers (there can't be any). Note that we may not be the popper
	// which claimed the last slot in the block, we're just the last one
	// to finish popping.
	if block.popped.Add(1) == spanSetBlockEntries {
		// Clear the block's pointer.
		blockp.StoreNoWB(nil)

		// Return the block to the block pool.
		spanSetBlockPool.free(block)
	}
	return s
}

// reset resets a spanSet which is empty. It will also clean up
// any left over blocks.
//
// Throws if the buf is not empty.
//
// reset may not be called concurrently with any other operations
// on the span set.
func (b *spanSet) reset() {
	head, tail := b.index.load().split()
	if head < tail {
		print("head = ", head, ", tail = ", tail, "\n")
		throw("attempt to clear non-empty span set")
	}
	top := head / spanSetBlockEntries
	if uintptr(top) < b.spineLen.Load() {
		// If the head catches up to the tail and the set is empty,
		// we may not clean up the block containing the head and tail
		// since it may be pushed into again. In order to avoid leaking
		// memory since we're going to reset the head and tail, clean
		// up such a block now, if it exists.
		blockp := b.spine.Load().lookup(uintptr(top))
		block := blockp.Load()
		if block != nil {
			// Check the popped value.
			if block.popped.Load() == 0 {
				// popped should never be zero because that means we have
				// pushed at least one value but not yet popped if this
				// block pointer is not nil.
				throw("span set block with unpopped elements found in reset")
			}
			if block.popped.Load() == spanSetBlockEntries {
				// popped should also never be equal to spanSetBlockEntries
				// because the last popper should have made the block pointer
				// in this slot nil.
				throw("fully empty unfreed span set block found in reset")
			}

			// Clear the pointer to the block.
			blockp.StoreNoWB(nil)

			// Return the block to the block pool.
			spanSetBlockPool.free(block)
		}
	}
	b.index.reset()
	b.spineLen.Store(0)
}

// atomicSpanSetSpinePointer is an atomically-accessed spanSetSpinePointer.
//
// It has the same semantics as atomic.UnsafePointer.
type atomicSpanSetSpinePointer struct {
	a atomic.UnsafePointer
}

// Loads the spanSetSpinePointer and returns it.
//
// It has the same semantics as atomic.UnsafePointer.
func (s *atomicSpanSetSpinePointer) Load() spanSetSpinePointer {
	return spanSetSpinePointer{s.a.Load()}
}

// Stores the spanSetSpinePointer.
//
// It has the same semantics as [atomic.UnsafePointer].
func (s *atomicSpanSetSpinePointer) StoreNoWB(p spanSetSpinePointer) {
	s.a.StoreNoWB(p.p)
}

// spanSetSpinePointer represents a pointer to a contiguous block of atomic.Pointer[spanSetBlock].
type spanSetSpinePointer struct {
	p unsafe.Pointer
}

// lookup returns &s[idx].
func (s spanSetSpinePointer) lookup(idx uintptr) *atomic.Pointer[spanSetBlock] {
	return (*atomic.Pointer[spanSetBlock])(add(s.p, goarch.PtrSize*idx))
}

// spanSetBlockPool is a global pool of spanSetBlocks.
var spanSetBlockPool spanSetBlockAlloc

// spanSetBlockAlloc represents a concurrent pool of spanSetBlocks.
type spanSetBlockAlloc struct {
	stack lfstack
}

// alloc tries to grab a spanSetBlock out of the pool, and if it fails
// persistentallocs a new one and returns it.
func (p *spanSetBlockAlloc) alloc() *spanSetBlock {
	if s := (*spanSetBlock)(p.stack.pop()); s != nil {
		return s
	}
	return (*spanSetBlock)(persistentalloc(unsafe.Sizeof(spanSetBlock{}), cpu.CacheLineSize, &memstats.gcMiscSys))
}

// free returns a spanSetBlock back to the pool.
func (p *spanSetBlockAlloc) free(block *spanSetBlock) {
	block.popped.Store(0)
	p.stack.push(&block.lfnode)
}

// headTailIndex represents a combined 32-bit head and 32-bit tail
// of a queue into a single 64-bit value.
type headTailIndex uint64

// makeHeadTailIndex creates a headTailIndex value from a separate
// head and tail.
func makeHeadTailIndex(head, tail uint32) headTailIndex {
	return headTailIndex(uint64(head)<<32 | uint64(tail))
}

// head returns the head of a headTailIndex value.
func (h headTailIndex) head() uint32 {
	return uint32(h >> 32)
}

// tail returns the tail of a headTailIndex value.
func (h headTailIndex) tail() uint32 {
	return uint32(h)
}

// split splits the headTailIndex value into its parts.
func (h headTailIndex) split() (head uint32, tail uint32) {
	return h.head(), h.tail()
}

// atomicHeadTailIndex is an atomically-accessed headTailIndex.
type atomicHeadTailIndex struct {
	u atomic.Uint64
}

// load atomically reads a headTailIndex value.
func (h *atomicHeadTailIndex) load() headTailIndex {
	return headTailIndex(h.u.Load())
}

// cas atomically compares-and-swaps a headTailIndex value.
func (h *atomicHeadTailIndex) cas(old, new headTailIndex) bool {
	return h.u.CompareAndSwap(uint64(old), uint64(new))
}

// incHead atomically increments the head of a headTailIndex.
func (h *atomicHeadTailIndex) incHead() headTailIndex {
	return headTailIndex(h.u.Add(1 << 32))
}

// decHead atomically decrements the head of a headTailIndex.
func (h *atomicHeadTailIndex) decHead() headTailIndex {
	return headTailIndex(h.u.Add(-(1 << 32)))
}

// incTail atomically increments the tail of a headTailIndex.
func (h *atomicHeadTailIndex) incTail() headTailIndex {
	ht := headTailIndex(h.u.Add(1))
	// Check for overflow.
	if ht.tail() == 0 {
		print("runtime: head = ", ht.head(), ", tail = ", ht.tail(), "\n")
		throw("headTailIndex overflow")
	}
	return ht
}

// reset clears the headTailIndex to (0, 0).
func (h *atomicHeadTailIndex) reset() {
	h.u.Store(0)
}

// atomicMSpanPointer is an atomic.Pointer[mspan]. Can't use generics because it's NotInHeap.
type atomicMSpanPointer struct {
	p atomic.UnsafePointer
}

// Load returns the *mspan.
func (p *atomicMSpanPointer) Load() *mspan {
	return (*mspan)(p.p.Load())
}

// Store stores an *mspan.
func (p *atomicMSpanPointer) StoreNoWB(s *mspan) {
	p.p.StoreNoWB(unsafe.Pointer(s))
}
b IDATxytVսϓ22 A@IR :hCiZ[v*E:WũZA ^dQeQ @ !jZ'>gsV仿$|?g)&x-EIENT ;@xT.i%-X}SvS5.r/UHz^_$-W"w)Ɗ/@Z &IoX P$K}JzX:;` &, ŋui,e6mX ԵrKb1ԗ)DADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADA݀!I*]R;I2$eZ#ORZSrr6mteffu*((Pu'v{DIߔ4^pIm'77WEEE;vƎ4-$]'RI{\I&G :IHJ DWBB=\WR޽m o$K(V9ABB.}jѢv`^?IOȅ} ڶmG}T#FJ`56$-ھ}FI&v;0(h;Б38CӧOWf!;A i:F_m9s&|q%=#wZprrrla A &P\\СC[A#! {olF} `E2}MK/vV)i{4BffV\|ۭX`b@kɶ@%i$K z5zhmX[IXZ` 'b%$r5M4º/l ԃߖxhʔ)[@=} K6IM}^5k㏷݆z ΗÿO:gdGBmyT/@+Vɶ纽z񕏵l.y޴it뭷zV0[Y^>Wsqs}\/@$(T7f.InݺiR$푔n.~?H))\ZRW'Mo~v Ov6oԃxz! S,&xm/yɞԟ?'uaSѽb,8GלKboi&3t7Y,)JJ c[nzӳdE&KsZLӄ I?@&%ӟ۶mSMMњ0iؐSZ,|J+N ~,0A0!5%Q-YQQa3}$_vVrf9f?S8`zDADADADADADADADADAdqP,تmMmg1V?rSI꒟]u|l RCyEf٢9 jURbztѰ!m5~tGj2DhG*{H9)꒟ר3:(+3\?/;TUݭʴ~S6lڧUJ*i$d(#=Yݺd{,p|3B))q:vN0Y.jkק6;SɶVzHJJЀ-utѹսk>QUU\޲~]fFnK?&ߡ5b=z9)^|u_k-[y%ZNU6 7Mi:]ۦtk[n X(e6Bb."8cۭ|~teuuw|ήI-5"~Uk;ZicEmN/:]M> cQ^uiƞ??Ңpc#TUU3UakNwA`:Y_V-8.KKfRitv޲* 9S6ֿj,ՃNOMߤ]z^fOh|<>@Å5 _/Iu?{SY4hK/2]4%it5q]GGe2%iR| W&f*^]??vq[LgE_3f}Fxu~}qd-ږFxu~I N>\;͗O֊:̗WJ@BhW=y|GgwܷH_NY?)Tdi'?խwhlmQi !SUUsw4kӺe4rfxu-[nHtMFj}H_u~w>)oV}(T'ebʒv3_[+vn@Ȭ\S}ot}w=kHFnxg S 0eޢm~l}uqZfFoZuuEg `zt~? b;t%>WTkķh[2eG8LIWx,^\thrl^Ϊ{=dž<}qV@ ⠨Wy^LF_>0UkDuʫuCs$)Iv:IK;6ֲ4{^6եm+l3>݆uM 9u?>Zc }g~qhKwڭeFMM~pМuqǿz6Tb@8@Y|jx](^]gf}M"tG -w.@vOqh~/HII`S[l.6nØXL9vUcOoB\xoǤ'T&IǍQw_wpv[kmO{w~>#=P1Pɞa-we:iǏlHo׈꒟f9SzH?+shk%Fs:qVhqY`jvO'ρ?PyX3lх]˾uV{ݞ]1,MzYNW~̈́ joYn}ȚF߾׮mS]F z+EDxm/d{F{-W-4wY듏:??_gPf ^3ecg ҵs8R2מz@TANGj)}CNi/R~}c:5{!ZHӋӾ6}T]G]7W6^n 9*,YqOZj:P?Q DFL|?-^.Ɵ7}fFh׶xe2Pscz1&5\cn[=Vn[ĶE鎀uˌd3GII k;lNmشOuuRVfBE]ۣeӶu :X-[(er4~LHi6:Ѻ@ԅrST0trk%$Č0ez" *z"T/X9|8.C5Feg}CQ%͞ˣJvL/?j^h&9xF`њZ(&yF&Iݻfg#W;3^{Wo^4'vV[[K';+mӍִ]AC@W?1^{එyh +^]fm~iԵ]AB@WTk̏t uR?l.OIHiYyԶ]Aˀ7c:q}ힽaf6Z~қm(+sK4{^6}T*UUu]n.:kx{:2 _m=sAߤU@?Z-Vކеz왍Nэ{|5 pڶn b p-@sPg]0G7fy-M{GCF'%{4`=$-Ge\ eU:m+Zt'WjO!OAF@ik&t݆ϥ_ e}=]"Wz_.͜E3leWFih|t-wZۍ-uw=6YN{6|} |*={Ѽn.S.z1zjۻTH]흾 DuDvmvK.`V]yY~sI@t?/ϓ. m&["+P?MzovVЫG3-GRR[(!!\_,^%?v@ҵő m`Y)tem8GMx.))A]Y i`ViW`?^~!S#^+ѽGZj?Vģ0.))A꨷lzL*]OXrY`DBBLOj{-MH'ii-ϰ ok7^ )쭡b]UXSְmռY|5*cֽk0B7镹%ڽP#8nȎq}mJr23_>lE5$iwui+ H~F`IjƵ@q \ @#qG0".0" l`„.0! ,AQHN6qzkKJ#o;`Xv2>,tێJJ7Z/*A .@fفjMzkg @TvZH3Zxu6Ra'%O?/dQ5xYkU]Rֽkق@DaS^RSּ5|BeHNN͘p HvcYcC5:y #`οb;z2.!kr}gUWkyZn=f Pvsn3p~;4p˚=ē~NmI] ¾ 0lH[_L hsh_ғߤc_њec)g7VIZ5yrgk̞W#IjӪv>՞y睝M8[|]\շ8M6%|@PZڨI-m>=k='aiRo-x?>Q.}`Ȏ:Wsmu u > .@,&;+!!˱tﭧDQwRW\vF\~Q7>spYw$%A~;~}6¾ g&if_=j,v+UL1(tWake:@Ș>j$Gq2t7S?vL|]u/ .(0E6Mk6hiۺzښOrifޱxm/Gx> Lal%%~{lBsR4*}{0Z/tNIɚpV^#Lf:u@k#RSu =S^ZyuR/.@n&΃z~B=0eg뺆#,Þ[B/?H uUf7y Wy}Bwegל`Wh(||`l`.;Ws?V@"c:iɍL֯PGv6zctM̠':wuW;d=;EveD}9J@B(0iհ bvP1{\P&G7D޴Iy_$-Qjm~Yrr&]CDv%bh|Yzni_ˆR;kg}nJOIIwyuL}{ЌNj}:+3Y?:WJ/N+Rzd=hb;dj͒suݔ@NKMԄ jqzC5@y°hL m;*5ezᕏ=ep XL n?מ:r`۵tŤZ|1v`V뽧_csج'ߤ%oTuumk%%%h)uy]Nk[n 'b2 l.=͜E%gf$[c;s:V-͞WߤWh-j7]4=F-X]>ZLSi[Y*We;Zan(ӇW|e(HNNP5[= r4tP &0<pc#`vTNV GFqvTi*Tyam$ߏWyE*VJKMTfFw>'$-ؽ.Ho.8c"@DADADADADADADADADA~j*֘,N;Pi3599h=goضLgiJ5փy~}&Zd9p֚ e:|hL``b/d9p? fgg+%%hMgXosج, ΩOl0Zh=xdjLmhݻoO[g_l,8a]٭+ӧ0$I]c]:粹:Teꢢ"5a^Kgh,&= =՟^߶“ߢE ܹS J}I%:8 IDAT~,9/ʃPW'Mo}zNƍ쨓zPbNZ~^z=4mswg;5 Y~SVMRXUյڱRf?s:w ;6H:ºi5-maM&O3;1IKeamZh͛7+##v+c ~u~ca]GnF'ټL~PPPbn voC4R,ӟgg %hq}@#M4IÇ Oy^xMZx ) yOw@HkN˖-Sǎmb]X@n+i͖!++K3gd\$mt$^YfJ\8PRF)77Wא!Cl$i:@@_oG I{$# 8磌ŋ91A (Im7֭>}ߴJq7ޗt^ -[ԩSj*}%]&' -ɓ'ꫯVzzvB#;a 7@GxI{j޼ƌ.LÇWBB7`O"I$/@R @eee@۷>}0,ɒ2$53Xs|cS~rpTYYY} kHc %&k.], @ADADADADADADADADA@lT<%''*Lo^={رc5h %$+CnܸQ3fҥK}vUVVs9G R,_{xˇ3o߾;TTTd}馛]uuuG~iԩ@4bnvmvfϞ /Peeeq}}za I~,誫{UWW뮻}_~YƍSMMMYχ֝waw\ďcxꩧtEƍկ_?۷5@u?1kNׯWzz/wy>}zj3 k(ٺuq_Zvf̘:~ ABQ&r|!%KҥKgԞ={<_X-z !CyFUUz~ ABQIIIjݺW$UXXDٳZ~ ABQƍecW$<(~<RSSvZujjjԧOZQu@4 8m&&&jԩg$ď1h ͟?_{768@g =@`)))5o6m3)ѣƌJ;wҿUTT /KZR{~a=@0o<*狔iFɶ[ˎ;T]]OX@?K.ۈxN pppppppppppppppppPfl߾] ,{ァk۶mڿo5BTӦMӴiӴ|r DB2e|An!Dy'tkΝ[A $***t5' "!駟oaDnΝ:t֭[gDШQ06qD;@ x M6v(PiizmZ4ew"@̴ixf [~-Fٱc&IZ2|n!?$@{[HTɏ#@hȎI# _m(F /6Z3z'\r,r!;w2Z3j=~GY7"I$iI.p_"?pN`y DD?: _  Gÿab7J !Bx@0 Bo cG@`1C[@0G @`0C_u V1 aCX>W ` | `!<S `"<. `#c`?cAC4 ?c p#~@0?:08&_MQ1J h#?/`7;I  q 7a wQ A 1 Hp !#<8/#@1Ul7=S=K.4Z?E_$i@!1!E4?`P_  @Bă10#: "aU,xbFY1 [n|n #'vEH:`xb #vD4Y hi.i&EΖv#O H4IŶ}:Ikh @tZRF#(tXҙzZ ?I3l7q@õ|ۍ1,GpuY Ꮿ@hJv#xxk$ v#9 5 }_$c S#=+"K{F*m7`#%H:NRSp6I?sIՖ{Ap$I$I:QRv2$Z @UJ*$]<FO4IENDB`