diff --git a/runtime/build.go b/runtime/build.go index 8afce1cc69..0764eadce5 100644 --- a/runtime/build.go +++ b/runtime/build.go @@ -61,6 +61,7 @@ var altPkgs = map[string]altPkgSpec{ "reflect": {mode: altPkgReplace}, "runtime": {mode: altPkgReplace}, "sync/atomic": {mode: altPkgReplace}, + "sync": {mode: altPkgReplace}, "syscall/js": {mode: altPkgReplace}, "syscall": {mode: altPkgReplace}, "unique": {mode: altPkgReplace}, diff --git a/runtime/internal/lib/runtime/sync_runtime_llgo.go b/runtime/internal/lib/runtime/sync_runtime_llgo.go index cee3441a8d..8cddac6795 100644 --- a/runtime/internal/lib/runtime/sync_runtime_llgo.go +++ b/runtime/internal/lib/runtime/sync_runtime_llgo.go @@ -2,7 +2,10 @@ package runtime -import _ "unsafe" +import ( + _ "sync/atomic" + _ "unsafe" +) var poolCleanup func() diff --git a/runtime/internal/lib/runtime/unique_runtime_llgo.go b/runtime/internal/lib/runtime/unique_runtime_llgo.go index 887db777cf..26edac0047 100644 --- a/runtime/internal/lib/runtime/unique_runtime_llgo.go +++ b/runtime/internal/lib/runtime/unique_runtime_llgo.go @@ -3,8 +3,9 @@ package runtime import ( - "sync" _ "unsafe" + + "github.com/goplus/llgo/runtime/internal/clite/pthread/sync" ) var ( @@ -13,15 +14,17 @@ var ( ) //go:linkname unique_runtime_registerUniqueMapCleanup unique.runtime_registerUniqueMapCleanup -func unique_runtime_registerUniqueMapCleanup(cleanup func()) { +func unique_runtime_registerUniqueMapCleanup(f func()) { uniqueMapCleanupOnce.Do(func() { uniqueMapCleanup = make(chan struct{}, 1) - go func() { - for range uniqueMapCleanup { - cleanup() - } - }() }) + // Start the goroutine in the runtime so it's counted as a system goroutine. + go func(cleanup func()) { + for { + <-uniqueMapCleanup + cleanup() + } + }(f) } func unique_runtime_notifyMapCleanup() { diff --git a/runtime/internal/lib/sync/pool.go b/runtime/internal/lib/sync/pool.go new file mode 100644 index 0000000000..08227e713d --- /dev/null +++ b/runtime/internal/lib/sync/pool.go @@ -0,0 +1,184 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "sync" + "sync/atomic" + "unsafe" + + "github.com/goplus/llgo/runtime/internal/clite/tls" +) + +// A Pool is a set of temporary objects that may be individually saved and +// retrieved. +// +// Any item stored in the Pool may be removed automatically at any time without +// notification. If the Pool holds the only reference when this happens, the +// item might be deallocated. +// +// A Pool is safe for use by multiple goroutines simultaneously. +// +// Pool's purpose is to cache allocated but unused items for later reuse, +// relieving pressure on the garbage collector. That is, it makes it easy to +// build efficient, thread-safe free lists. However, it is not suitable for all +// free lists. +// +// An appropriate use of a Pool is to manage a group of temporary items +// silently shared among and potentially reused by concurrent independent +// clients of a package. Pool provides a way to amortize allocation overhead +// across many clients. +// +// An example of good use of a Pool is in the fmt package, which maintains a +// dynamically-sized store of temporary output buffers. The store scales under +// load (when many goroutines are actively printing) and shrinks when +// quiescent. +// +// On the other hand, a free list maintained as part of a short-lived object is +// not a suitable use for a Pool, since the overhead does not amortize well in +// that scenario. It is more efficient to have such objects implement their own +// free list. +// +// A Pool must not be copied after first use. +// +// In the terminology of [the Go memory model], a call to Put(x) “synchronizes before” +// a call to [Pool.Get] returning that same value x. +// Similarly, a call to New returning x “synchronizes before” +// a call to Get returning that same value x. +// +// [the Go memory model]: https://go.dev/ref/mem +type Pool struct { + noCopy noCopy + + local unsafe.Pointer // local fixed-size per-P pool, actual type is [P]poolLocal + localSize uintptr // size of the local array + + victim unsafe.Pointer // local from previous cycle + victimSize uintptr // size of victims array + + // New optionally specifies a function to generate + // a value when Get would otherwise return nil. + // It may not be changed concurrently with calls to Get. + New func() any + once sync.Once +} + +// Local per-P Pool appendix. +type poolLocalInternal struct { + private any // Can be used only by the respective P. + shared poolChain // Local P can pushHead/popHead; any P can popTail. +} + +type poolLocal struct { + poolLocalInternal + + // Prevents false sharing on widespread platforms with + // 128 mod (cache line size) = 0 . + pad [128 - unsafe.Sizeof(poolLocalInternal{})%128]byte +} + +// Put adds x to the pool. +func (p *Pool) Put(x any) { + if x == nil { + return + } + l, _ := p.pin() + if l.private == nil { + l.private = x + } else { + l.shared.pushHead(x) + } +} + +// Get selects an arbitrary item from the [Pool], removes it from the +// Pool, and returns it to the caller. +// Get may choose to ignore the pool and treat it as empty. +// Callers should not assume any relation between values passed to [Pool.Put] and +// the values returned by Get. +// +// If Get would otherwise return nil and p.New is non-nil, Get returns +// the result of calling p.New. +func (p *Pool) Get() any { + l, _ := p.pin() + x := l.private + l.private = nil + if x == nil { + x, _ = l.shared.popHead() + if x == nil { + x = p.getSlow(0) + } + } + if x == nil && p.New != nil { + x = p.New() + } + return x +} + +// pin pins the current goroutine to P, disables preemption and +// returns poolLocal pool for the P and the P's id. +// Caller must call runtime_procUnpin() when done with the pool. +func (p *Pool) pin() (*poolLocal, int) { + // Check whether p is nil to get a panic. + // Otherwise the nil dereference happens while the m is pinned, + // causing a fatal error rather than a panic. + if p == nil { + panic("nil Pool") + } + + if ptr := atomic.LoadPointer(&p.local); ptr != nil { + handle := (*tls.Handle[*poolLocal])(ptr) + l := handle.Get() + if l == nil { + l = &poolLocal{} + handle.Set(l) + } + return l, 0 + } + + return p.pinSlow() +} + +func (p *Pool) pinSlow() (*poolLocal, int) { + p.once.Do(func() { + handle := tls.Alloc[*poolLocal](func(head **poolLocal) { + if head != nil { + atomic.StorePointer(&p.victim, unsafe.Pointer(*head)) + } + }) + atomic.StorePointer(&p.local, unsafe.Pointer(&handle)) + }) + handle := (*tls.Handle[*poolLocal])(p.local) + l := &poolLocal{} + handle.Set(l) + return l, 0 +} + +func (p *Pool) getSlow(pid int) any { + if ptr := atomic.LoadPointer(&p.victim); ptr != nil { + l := (*poolLocal)(ptr) + if x := l.private; x != nil { + l.private = nil + return x + } + if x, _ := l.shared.popTail(); x != nil { + return x + } + atomic.StorePointer(&p.victim, nil) + } + return nil +} + +// noCopy may be added to structs which must not be copied +// after the first use. +// +// See https://golang.org/issues/8005#issuecomment-190753527 +// for details. +// +// Note that it must not be embedded, due to the Lock and Unlock methods. +type noCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} diff --git a/runtime/internal/lib/sync/poolqueue.go b/runtime/internal/lib/sync/poolqueue.go new file mode 100644 index 0000000000..e9593f8c44 --- /dev/null +++ b/runtime/internal/lib/sync/poolqueue.go @@ -0,0 +1,302 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "sync/atomic" + "unsafe" +) + +// poolDequeue is a lock-free fixed-size single-producer, +// multi-consumer queue. The single producer can both push and pop +// from the head, and consumers can pop from the tail. +// +// It has the added feature that it nils out unused slots to avoid +// unnecessary retention of objects. This is important for sync.Pool, +// but not typically a property considered in the literature. +type poolDequeue struct { + // headTail packs together a 32-bit head index and a 32-bit + // tail index. Both are indexes into vals modulo len(vals)-1. + // + // tail = index of oldest data in queue + // head = index of next slot to fill + // + // Slots in the range [tail, head) are owned by consumers. + // A consumer continues to own a slot outside this range until + // it nils the slot, at which point ownership passes to the + // producer. + // + // The head index is stored in the most-significant bits so + // that we can atomically add to it and the overflow is + // harmless. + headTail atomic.Uint64 + + // vals is a ring buffer of interface{} values stored in this + // dequeue. The size of this must be a power of 2. + // + // vals[i].typ is nil if the slot is empty and non-nil + // otherwise. A slot is still in use until *both* the tail + // index has moved beyond it and typ has been set to nil. This + // is set to nil atomically by the consumer and read + // atomically by the producer. + vals []eface +} + +type eface struct { + typ, val unsafe.Pointer +} + +const dequeueBits = 32 + +// dequeueLimit is the maximum size of a poolDequeue. +// +// This must be at most (1<> dequeueBits) & mask) + tail = uint32(ptrs & mask) + return +} + +func (d *poolDequeue) pack(head, tail uint32) uint64 { + const mask = 1<= dequeueLimit { + // Can't make it any bigger. + newSize = dequeueLimit + } + + d2 := &poolChainElt{} + d2.prev.Store(d) + d2.vals = make([]eface, newSize) + c.head = d2 + d.next.Store(d2) + d2.pushHead(val) +} + +func (c *poolChain) popHead() (any, bool) { + d := c.head + for d != nil { + if val, ok := d.popHead(); ok { + return val, ok + } + // There may still be unconsumed elements in the + // previous dequeue, so try backing up. + d = d.prev.Load() + } + return nil, false +} + +func (c *poolChain) popTail() (any, bool) { + d := c.tail.Load() + if d == nil { + return nil, false + } + + for { + // It's important that we load the next pointer + // *before* popping the tail. In general, d may be + // transiently empty, but if next is non-nil before + // the pop and the pop fails, then d is permanently + // empty, which is the only condition under which it's + // safe to drop d from the chain. + d2 := d.next.Load() + + if val, ok := d.popTail(); ok { + return val, ok + } + + if d2 == nil { + // This is the only dequeue. It's empty right + // now, but could be pushed to in the future. + return nil, false + } + + // The tail of the chain has been drained, so move on + // to the next dequeue. Try to drop it from the chain + // so the next pop doesn't have to look at the empty + // dequeue again. + if c.tail.CompareAndSwap(d, d2) { + // We won the race. Clear the prev pointer so + // the garbage collector can collect the empty + // dequeue and so popHead doesn't back up + // further than necessary. + d2.prev.Store(nil) + } + d = d2 + } +}