diff --git a/gnovm/pkg/gnolang/machine.go b/gnovm/pkg/gnolang/machine.go index fad630766ca..d6210155ead 100644 --- a/gnovm/pkg/gnolang/machine.go +++ b/gnovm/pkg/gnolang/machine.go @@ -51,6 +51,17 @@ type Machine struct { // accounting is unaffected by pooling. blockPool []*Block + // callArgsScratch is the reusable buffer doOpCall hands to + // popCopyArgs; the args are copied into the call block immediately, + // so the slice never outlives call setup. Cleared after each use. + callArgsScratch []TypedValue + + // convertScratch is doOpConvert's working slot: ConvertTo and + // IsReadonly take the value's address, which would make a local + // escape to the heap once per conversion; a machine field's address + // is free. Transient within the op; cleared after each use. + convertScratch TypedValue + // Configuration Output io.Writer Store Store @@ -274,6 +285,12 @@ func (m *Machine) Release() { Stmts: stmts, Blocks: blocks, Frames: frames, + // NOTE: blockPool and callArgsScratch are deliberately dropped: + // carrying them through the cross-goroutine machinePool was + // measured to hurt parallel workloads (pooled blocks keep extra + // heap live across GC cycles and lose cache locality when the + // machine migrates cores) without helping machine-churn + // workloads, since sync.Pool eviction discards them anyway. } machinePool.Put(m) } diff --git a/gnovm/pkg/gnolang/nodes.go b/gnovm/pkg/gnolang/nodes.go index 5ad57050cba..df4e45bf8bf 100644 --- a/gnovm/pkg/gnolang/nodes.go +++ b/gnovm/pkg/gnolang/nodes.go @@ -750,6 +750,13 @@ type constTypeExpr struct { Last BlockNode // for GetTypeExprForExpr to resolve a *NameExpr. Source Expr // (preprocessed) source of this value. Type Type // (jae) just `Type`? ConstExpr does it... + + // cachedValue is the boxed TypedValue form of Type, filled by + // toConstTypeExpr so that evaluating the node doesn't re-box the + // type on every evaluation. Not persisted; nodes loaded from the + // store fall back to boxing in doOpEval (read-only there — no lazy + // fill, as nodes can be shared across machines). + cachedValue TypedValue } // ---------------------------------------- diff --git a/gnovm/pkg/gnolang/op_call.go b/gnovm/pkg/gnolang/op_call.go index 730766c36f5..0983a1cb969 100644 --- a/gnovm/pkg/gnolang/op_call.go +++ b/gnovm/pkg/gnolang/op_call.go @@ -316,11 +316,16 @@ func (m *Machine) doOpCall() { if !fr.Receiver.IsUndefined() { bft = ft.BoundType() } - args := m.popCopyArgs(bft, fr.NumArgs, fr.IsVarg, fr.Receiver) + args := m.popCopyArgs(bft, fr.NumArgs, fr.IsVarg, fr.Receiver, m.callArgsScratch) // Assign parameters in forward order. for i, argtv := range args { b.Values[i].AssignToBlock(argtv) } + // args is machine-owned scratch consumed by the loop above: keep the + // (possibly grown) buffer for the next call and drop the value + // references so they don't outlive the call setup. + clear(args) + m.callArgsScratch = args[:0] // Inherit fr.Cur from the block for crossing functions entered without // a cross-call (doOpPrecall sets fr.Cur only for cross-call entries). // Bound-method receivers occupy block[0], so cur is at block[1] for @@ -624,17 +629,29 @@ func (m *Machine) doOpReturnCallDefers() { // numArgs: number of arguments provided. // isVarg: true if called with ...varg. // recv: receiver if bound otherwise undefined. +// scratch: optional reusable buffer; used when its capacity suffices, in +// which case the result aliases it and must not outlive the caller (pass +// nil when the result escapes, as in doOpDefer). // Returns a slice of parameters with receiver (if any) and varg conversion. // For bound method calls the returned slice is 1 greater than len(ft.Params). -// Constructed varg slice is allocated, but the result slice is not. -func (m *Machine) popCopyArgs(ft *FuncType, numArgs int, isVarg bool, recv TypedValue) []TypedValue { +// Constructed varg slice is allocated; the result slice aliases scratch when +// its capacity suffices and is heap-allocated otherwise (not VM-allocated +// either way). +func (m *Machine) popCopyArgs(ft *FuncType, numArgs int, isVarg bool, recv TypedValue, scratch []TypedValue) []TypedValue { pts := ft.Params numParams := len(pts) isMethod := 0 if !recv.IsUndefined() { isMethod = 1 } - args := make([]TypedValue, isMethod+numParams) + var args []TypedValue + if cap(scratch) >= isMethod+numParams { + // Every slot is overwritten below: args[0] for methods, the rest + // by PopCopyValues and the varg assignment. + args = scratch[:isMethod+numParams] + } else { + args = make([]TypedValue, isMethod+numParams) + } if isMethod == 1 { args[0] = recv } @@ -689,7 +706,8 @@ func (m *Machine) doOpDefer() { baseOf(ftv.T).(*FuncType), numArgs, ds.Call.Varg, - TypedValue{}) + TypedValue{}, + nil) // args escape into the Defer. cfr.PushDefer(Defer{ Func: fv, Args: args, @@ -703,7 +721,8 @@ func (m *Machine) doOpDefer() { baseOf(ftv.T).(*FuncType), numArgs, ds.Call.Varg, - recv) + recv, + nil) // args escape into the Defer. cfr.PushDefer(Defer{ Func: fv, IsBoundMethod: true, diff --git a/gnovm/pkg/gnolang/op_decl.go b/gnovm/pkg/gnolang/op_decl.go index a63b2e30e8b..800cd5f5d5f 100644 --- a/gnovm/pkg/gnolang/op_decl.go +++ b/gnovm/pkg/gnolang/op_decl.go @@ -35,10 +35,19 @@ func (m *Machine) doOpValueDecl() { if isUntyped(tv.T) { if !s.Const { - if m.Stage != StagePre && rvs[i].T.Kind() != BoolKind { - panic("untyped conversion should not happen at runtime") + if m.Stage != StagePre { + // Only untyped bools (from comparisons) reach here + // at runtime; retype directly. This also keeps + // tv's address out of ConvertUntypedTo, which + // would otherwise make tv escape to the heap once + // per declaration executed. + if rvs[i].T.Kind() != BoolKind { + panic("untyped conversion should not happen at runtime") + } + tv.T = BoolType + } else { + tv = convertUntypedByValue(tv) } - ConvertUntypedTo(&tv, nil) } } else if nt != nil { // if nt.T is an interface, maintain tv.T as-is. @@ -71,3 +80,11 @@ func (m *Machine) doOpTypeDecl() { ptr := last.GetPointerTo(m.Store, s.Path) ptr.Assign2(m, m.Alloc, m.Store, m.Realm, tv, false) } + +// convertUntypedByValue performs the preprocess-stage (cold path) untyped +// conversion for doOpValueDecl by value, so that the caller's hot-path +// variable never has its address taken. +func convertUntypedByValue(tv TypedValue) TypedValue { + ConvertUntypedTo(&tv, nil) + return tv +} diff --git a/gnovm/pkg/gnolang/op_eval.go b/gnovm/pkg/gnolang/op_eval.go index 9398e1eea17..63557c14cc8 100644 --- a/gnovm/pkg/gnolang/op_eval.go +++ b/gnovm/pkg/gnolang/op_eval.go @@ -330,7 +330,14 @@ func (m *Machine) doOpEval() { case *constTypeExpr: m.PopExpr() // push preprocessed type as value - m.PushValue(asValue(x.Type)) + tv := x.cachedValue + if tv.T == nil { + // Node loaded from the store: the cache is not persisted. + // Read-only fallback (no lazy fill — nodes can be shared + // across machines). + tv = asValue(x.Type) + } + m.PushValue(tv) case *FieldTypeExpr: m.PushOp(OpFieldType) // evaluate field type diff --git a/gnovm/pkg/gnolang/op_expressions.go b/gnovm/pkg/gnolang/op_expressions.go index 3438420b706..0a8a97c70e4 100644 --- a/gnovm/pkg/gnolang/op_expressions.go +++ b/gnovm/pkg/gnolang/op_expressions.go @@ -731,7 +731,11 @@ func (m *Machine) doOpFuncLit() { } func (m *Machine) doOpConvert() { - xv := m.PopValue().Copy(m.Alloc) + // Work on a machine-owned scratch slot: ConvertTo and IsReadonly take + // the value's address, which would otherwise make a local escape to + // the heap once per conversion executed. + m.convertScratch = m.PopValue().Copy(m.Alloc) + xv := &m.convertScratch t := m.PopValue().GetType() // Gas based on conversion variant. @@ -757,7 +761,7 @@ func (m *Machine) doOpConvert() { // victim authority. Blocking the conversion here (m.IsReadonly is // the cross-realm ownership check) stops the launder before any // method dispatch. See zrealm_launder_g_typepun.gno. - if xv.T != nil && !xv.T.IsImmutable() && m.IsReadonly(&xv) { + if xv.T != nil && !xv.T.IsImmutable() && m.IsReadonly(xv) { if xvdt, ok := xv.T.(*DeclaredType); ok && xvdt.PkgPath == m.Realm.Path { // Except allow conversion when xv.T is m.Realm's own @@ -801,6 +805,7 @@ func (m *Machine) doOpConvert() { } } - ConvertTo(m.Alloc, m.Store, &xv, t, false) - m.PushValue(xv) + ConvertTo(m.Alloc, m.Store, xv, t, false) + m.PushValue(*xv) + m.convertScratch = TypedValue{} } diff --git a/gnovm/pkg/gnolang/preprocess.go b/gnovm/pkg/gnolang/preprocess.go index 7387746b3fb..35cbabf2812 100644 --- a/gnovm/pkg/gnolang/preprocess.go +++ b/gnovm/pkg/gnolang/preprocess.go @@ -4403,6 +4403,7 @@ func toConstTypeExpr(last BlockNode, source Expr, t Type) *constTypeExpr { cx.Last = last cx.Source = source cx.Type = t + cx.cachedValue = asValue(t) cx.SetSpan(source.GetSpan()) setPreprocessed(cx) return cx diff --git a/gnovm/pkg/gnolang/values.go b/gnovm/pkg/gnolang/values.go index 2debb9dd789..13456efe015 100644 --- a/gnovm/pkg/gnolang/values.go +++ b/gnovm/pkg/gnolang/values.go @@ -109,8 +109,12 @@ func (biv *BigintValue) UnmarshalAmino(s string) error { return nil } +// Copy returns biv itself: BigintValues are immutable at runtime (all +// arithmetic allocates fresh results, conversions only read), so the +// underlying *big.Int can be shared. No allocator charge was ever made +// here, so sharing is gas-neutral. func (biv BigintValue) Copy(alloc *Allocator) BigintValue { - return BigintValue{V: big.NewInt(0).Set(biv.V)} + return biv } // ---------------------------------------- @@ -138,13 +142,11 @@ func (bdv *BigdecValue) UnmarshalAmino(s string) error { return nil } +// Copy returns bdv itself: like BigintValues, BigdecValues are immutable at +// runtime (apd operations write into fresh receivers), so the underlying +// *apd.Decimal can be shared. Gas-neutral: no allocator charge existed here. func (bdv BigdecValue) Copy(alloc *Allocator) BigdecValue { - cp := apd.New(0, 0) - _, err := apd.BaseContext.Add(cp, cp, bdv.V) - if err != nil { - panic("should not happen") - } - return BigdecValue{V: cp} + return bdv } // ----------------------------------------