diff --git a/blockdb/types/execdata_sections.go b/blockdb/types/execdata_sections.go index 810e6ed7..d78a484c 100644 --- a/blockdb/types/execdata_sections.go +++ b/blockdb/types/execdata_sections.go @@ -48,6 +48,27 @@ type FlatCallFrame struct { Error string `ssz-max:"10485760"` } +// TracePayloadLimit is the number of payload bytes kept per call frame for the +// Input and Output fields. A payload longer than the limit is captured as its +// first TracePayloadLimit+1 bytes, so a stored length above the limit is itself +// the marker that the value was truncated - no separate flag is needed on disk. +// +// Contracts can turn a single transaction's gas into hundreds of megabytes of +// tracer output by looping over calls that pass large memory buffers around +// (the identity precompile being the cheapest vehicle), so payloads are pruned +// while the tracer response is being read rather than after it is decoded. +const TracePayloadLimit = 16384 + +// TrimPrunedPayload splits a stored call frame payload into the bytes that are +// safe to display and whether the value was truncated when it was captured. +func TrimPrunedPayload(data []byte) (visible []byte, pruned bool) { + if len(data) > TracePayloadLimit { + return data[:TracePayloadLimit], true + } + + return data, false +} + // State change section version. const ( StateChangesVersion1 = 1 diff --git a/blockdb/types/execdata_sections_test.go b/blockdb/types/execdata_sections_test.go new file mode 100644 index 00000000..da777443 --- /dev/null +++ b/blockdb/types/execdata_sections_test.go @@ -0,0 +1,35 @@ +package types + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTrimPrunedPayload(t *testing.T) { + tests := []struct { + name string + size int + wantVisible int + wantPruned bool + }{ + {name: "empty", size: 0, wantVisible: 0, wantPruned: false}, + {name: "below limit", size: 128, wantVisible: 128, wantPruned: false}, + {name: "at limit", size: TracePayloadLimit, wantVisible: TracePayloadLimit, wantPruned: false}, + { + name: "one past limit marks truncation", + size: TracePayloadLimit + 1, + wantVisible: TracePayloadLimit, + wantPruned: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + visible, pruned := TrimPrunedPayload(bytes.Repeat([]byte{0xab}, test.size)) + assert.Len(t, visible, test.wantVisible) + assert.Equal(t, test.wantPruned, pruned) + }) + } +} diff --git a/clients/execution/rpc/calltrace_stream.go b/clients/execution/rpc/calltrace_stream.go new file mode 100644 index 00000000..d544da7e --- /dev/null +++ b/clients/execution/rpc/calltrace_stream.go @@ -0,0 +1,333 @@ +package rpc + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" +) + +// decodeCallTraceResults returns a decodeFn for streamRPCCall that walks the +// callTracer result array token by token, appending one CallTraceResult per +// transaction to *results. +// +// Decoding the array with json.Decoder.Decode, as streamDecodeArray does, +// makes the decoder buffer one whole transaction's trace before it hands the +// value over. A contract that loops over calls carrying large memory buffers +// turns a single transaction into hundreds of megabytes of tracer output, all +// of which would then have to sit in the decoder buffer at once. Walking the +// tree scalar by scalar bounds that buffer by the largest single value in the +// response instead, and payloadLimit bounds what is retained from each of +// those values. +func decodeCallTraceResults(results *[]CallTraceResult, payloadLimit int) func(*json.Decoder) error { + return func(dec *json.Decoder) error { + tok, err := dec.Token() + if err != nil { + return asDecodeError(fmt.Errorf("read trace array start: %w", err)) + } + + // Handle null result + if tok == nil { + return nil + } + + if tok != json.Delim('[') { + return &ResponseDecodeError{Err: fmt.Errorf("expected '[', got %v", tok)} + } + + for dec.More() { + var result CallTraceResult + + if err := decodeCallTraceResult(dec, &result, payloadLimit); err != nil { + return err + } + + *results = append(*results, result) + } + + if _, err := dec.Token(); err != nil { + return asDecodeError(fmt.Errorf("read trace array end: %w", err)) + } + + return nil + } +} + +// decodeCallTraceResult decodes one per-transaction entry of the callTracer +// result array. +func decodeCallTraceResult(dec *json.Decoder, result *CallTraceResult, payloadLimit int) error { + _, err := decodeJSONObject(dec, "trace result", func(key string) error { + switch key { + case "txHash": + return decodeScalar(dec, key, &result.TxHash) + + case "result": + call, err := decodeCallFrame(dec, payloadLimit) + if err != nil { + return err + } + + result.Result = call + + return nil + + default: + return skipJSONValue(dec) + } + }) + + return err +} + +// decodeCallFrame decodes one callTracer frame and the subtree below it, +// pruning the input and output payloads to payloadLimit as they are read. +// Returns nil when the frame is JSON null. +func decodeCallFrame(dec *json.Decoder, payloadLimit int) (*CallTraceCall, error) { + call := &CallTraceCall{} + + present, err := decodeJSONObject(dec, "call frame", func(key string) error { + switch key { + case "type": + return decodeScalar(dec, key, &call.Type) + case "from": + return decodeScalar(dec, key, &call.From) + case "to": + return decodeScalar(dec, key, &call.To) + case "value": + return decodeScalar(dec, key, &call.Value) + case "gas": + return decodeScalar(dec, key, &call.Gas) + case "gasUsed": + return decodeScalar(dec, key, &call.GasUsed) + case "error": + return decodeScalar(dec, key, &call.Error) + + case "input": + return decodePrunedPayload(dec, key, payloadLimit, false, (*[]byte)(&call.Input)) + case "output": + return decodePrunedPayload(dec, key, payloadLimit, false, (*[]byte)(&call.Output)) + case "revertReason": + return decodePrunedPayload(dec, key, payloadLimit, true, (*[]byte)(&call.RevertReason)) + + case "calls": + calls, err := decodeCallFrames(dec, payloadLimit) + if err != nil { + return err + } + + call.Calls = calls + + return nil + + default: + return skipJSONValue(dec) + } + }) + if err != nil { + return nil, err + } + + if !present { + return nil, nil + } + + return call, nil +} + +// decodeCallFrames decodes the nested "calls" array of a call frame. +func decodeCallFrames(dec *json.Decoder, payloadLimit int) ([]CallTraceCall, error) { + tok, err := dec.Token() + if err != nil { + return nil, asDecodeError(fmt.Errorf("read calls array start: %w", err)) + } + + if tok == nil { + return nil, nil + } + + if tok != json.Delim('[') { + return nil, &ResponseDecodeError{Err: fmt.Errorf("expected '[' for calls, got %v", tok)} + } + + calls := make([]CallTraceCall, 0, 4) + + for dec.More() { + call, err := decodeCallFrame(dec, payloadLimit) + if err != nil { + return nil, err + } + + if call != nil { + calls = append(calls, *call) + } + } + + if _, err := dec.Token(); err != nil { + return nil, asDecodeError(fmt.Errorf("read calls array end: %w", err)) + } + + return calls, nil +} + +// decodeJSONObject reads one JSON object, invoking decodeField for every key. +// decodeField must consume exactly the value belonging to the key it is given. +// Reports whether an object was present; a JSON null yields false. +func decodeJSONObject(dec *json.Decoder, what string, decodeField func(key string) error) (bool, error) { + tok, err := dec.Token() + if err != nil { + return false, asDecodeError(fmt.Errorf("read %s start: %w", what, err)) + } + + if tok == nil { + return false, nil + } + + if tok != json.Delim('{') { + return false, &ResponseDecodeError{Err: fmt.Errorf("expected '{' for %s, got %v", what, tok)} + } + + for dec.More() { + keyTok, err := dec.Token() + if err != nil { + return false, asDecodeError(fmt.Errorf("read %s key: %w", what, err)) + } + + key, ok := keyTok.(string) + if !ok { + return false, &ResponseDecodeError{ + Err: fmt.Errorf("expected string key in %s, got %T", what, keyTok), + } + } + + if err := decodeField(key); err != nil { + return false, err + } + } + + if _, err := dec.Token(); err != nil { + return false, asDecodeError(fmt.Errorf("read %s end: %w", what, err)) + } + + return true, nil +} + +// decodeScalar decodes the value of a single object field. +func decodeScalar(dec *json.Decoder, key string, target any) error { + if err := dec.Decode(target); err != nil { + return asDecodeError(fmt.Errorf("decode %s: %w", key, err)) + } + + return nil +} + +// skipJSONValue consumes exactly one JSON value without materialising it. +func skipJSONValue(dec *json.Decoder) error { + depth := 0 + + for { + tok, err := dec.Token() + if err != nil { + return asDecodeError(fmt.Errorf("skip value: %w", err)) + } + + if delim, ok := tok.(json.Delim); ok { + switch delim { + case '[', '{': + depth++ + case ']', '}': + depth-- + } + } + + if depth == 0 { + return nil + } + } +} + +// decodePrunedPayload decodes a hex payload field, keeping at most limit+1 +// bytes of it. +func decodePrunedPayload(dec *json.Decoder, key string, limit int, lenient bool, out *[]byte) error { + payload := prunedHex{limit: limit, lenient: lenient} + + if err := dec.Decode(&payload); err != nil { + return asDecodeError(fmt.Errorf("decode %s: %w", key, err)) + } + + *out = payload.data + + return nil +} + +// prunedHex decodes a hex string into at most limit+1 bytes. Keeping one byte +// past the limit is what marks the value as truncated for later consumers, see +// blockdb/types.TrimPrunedPayload. +// +// The bytes handed to UnmarshalJSON alias the decoder's own buffer, so the +// oversized part of a payload is discarded without ever being copied: what the +// tracer sent is bounded by the response itself, what dora keeps is bounded by +// the limit, and nothing in between is allocated. +// +// A lenient payload that is not valid hex is kept as raw text, matching +// LenientHexBytes - some clients report revert reasons in plain text. A limit +// of zero or less keeps payloads intact. +type prunedHex struct { + limit int + lenient bool + data []byte +} + +func (p *prunedHex) UnmarshalJSON(input []byte) error { + if bytes.Equal(input, []byte("null")) { + p.data = nil + + return nil + } + + // Hex payloads carry no escape sequences, so the quoted form can be sliced + // as-is. Anything else goes through the regular string decoder first. + if len(input) >= 2 && input[0] == '"' && input[len(input)-1] == '"' && + bytes.IndexByte(input, '\\') < 0 { + return p.store(input[1 : len(input)-1]) + } + + var str string + if err := json.Unmarshal(input, &str); err != nil { + return fmt.Errorf("expected hex string: %w", err) + } + + return p.store([]byte(str)) +} + +// store decodes the unquoted body of a hex payload, truncating it first. +func (p *prunedHex) store(body []byte) error { + if len(body) >= 2 && body[0] == '0' && (body[1] == 'x' || body[1] == 'X') { + body = body[2:] + } + + if maxChars := 2 * (p.limit + 1); p.limit > 0 && len(body) > maxChars { + body = body[:maxChars] + } + + if len(body) == 0 { + p.data = nil + + return nil + } + + decoded := make([]byte, len(body)/2) + if _, err := hex.Decode(decoded, body); err != nil { + if !p.lenient { + return fmt.Errorf("decode hex payload: %w", err) + } + + // body aliases the decoder buffer and must not outlive this call. + p.data = bytes.Clone(body) + + return nil + } + + p.data = decoded + + return nil +} diff --git a/clients/execution/rpc/calltrace_stream_test.go b/clients/execution/rpc/calltrace_stream_test.go new file mode 100644 index 00000000..f4ddd9be --- /dev/null +++ b/clients/execution/rpc/calltrace_stream_test.go @@ -0,0 +1,233 @@ +package rpc + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// decodeTraceArray runs the streaming call trace decoder over a raw result array. +func decodeTraceArray(t *testing.T, body string, payloadLimit int) ([]CallTraceResult, error) { + t.Helper() + + var results []CallTraceResult + + err := decodeCallTraceResults(&results, payloadLimit)(json.NewDecoder(strings.NewReader(body))) + + return results, err +} + +func TestDecodeCallTraceResults(t *testing.T) { + body := `[ + { + "txHash": "0x2222222222222222222222222222222222222222222222222222222222222222", + "result": { + "type": "CALL", + "from": "0xafe6578eb95f3030b7ffabbf6011a1dc2a4d7967", + "to": "0x722d316672c8be206c3d228d087d1dc948a61345", + "value": "0x1bc16d674ec80000", + "gas": "0x1000000", + "gasUsed": "0xfb8bd7", + "input": "0x00000055", + "output": "0xdeadbeef", + "calls": [ + { + "type": "STATICCALL", + "from": "0x722d316672c8be206c3d228d087d1dc948a61345", + "to": "0x0000000000000000000000000000000000000004", + "gas": "0x100", + "gasUsed": "0x30", + "input": "0xaabb", + "output": "0xaabb", + "unknownField": {"nested": [1, 2, {"deep": true}]} + }, + { + "type": "DELEGATECALL", + "from": "0x722d316672c8be206c3d228d087d1dc948a61345", + "to": "0x1111111111111111111111111111111111111111", + "gas": "0x50", + "gasUsed": "0x10", + "input": "0x", + "error": "execution reverted", + "calls": null + } + ] + } + }, + { + "txHash": "0x3333333333333333333333333333333333333333333333333333333333333333", + "result": null + } + ]` + + results, err := decodeTraceArray(t, body, 1024) + require.NoError(t, err) + require.Len(t, results, 2) + + root := results[0].Result + require.NotNil(t, root) + assert.Equal(t, "CALL", root.Type) + assert.Equal(t, "0x722d316672c8be206c3d228d087d1dc948a61345", strings.ToLower(root.To.Hex())) + assert.Equal(t, uint64(0x1000000), uint64(root.Gas)) + assert.Equal(t, uint64(0xfb8bd7), uint64(root.GasUsed)) + assert.Equal(t, "1bc16d674ec80000", root.Value.ToInt().Text(16)) + assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x55}, []byte(root.Input)) + assert.Equal(t, []byte{0xde, 0xad, 0xbe, 0xef}, []byte(root.Output)) + require.Len(t, root.Calls, 2) + + assert.Equal(t, "STATICCALL", root.Calls[0].Type) + assert.Equal(t, []byte{0xaa, 0xbb}, []byte(root.Calls[0].Input)) + + assert.Equal(t, "DELEGATECALL", root.Calls[1].Type) + assert.Equal(t, "execution reverted", root.Calls[1].Error) + assert.Empty(t, root.Calls[1].Input) + assert.Nil(t, root.Calls[1].Calls) + + assert.Nil(t, results[1].Result) +} + +func TestDecodeCallTraceResultsPrunesPayloads(t *testing.T) { + const limit = 64 + + // A frame that carries far more payload than the limit, the shape the + // memory-inflating trace attack produces on every one of its call frames. + oversized := strings.Repeat("ab", 8192) + body := fmt.Sprintf(`[{"txHash":"0x%064x","result":{ + "type":"STATICCALL", + "from":"0x%040x","to":"0x%040x", + "gas":"0x1","gasUsed":"0x1", + "input":"0x%s","output":"0x%s" + }}]`, 1, 2, 4, oversized, oversized) + + results, err := decodeTraceArray(t, body, limit) + require.NoError(t, err) + require.Len(t, results, 1) + + root := results[0].Result + require.NotNil(t, root) + + // One byte past the limit is what marks the payload as truncated. + assert.Len(t, []byte(root.Input), limit+1) + assert.Len(t, []byte(root.Output), limit+1) + assert.Equal(t, byte(0xab), root.Input[limit]) +} + +func TestDecodeCallTraceResultsKeepsPayloadsWithinLimit(t *testing.T) { + body := `[{"txHash":"0x2222222222222222222222222222222222222222222222222222222222222222", + "result":{"type":"CALL","from":"0x0000000000000000000000000000000000000001", + "to":"0x0000000000000000000000000000000000000002","gas":"0x1","gasUsed":"0x1", + "input":"0x0102030405"}}]` + + results, err := decodeTraceArray(t, body, 5) + require.NoError(t, err) + require.Len(t, results, 1) + + // Exactly at the limit, so nothing is truncated and no marker byte is added. + assert.Equal(t, []byte{1, 2, 3, 4, 5}, []byte(results[0].Result.Input)) +} + +func TestDecodeCallTraceResultsNullAndEmpty(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "null result", body: `null`}, + {name: "empty array", body: `[]`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + results, err := decodeTraceArray(t, test.body, 1024) + require.NoError(t, err) + assert.Empty(t, results) + }) + } +} + +func TestDecodeCallTraceResultsLenientRevertReason(t *testing.T) { + tests := []struct { + name string + reason string + expected []byte + }{ + {name: "hex", reason: "0x1234", expected: []byte{0x12, 0x34}}, + {name: "unprefixed hex", reason: "1234", expected: []byte{0x12, 0x34}}, + {name: "plain text", reason: "insufficient balance", expected: []byte("insufficient balance")}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + body := fmt.Sprintf(`[{"txHash":"0x%064x","result":{"type":"CALL", + "from":"0x%040x","to":"0x%040x","gas":"0x1","gasUsed":"0x1", + "revertReason":%q}}]`, 1, 2, 3, test.reason) + + results, err := decodeTraceArray(t, body, 1024) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, test.expected, []byte(results[0].Result.RevertReason)) + }) + } +} + +func TestDecodeCallTraceResultsStructuralErrorIsNotRetryable(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "not an array", body: `{"type":"CALL"}`}, + { + name: "frame is not an object", + body: `[{"txHash":"0x2222222222222222222222222222222222222222222222222222222222222222", + "result":"CALL"}]`, + }, + { + name: "wrong field type", + body: `[{"txHash":"0x2222222222222222222222222222222222222222222222222222222222222222", + "result":{"type":[1,2,3]}}]`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := decodeTraceArray(t, test.body, 1024) + require.Error(t, err) + + var decodeErr *ResponseDecodeError + assert.True(t, errors.As(err, &decodeErr), + "expected a ResponseDecodeError, got %T: %v", err, err) + }) + } +} + +func TestDecodeCallTraceResultsTruncatedResponseIsRetryable(t *testing.T) { + // A body that stops mid-array is an I/O level problem, so it must stay + // retryable against another client. + _, err := decodeTraceArray(t, `[{"txHash":"0x2222","result":{"type":"CA`, 1024) + require.Error(t, err) + + var decodeErr *ResponseDecodeError + assert.False(t, errors.As(err, &decodeErr), "unexpected ResponseDecodeError: %v", err) +} + +func TestSkipJSONValue(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(`{"a":[1,{"b":[[]]},null],"c":42}`)) + + _, err := dec.Token() // '{' + require.NoError(t, err) + + key, err := dec.Token() // "a" + require.NoError(t, err) + require.Equal(t, "a", key) + + require.NoError(t, skipJSONValue(dec)) + + // The skip must land exactly on the next key. + key, err = dec.Token() + require.NoError(t, err) + assert.Equal(t, "c", key) +} diff --git a/clients/execution/rpc/streaming.go b/clients/execution/rpc/streaming.go index 5a3818d1..4d52c701 100644 --- a/clients/execution/rpc/streaming.go +++ b/clients/execution/rpc/streaming.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -28,6 +29,46 @@ func (e *jsonRPCError) Error() string { return fmt.Sprintf("RPC error (code %d): %s", e.Code, e.Message) } +// ResponseDecodeError marks a failure that happened while decoding a response +// that was received in full. The shape of the payload rather than the +// connection is the problem, so repeating the same call against another client +// yields the same failure. +type ResponseDecodeError struct { + Err error +} + +func (e *ResponseDecodeError) Error() string { + return e.Err.Error() +} + +func (e *ResponseDecodeError) Unwrap() error { + return e.Err +} + +// asDecodeError classifies a decoding failure. Structural JSON problems become +// a ResponseDecodeError, while I/O and context failures pass through unchanged +// so callers can still fail over to another client. +func asDecodeError(err error) error { + if err == nil { + return nil + } + + var ( + decodeErr *ResponseDecodeError + syntaxErr *json.SyntaxError + typeErr *json.UnmarshalTypeError + ) + + switch { + case errors.As(err, &decodeErr): + return err + case errors.As(err, &syntaxErr), errors.As(err, &typeErr): + return &ResponseDecodeError{Err: err} + default: + return err + } +} + // streamRPCCall makes a JSON-RPC call and stream-decodes the "result" // field by passing a *json.Decoder to decodeFn. For HTTP endpoints, the // response body is streamed directly from the network, avoiding the @@ -117,11 +158,11 @@ func decodeStreamedJSONRPCResult( // Read opening '{' of JSON-RPC envelope t, err := dec.Token() if err != nil { - return fmt.Errorf("read response: %w", err) + return asDecodeError(fmt.Errorf("read response: %w", err)) } if t != json.Delim('{') { - return fmt.Errorf("expected '{', got %v", t) + return &ResponseDecodeError{Err: fmt.Errorf("expected '{', got %v", t)} } var foundResult bool @@ -129,19 +170,19 @@ func decodeStreamedJSONRPCResult( for dec.More() { keyTok, err := dec.Token() if err != nil { - return fmt.Errorf("read key: %w", err) + return asDecodeError(fmt.Errorf("read key: %w", err)) } key, ok := keyTok.(string) if !ok { - return fmt.Errorf("expected string key, got %T", keyTok) + return &ResponseDecodeError{Err: fmt.Errorf("expected string key, got %T", keyTok)} } switch key { case "error": var rpcErr jsonRPCError if err := dec.Decode(&rpcErr); err != nil { - return fmt.Errorf("decode error field: %w", err) + return asDecodeError(fmt.Errorf("decode error field: %w", err)) } return &rpcErr @@ -154,16 +195,15 @@ func decodeStreamedJSONRPCResult( foundResult = true default: - // Skip other fields (jsonrpc, id) - var skip json.RawMessage - if err := dec.Decode(&skip); err != nil { + // Skip other fields (jsonrpc, id) without materialising them. + if err := skipJSONValue(dec); err != nil { return fmt.Errorf("skip field %q: %w", key, err) } } } if !foundResult { - return fmt.Errorf("no 'result' field in JSON-RPC response") + return &ResponseDecodeError{Err: fmt.Errorf("no 'result' field in JSON-RPC response")} } return nil @@ -171,14 +211,17 @@ func decodeStreamedJSONRPCResult( // streamDecodeArray returns a decodeFn for streamRPCCall that // stream-decodes a JSON array one element at a time, appending each -// to *results. Only one decoded element is in the json.Decoder's -// internal buffer at a time, avoiding the need to buffer the entire -// raw JSON array. +// to *results, so the raw JSON of the whole array never has to be held +// at once. +// +// The decoder still buffers each element in full before it is handed over. +// Where a single element can be arbitrarily large - a call trace, for +// instance - decode it field by field instead, as decodeCallTraceResults does. func streamDecodeArray[T any](results *[]T) func(*json.Decoder) error { return func(dec *json.Decoder) error { t, err := dec.Token() if err != nil { - return fmt.Errorf("read array start: %w", err) + return asDecodeError(fmt.Errorf("read array start: %w", err)) } // Handle null result @@ -187,13 +230,13 @@ func streamDecodeArray[T any](results *[]T) func(*json.Decoder) error { } if t != json.Delim('[') { - return fmt.Errorf("expected '[', got %v", t) + return &ResponseDecodeError{Err: fmt.Errorf("expected '[', got %v", t)} } for dec.More() { var elem T if err := dec.Decode(&elem); err != nil { - return fmt.Errorf("decode element: %w", err) + return asDecodeError(fmt.Errorf("decode element: %w", err)) } *results = append(*results, elem) @@ -201,7 +244,7 @@ func streamDecodeArray[T any](results *[]T) func(*json.Decoder) error { // Read closing ']' if _, err := dec.Token(); err != nil { - return fmt.Errorf("read array end: %w", err) + return asDecodeError(fmt.Errorf("read array end: %w", err)) } return nil diff --git a/clients/execution/rpc/traces.go b/clients/execution/rpc/traces.go index 7c9b9665..2ba69025 100644 --- a/clients/execution/rpc/traces.go +++ b/clients/execution/rpc/traces.go @@ -68,6 +68,9 @@ type CallTraceResult struct { } // CallTraceCall is a single call frame in the callTracer output. +// +// Input, Output and RevertReason are pruned while the response is read, see +// decodeCallTraceResults, so they hold at most the payload limit plus one byte. type CallTraceCall struct { Type string `json:"type"` From common.Address `json:"from"` @@ -115,11 +118,16 @@ func CallTraceCallValue(c *CallTraceCall) uint256.Int { // TraceBlockByHash calls debug_traceBlockByHash with the callTracer configuration. // Returns one CallTraceResult per transaction in the block. -// Uses streaming JSON decoding to avoid buffering the entire (potentially -// hundreds of MB) response as an intermediate json.RawMessage. +// +// The response is decoded as it arrives and every call frame payload is pruned +// to payloadLimit bytes (plus the one byte that marks it as truncated) on the +// way in, so neither the raw JSON nor the decoded frames of a transaction that +// deliberately inflates its trace are ever held in full. A payloadLimit of zero +// or less keeps payloads intact. func (ec *ExecutionClient) TraceBlockByHash( ctx context.Context, blockHash common.Hash, + payloadLimit int, ) ([]CallTraceResult, error) { tracerConfig := CallTracerConfig{ Tracer: "callTracer", @@ -128,7 +136,7 @@ func (ec *ExecutionClient) TraceBlockByHash( var results []CallTraceResult err := ec.streamRPCCall(ctx, "debug_traceBlockByHash", - streamDecodeArray(&results), + decodeCallTraceResults(&results, payloadLimit), blockHash, tracerConfig, ) if err != nil { diff --git a/handlers/transaction.go b/handlers/transaction.go index 20917d87..7b6fd485 100644 --- a/handlers/transaction.go +++ b/handlers/transaction.go @@ -1251,6 +1251,9 @@ func buildInternalTxsFromBlockdb(ctx context.Context, pageData *models.Transacti valueFloat, _ := bigFloat.Float64() valueRaw := f.Value.Bytes() + input, inputPruned := bdbtypes.TrimPrunedPayload(f.Input) + output, outputPruned := bdbtypes.TrimPrunedPayload(f.Output) + itx := &models.TransactionPageDataInternalTx{ CallIndex: uint32(i), Depth: f.Depth, @@ -1263,8 +1266,10 @@ func buildInternalTxsFromBlockdb(ctx context.Context, pageData *models.Transacti GasUsed: f.GasUsed, Status: f.Status, ErrorText: f.Error, - Input: f.Input, - Output: f.Output, + Input: input, + InputPruned: inputPruned, + Output: output, + OutputPruned: outputPruned, HasTraceData: true, } @@ -1274,8 +1279,10 @@ func buildInternalTxsFromBlockdb(ctx context.Context, pageData *models.Transacti itx.TypeName = fmt.Sprintf("TYPE_%d", f.Type) } - // Method ID, name, and decoded calldata from input data - if len(f.Input) >= 4 { + // Method ID, name, and decoded calldata from input data. A pruned input + // only carries its leading bytes, and ABI decoding follows offsets that + // may point past them, so it stays on the selector-derived fields. + if len(input) >= 4 { isCreate := f.Type == 3 || f.Type == 4 precompileInfo := utils.GetPrecompileInfo(f.To[:]) sysName, isSysContract := sysContracts[f.To] @@ -1285,26 +1292,30 @@ func buildInternalTxsFromBlockdb(ctx context.Context, pageData *models.Transacti itx.MethodName = "deploy" } else if precompileInfo != nil { itx.MethodName = precompileInfo.Name - itx.DecodedCalldata = utils.DecodePrecompileInput(precompileInfo.Index, f.Input) + if !inputPruned { + itx.DecodedCalldata = utils.DecodePrecompileInput(precompileInfo.Index, input) + } } else if isNonDepositSys { itx.MethodName = sysName - switch sysName { - case "Withdrawal Request (EIP-7002)": - itx.DecodedCalldata = utils.DecodeWithdrawalRequestInput(f.Input) - case "Consolidation Request (EIP-7251)": - itx.DecodedCalldata = utils.DecodeConsolidationRequestInput(f.Input) + if !inputPruned { + switch sysName { + case "Withdrawal Request (EIP-7002)": + itx.DecodedCalldata = utils.DecodeWithdrawalRequestInput(input) + case "Consolidation Request (EIP-7251)": + itx.DecodedCalldata = utils.DecodeConsolidationRequestInput(input) + } } } else { // Normal call: use fn signature lookup - itx.MethodID = f.Input[:4] + itx.MethodID = input[:4] var sig types.TxSignatureBytes - copy(sig[:], f.Input[:4]) + copy(sig[:], input[:4]) if sigLookups != nil { if lookup, found := sigLookups[sig]; found && lookup.Status == types.TxSigStatusFound { itx.MethodName = lookup.Name itx.MethodSignature = lookup.Signature - if len(f.Input) > 4 && lookup.Signature != "" { - itx.DecodedCalldata = utils.DecodeCalldata(lookup.Signature, f.Input) + if len(input) > 4 && lookup.Signature != "" && !inputPruned { + itx.DecodedCalldata = utils.DecodeCalldata(lookup.Signature, input) } } } diff --git a/indexer/execution/txindexer/loader.go b/indexer/execution/txindexer/loader.go index 8a128838..36108d4c 100644 --- a/indexer/execution/txindexer/loader.go +++ b/indexer/execution/txindexer/loader.go @@ -3,6 +3,7 @@ package txindexer import ( "context" "encoding/json" + "errors" "fmt" "math/big" "sort" @@ -11,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" + bdbtypes "github.com/ethpandaops/dora/blockdb/types" "github.com/ethpandaops/dora/clients/execution" exerpc "github.com/ethpandaops/dora/clients/execution/rpc" "github.com/ethpandaops/dora/indexer/beacon" @@ -374,28 +376,51 @@ func (t *TxIndexer) fetchBlockTraces( clients := t.getTraceClients(primaryClient, ref) - for i, client := range clients { - results, err := client.GetRPCClient().TraceBlockByHash(ctx, blockHash) - if err != nil { - t.logger.WithError(err).WithFields(logrus.Fields{ - "blockHash": blockHash.Hex(), - "client": client.GetName(), - "attempt": i + 1, - }).Debug("failed to fetch block traces, trying next client") + var lastErr error - continue + for i, client := range clients { + results, err := client.GetRPCClient().TraceBlockByHash( + ctx, blockHash, bdbtypes.TracePayloadLimit, + ) + if err == nil { + return results, nil } - return results, nil + lastErr = err + + t.logger.WithError(err).WithFields(logrus.Fields{ + "blockHash": blockHash.Hex(), + "client": client.GetName(), + "attempt": i + 1, + }).Debug("failed to fetch block traces, trying next client") + + if !shouldRetryOnOtherClient(ctx, err) { + break + } } - t.logger.WithField("blockHash", blockHash.Hex()).Warn( - "all clients failed to fetch block traces, proceeding without traces", + t.logger.WithError(lastErr).WithField("blockHash", blockHash.Hex()).Warn( + "could not fetch block traces, proceeding without traces", ) return nil, nil } +// shouldRetryOnOtherClient reports whether a failed tracer call is worth +// repeating against another client. A cancelled context means the deadline +// shared by all attempts is already gone, and a decode failure means the +// response arrived but could not be parsed - the next client returns the same +// shape and would only burn another timeout on it. +func shouldRetryOnOtherClient(ctx context.Context, err error) bool { + if ctx.Err() != nil { + return false + } + + var decodeErr *exerpc.ResponseDecodeError + + return !errors.As(err, &decodeErr) +} + // fetchBlockStateDiffs fetches per-tx state diffs (storage changes) for a block // using debug_traceBlockByHash with prestateTracer in diffMode. // Tries the primary client first (unless Besu), then other clients in priority order. @@ -414,23 +439,29 @@ func (t *TxIndexer) fetchBlockStateDiffs( clients := t.getTraceClients(primaryClient, ref) + var lastErr error + for i, client := range clients { results, err := client.GetRPCClient().TraceBlockStateDiffsByHash(ctx, blockHash) - if err != nil { - t.logger.WithError(err).WithFields(logrus.Fields{ - "blockHash": blockHash.Hex(), - "client": client.GetName(), - "attempt": i + 1, - }).Debug("failed to fetch block state diffs, trying next client") - - continue + if err == nil { + return results, nil } - return results, nil + lastErr = err + + t.logger.WithError(err).WithFields(logrus.Fields{ + "blockHash": blockHash.Hex(), + "client": client.GetName(), + "attempt": i + 1, + }).Debug("failed to fetch block state diffs, trying next client") + + if !shouldRetryOnOtherClient(ctx, err) { + break + } } - t.logger.WithField("blockHash", blockHash.Hex()).Warn( - "all clients failed to fetch block state diffs, proceeding without state diffs", + t.logger.WithError(lastErr).WithField("blockHash", blockHash.Hex()).Warn( + "could not fetch block state diffs, proceeding without state diffs", ) return nil, nil diff --git a/templates/transaction/internaltxs.html b/templates/transaction/internaltxs.html index 24797df0..64c00fa8 100644 --- a/templates/transaction/internaltxs.html +++ b/templates/transaction/internaltxs.html @@ -179,8 +179,9 @@ {{ if gt (len $itx.Input) 0 }}
- Input ({{ len $itx.Input }} bytes): - + Input ({{ len $itx.Input }}{{ if $itx.InputPruned }}+{{ end }} bytes): + {{ if $itx.InputPruned }}pruned{{ end }} +
{{ if $itx.DecodedCalldata }} @@ -203,18 +204,19 @@
- + {{ else }} -
{{ formatHexBytes $itx.Input }}
+
{{ formatHexBytes $itx.Input }}{{ if $itx.InputPruned }} … (pruned){{ end }}
{{ end }}
{{ end }} {{ if gt (len $itx.Output) 0 }}
- Output ({{ len $itx.Output }} bytes): - -
{{ formatHexBytes $itx.Output }}
+ Output ({{ len $itx.Output }}{{ if $itx.OutputPruned }}+{{ end }} bytes): + {{ if $itx.OutputPruned }}pruned{{ end }} + +
{{ formatHexBytes $itx.Output }}{{ if $itx.OutputPruned }} … (pruned){{ end }}
{{ end }}
@@ -268,8 +270,12 @@ if (view === 'ascii' && !itxAsciiCache[idx]) { var hexEl = document.getElementById('itx-input-hex-' + idx); if (hexEl) { - itxAsciiCache[idx] = itxHexToAscii(hexEl.textContent.trim()); - target.textContent = itxAsciiCache[idx]; + // The box may carry a "pruned" marker next to the hex payload. + var hexData = hexEl.querySelector('.itx-hex-data') || hexEl; + var ascii = itxHexToAscii(hexData.textContent.trim()); + if (hexEl.dataset.pruned === '1') ascii += ' … (pruned)'; + itxAsciiCache[idx] = ascii; + target.textContent = ascii; } } target.style.display = 'block'; diff --git a/types/models/transaction.go b/types/models/transaction.go index 0f003d23..edce47d4 100644 --- a/types/models/transaction.go +++ b/types/models/transaction.go @@ -291,9 +291,13 @@ type TransactionPageDataInternalTx struct { Status uint8 `json:"status"` // 0=success, 1=reverted, 2=error ErrorText string `json:"error_text"` - // Input/Output (from blockdb call trace, empty for DB-only fallback) + // Input/Output (from blockdb call trace, empty for DB-only fallback). + // The *Pruned flags mark payloads that were truncated when the trace was + // captured, so the bytes here are only the retained prefix. Input []byte `json:"input"` + InputPruned bool `json:"input_pruned"` Output []byte `json:"output"` + OutputPruned bool `json:"output_pruned"` MethodID []byte `json:"method_id"` MethodName string `json:"method_name"` MethodSignature string `json:"method_signature"`