From f0ca0c0e37770b1044de2531a39dcf7ae9952c24 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Wed, 25 Feb 2026 15:14:31 -0700 Subject: [PATCH 1/9] Go SDK V2 implementation --- go/client.go | 63 ++++++++--- go/client_test.go | 69 +++++++++--- go/endpoints.go | 2 +- go/example_test.go | 8 +- go/feed/feed.go | 38 ++++++- go/feed/feed_test.go | 126 +++++++++++++++++++--- go/go.mod | 9 +- go/go.sum | 18 ++-- go/report/example_test.go | 6 +- go/report/report.go | 70 ++++++++---- go/report/report_test.go | 212 ++++++++++++++++++------------------- go/report/v1/data.go | 39 ++++++- go/report/v1/data_test.go | 73 ++++++++----- go/report/v10/data.go | 59 +++++++++-- go/report/v10/data_test.go | 101 ++++++++++++------ go/report/v11/data.go | 58 ++++++++-- go/report/v11/data_test.go | 112 +++++++++++++------- go/report/v12/data.go | 50 +++++++-- go/report/v12/data_test.go | 83 +++++++++------ go/report/v13/data.go | 51 +++++++-- go/report/v13/data_test.go | 87 ++++++++++----- go/report/v2/data.go | 43 ++++++-- go/report/v2/data_test.go | 59 +++++++---- go/report/v3/data.go | 47 ++++++-- go/report/v3/data_test.go | 73 ++++++++----- go/report/v4/data.go | 45 ++++++-- go/report/v4/data_test.go | 68 +++++++----- go/report/v5/data.go | 50 +++++++-- go/report/v5/data_test.go | 78 +++++++++----- go/report/v6/data.go | 51 +++++++-- go/report/v6/data_test.go | 91 ++++++++++------ go/report/v7/data.go | 43 ++++++-- go/report/v7/data_test.go | 63 +++++++---- go/report/v8/data.go | 47 ++++++-- go/report/v8/data_test.go | 75 ++++++++----- go/report/v9/data.go | 49 +++++++-- go/report/v9/data_test.go | 80 +++++++++----- go/stream.go | 21 ++-- go/stream_test.go | 75 +++++++------ 39 files changed, 1680 insertions(+), 712 deletions(-) diff --git a/go/client.go b/go/client.go index 8e93daa..359bc65 100644 --- a/go/client.go +++ b/go/client.go @@ -15,7 +15,7 @@ import ( "strings" "time" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) // Client is the data streams client interface. @@ -151,25 +151,41 @@ func (c *client) GetLatestReport(ctx context.Context, id feed.ID) (r *ReportResp // ReportResponse implements the report envelope that contains the full report payload, // its FeedID and timestamps. For decoding the Report Payload use report.Decode(). type ReportResponse struct { - FeedID feed.ID `json:"feedID"` - FullReport []byte `json:"fullReport"` - ValidFromTimestamp uint64 `json:"validFromTimestamp"` - ObservationsTimestamp uint64 `json:"observationsTimestamp"` + FeedID feed.ID + FullReport []byte + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time } func (r *ReportResponse) UnmarshalJSON(b []byte) (err error) { - type Alias ReportResponse aux := &struct { - FullReport string `json:"fullReport"` - *Alias - }{ - Alias: (*Alias)(r), - } + FeedID feed.ID `json:"feedID"` + FullReport string `json:"fullReport"` + ValidFromTimestamp uint64 `json:"validFromTimestamp"` + ObservationsTimestamp uint64 `json:"observationsTimestamp"` + ValidFromTimestampMs uint64 `json:"validFromTimestampMs"` + ObservationsTimestampMs uint64 `json:"observationsTimestampMs"` + }{} if err := json.Unmarshal(b, aux); err != nil { return err } + r.FeedID = aux.FeedID + + // V2 payloads use milliseconds, V1 payloads use seconds + if aux.ValidFromTimestampMs > 0 { + r.ValidFromTimestamp = time.UnixMilli(int64(aux.ValidFromTimestampMs)) + } else if aux.ValidFromTimestamp > 0 { + r.ValidFromTimestamp = time.Unix(int64(aux.ValidFromTimestamp), 0) + } + + if aux.ObservationsTimestampMs > 0 { + r.ObservationsTimestamp = time.UnixMilli(int64(aux.ObservationsTimestampMs)) + } else if aux.ObservationsTimestamp > 0 { + r.ObservationsTimestamp = time.Unix(int64(aux.ObservationsTimestamp), 0) + } + if len(aux.FullReport) < 3 { return nil } @@ -182,13 +198,26 @@ func (r *ReportResponse) UnmarshalJSON(b []byte) (err error) { } func (r *ReportResponse) MarshalJSON() ([]byte, error) { - type Alias ReportResponse + var validFrom, observationsTS uint64 + + // Wrapper timestamps are always in milliseconds + if !r.ValidFromTimestamp.IsZero() { + validFrom = uint64(r.ValidFromTimestamp.UnixMilli()) + } + if !r.ObservationsTimestamp.IsZero() { + observationsTS = uint64(r.ObservationsTimestamp.UnixMilli()) + } + return json.Marshal(&struct { - FullReport string `json:"fullReport"` - *Alias + FeedID feed.ID `json:"feedID"` + FullReport string `json:"fullReport"` + ValidFromTimestampMs uint64 `json:"validFromTimestampMs"` + ObservationsTimestampMs uint64 `json:"observationsTimestampMs"` }{ - FullReport: "0x" + hex.EncodeToString(r.FullReport), - Alias: (*Alias)(r), + FeedID: r.FeedID, + FullReport: "0x" + hex.EncodeToString(r.FullReport), + ValidFromTimestampMs: validFrom, + ObservationsTimestampMs: observationsTS, }) } @@ -243,7 +272,7 @@ func (c *client) GetReportPage(ctx context.Context, id feed.ID, pageTS uint64) ( } r.NextPageTS = 0 if len(r.Reports) > 0 { - r.NextPageTS = r.Reports[len(r.Reports)-1].ObservationsTimestamp + 1 + r.NextPageTS = uint64(r.Reports[len(r.Reports)-1].ObservationsTimestamp.Unix()) + 1 } return r, err } diff --git a/go/client_test.go b/go/client_test.go index d5253c1..1f5e41b 100644 --- a/go/client_test.go +++ b/go/client_test.go @@ -1,6 +1,7 @@ package streams import ( + "bytes" "context" "encoding/json" "fmt" @@ -9,11 +10,53 @@ import ( "reflect" "strconv" "testing" + "time" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) +// reportResponseEqual compares two ReportResponse structs, handling time.Time comparison properly +func reportResponseEqual(a, b *ReportResponse) bool { + if a.FeedID != b.FeedID { + return false + } + if !bytes.Equal(a.FullReport, b.FullReport) { + return false + } + if !a.ObservationsTimestamp.Equal(b.ObservationsTimestamp) { + return false + } + if !a.ValidFromTimestamp.Equal(b.ValidFromTimestamp) { + return false + } + return true +} + +// reportResponsesEqual compares slices of ReportResponse +func reportResponsesEqual(a, b []*ReportResponse) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if !reportResponseEqual(a[i], b[i]) { + return false + } + } + return true +} + +// reportPageEqual compares two ReportPage structs +func reportPageEqual(a, b *ReportPage) bool { + if !reportResponsesEqual(a.Reports, b.Reports) { + return false + } + if a.NextPageTS != b.NextPageTS { + return false + } + return true +} + func mustFeedIDfromString(s string) (f feed.ID) { err := f.FromString(s) if err != nil { @@ -67,8 +110,8 @@ func TestClient_GetFeeds(t *testing.T) { func TestClient_GetReports(t *testing.T) { expectedReports := []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 12344}, - {FeedID: feed2, ObservationsTimestamp: 12344}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12344, 0)}, } expectedFeedIdListStr := fmt.Sprintf("%s,%s", feed1.String(), feed2.String()) @@ -110,7 +153,7 @@ func TestClient_GetReports(t *testing.T) { fmt.Println(expectedReports[0], reports[0]) - if !reflect.DeepEqual(reports, expectedReports) { + if !reportResponsesEqual(reports, expectedReports) { t.Errorf("GetFeeds() = %v, want %v", reports, expectedReports) } } @@ -154,7 +197,7 @@ func TestClient_GetLatestReport(t *testing.T) { t.Fatalf("GetLatestReport() error = %v", err) } - if !reflect.DeepEqual(report, expectedReport) { + if !reportResponseEqual(report, expectedReport) { t.Errorf("GetLatestReport() = %v, want %v", report, expectedReport) } } @@ -164,18 +207,18 @@ func TestClient_GetReportPage(t *testing.T) { expectedReportPage1 := &ReportPage{ Reports: []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 1234567890, FullReport: hexutil.Bytes(`report1 payload`)}, - {FeedID: feed1, ObservationsTimestamp: 1234567891, FullReport: hexutil.Bytes(`report2 payload`)}, + {FeedID: feed1, FullReport: hexutil.Bytes(`report1 payload`), ObservationsTimestamp: time.Unix(1234567897, 0)}, + {FeedID: feed1, FullReport: hexutil.Bytes(`report2 payload`), ObservationsTimestamp: time.Unix(1234567898, 0)}, }, - NextPageTS: 1234567892, + NextPageTS: 1234567899, // Last ObservationsTimestamp (1234567898) + 1 } expectedReportPage2 := &ReportPage{ Reports: []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 1234567892, FullReport: hexutil.Bytes(`report3 payload`)}, - {FeedID: feed1, ObservationsTimestamp: 1234567893, FullReport: hexutil.Bytes(`report4 payload`)}, + {FeedID: feed1, FullReport: hexutil.Bytes(`report3 payload`), ObservationsTimestamp: time.Unix(1234567997, 0)}, + {FeedID: feed1, FullReport: hexutil.Bytes(`report4 payload`), ObservationsTimestamp: time.Unix(1234567998, 0)}, }, - NextPageTS: 1234567894, + NextPageTS: 1234567999, // Last ObservationsTimestamp (1234567998) + 1 } ms := newMockServer(func(w http.ResponseWriter, r *http.Request) { @@ -229,7 +272,7 @@ func TestClient_GetReportPage(t *testing.T) { t.Fatalf("GetReportPage() error = %v", err) } - if !reflect.DeepEqual(reportPage, expectedReportPage1) { + if !reportPageEqual(reportPage, expectedReportPage1) { t.Errorf("GetReportPage() = %v, want %v", reportPage, expectedReportPage1) } @@ -238,7 +281,7 @@ func TestClient_GetReportPage(t *testing.T) { t.Fatalf("GetReportPage() error = %v", err) } - if !reflect.DeepEqual(reportPage, expectedReportPage2) { + if !reportPageEqual(reportPage, expectedReportPage2) { t.Errorf("GetReportPage() = %v, want %v", reportPage, expectedReportPage2) } } diff --git a/go/endpoints.go b/go/endpoints.go index b430de7..7093d31 100644 --- a/go/endpoints.go +++ b/go/endpoints.go @@ -3,7 +3,7 @@ package streams import "net/textproto" const ( - apiV1WS = "/api/v1/ws" + apiV2WS = "/api/v2/ws" apiV1Feeds = "/api/v1/feeds" apiV1Reports = "/api/v1/reports" apiV1ReportsBulk = "/api/v1/reports/bulk" diff --git a/go/example_test.go b/go/example_test.go index b6ab29d..0192892 100644 --- a/go/example_test.go +++ b/go/example_test.go @@ -5,10 +5,10 @@ import ( "os" "time" - streams "github.com/smartcontractkit/data-streams-sdk/go" - "github.com/smartcontractkit/data-streams-sdk/go/feed" - streamsReport "github.com/smartcontractkit/data-streams-sdk/go/report" - v3 "github.com/smartcontractkit/data-streams-sdk/go/report/v3" + streams "github.com/smartcontractkit/data-streams-sdk/go/v2" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" + streamsReport "github.com/smartcontractkit/data-streams-sdk/go/v2/report" + v3 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v3" ) func ExampleClient() { diff --git a/go/feed/feed.go b/go/feed/feed.go index a47624d..44718d9 100644 --- a/go/feed/feed.go +++ b/go/feed/feed.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "time" ) // FeedVersion represents the feed report schema version @@ -27,6 +28,27 @@ const ( _ ) +// Resolution represents the timestamp resolution for a feed +type Resolution uint8 + +const ( + // ResolutionSeconds indicates timestamps are in seconds + ResolutionSeconds Resolution = 0 + // ResolutionMilliseconds indicates timestamps are in milliseconds + ResolutionMilliseconds Resolution = 1 +) + +func (r Resolution) String() string { + switch r { + case ResolutionSeconds: + return "seconds" + case ResolutionMilliseconds: + return "milliseconds" + default: + return "undefined" + } +} + // ID type type ID [32]byte @@ -76,6 +98,20 @@ type Feed struct { FeedID ID `json:"feedID"` } +// Version returns the feed schema version (masked to ignore resolution nibble) func (f *ID) Version() FeedVersion { - return FeedVersion(binary.BigEndian.Uint16(f[:2])) + return FeedVersion(binary.BigEndian.Uint16(f[:2]) & 0x0FFF) +} + +// Resolution returns the timestamp resolution for this feed +func (f *ID) Resolution() Resolution { + return Resolution(f[0] >> 4) +} + +// ParseTimestamp converts a raw uint64 timestamp to time.Time based on resolution. +func ParseTimestamp(ts uint64, res Resolution) time.Time { + if res == ResolutionMilliseconds { + return time.UnixMilli(int64(ts)) + } + return time.Unix(int64(ts), 0) } diff --git a/go/feed/feed_test.go b/go/feed/feed_test.go index d8c7ec2..01ee034 100644 --- a/go/feed/feed_test.go +++ b/go/feed/feed_test.go @@ -6,27 +6,40 @@ import ( ) var ( + // Seconds Resolution FeedIDs v1FeedID = (ID)([32]uint8{00, 01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) v2FeedID = (ID)([32]uint8{00, 02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) v3FeedID = (ID)([32]uint8{00, 03, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) v4FeedID = (ID)([32]uint8{00, 04, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) + // Milliseconds Resolution FeedIDs (first nibble = 1) + v1FeedIDMillis = (ID)([32]uint8{0x10, 0x01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) + v2FeedIDMillis = (ID)([32]uint8{0x10, 0x02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) + v3FeedIDMillis = (ID)([32]uint8{0x10, 0x03, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) + v4FeedIDMillis = (ID)([32]uint8{0x10, 0x04, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}) ) func TestFeedVersion(t *testing.T) { - if v1FeedID.Version() != FeedVersion1 { - t.Fatalf("expected feed version: %d, got: %d", FeedVersion1, v1FeedID.Version()) - } - - if v2FeedID.Version() != FeedVersion2 { - t.Fatalf("expected feed version: %d, got: %d", FeedVersion2, v2FeedID.Version()) - } - - if v3FeedID.Version() != FeedVersion3 { - t.Fatalf("expected feed version: %d, got: %d", FeedVersion3, v3FeedID.Version()) + tests := []struct { + name string + feed ID + want FeedVersion + }{ + {"v1", v1FeedID, FeedVersion1}, + {"v2", v2FeedID, FeedVersion2}, + {"v3", v3FeedID, FeedVersion3}, + {"v4", v4FeedID, FeedVersion4}, + {"v1_milliseconds", v1FeedIDMillis, FeedVersion1}, + {"v2_milliseconds", v2FeedIDMillis, FeedVersion2}, + {"v3_milliseconds", v3FeedIDMillis, FeedVersion3}, + {"v4_milliseconds", v4FeedIDMillis, FeedVersion4}, } - if v4FeedID.Version() != FeedVersion4 { - t.Fatalf("expected feed version: %d, got: %d", FeedVersion4, v4FeedID.Version()) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.feed.Version() != tt.want { + t.Fatalf("expected feed version: %d, got: %d", tt.want, tt.feed.Version()) + } + }) } } @@ -59,6 +72,27 @@ func TestFeedMarshalJSON(t *testing.T) { feed: v4FeedID, want: `"0x00046b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`, }, + // milliseconds resolution feedIDs + { + name: "v1_milliseconds", + feed: v1FeedIDMillis, + want: `"0x10016b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`, + }, + { + name: "v2_milliseconds", + feed: v2FeedIDMillis, + want: `"0x10026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`, + }, + { + name: "v3_milliseconds", + feed: v3FeedIDMillis, + want: `"0x10036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`, + }, + { + name: "v4_milliseconds", + feed: v4FeedIDMillis, + want: `"0x10046b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`, + }, } for _, tt := range tests { @@ -103,6 +137,27 @@ func TestFeedUnMarshalJSON(t *testing.T) { feed: v4FeedID, want: []byte(`"0x00046b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`), }, + // milliseconds resolution feedIDs + { + name: "v1_milliseconds", + feed: v1FeedIDMillis, + want: []byte(`"0x10016b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`), + }, + { + name: "v2_milliseconds", + feed: v2FeedIDMillis, + want: []byte(`"0x10026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`), + }, + { + name: "v3_milliseconds", + feed: v3FeedIDMillis, + want: []byte(`"0x10036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`), + }, + { + name: "v4_milliseconds", + feed: v4FeedIDMillis, + want: []byte(`"0x10046b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472"`), + }, } for _, tt := range tests { @@ -119,3 +174,50 @@ func TestFeedUnMarshalJSON(t *testing.T) { }) } } + +func TestFeedResolution(t *testing.T) { + // Test Seconds Resolution FeedIDs + secondsFeeds := []struct { + name string + feed ID + resolution Resolution + }{ + {"v1", v1FeedID, ResolutionSeconds}, + {"v2", v2FeedID, ResolutionSeconds}, + {"v3", v3FeedID, ResolutionSeconds}, + {"v4", v4FeedID, ResolutionSeconds}, + {"v1_milliseconds", v1FeedIDMillis, ResolutionMilliseconds}, + {"v2_milliseconds", v2FeedIDMillis, ResolutionMilliseconds}, + {"v3_milliseconds", v3FeedIDMillis, ResolutionMilliseconds}, + {"v4_milliseconds", v4FeedIDMillis, ResolutionMilliseconds}, + } + + for _, tt := range secondsFeeds { + t.Run(tt.name, func(t *testing.T) { + if tt.feed.Resolution() != tt.resolution { + t.Fatalf("expected feed resolution: %v, got: %v", tt.resolution, tt.feed.Resolution()) + } + }) + } +} + +func TestParseTimestamp(t *testing.T) { + tests := []struct { + name string + ts uint64 + resolution Resolution + wantUnix int64 + }{ + {"seconds", 1700000000, ResolutionSeconds, 1700000000}, + {"milliseconds", 1700000000000, ResolutionMilliseconds, 1700000000}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseTimestamp(tt.ts, tt.resolution) + if got.Unix() != tt.wantUnix { + t.Fatalf("ParseTimestamp() = %v, want Unix %v", got.Unix(), tt.wantUnix) + } + }) + } +} diff --git a/go/go.mod b/go/go.mod index 84ce58d..f820158 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,16 +1,15 @@ -module github.com/smartcontractkit/data-streams-sdk/go +module github.com/smartcontractkit/data-streams-sdk/go/v2 go 1.24.0 require ( - github.com/ethereum/go-ethereum v1.16.7 - nhooyr.io/websocket v1.8.11 + github.com/ethereum/go-ethereum v1.17.0 + nhooyr.io/websocket v1.8.17 ) require ( github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/holiman/uint256 v1.3.2 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/sys v0.36.0 // indirect + golang.org/x/sys v0.39.0 // indirect ) diff --git a/go/go.sum b/go/go.sum index 3faa390..e4dd149 100644 --- a/go/go.sum +++ b/go/go.sum @@ -6,21 +6,19 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/ethereum/go-ethereum v1.16.7 h1:qeM4TvbrWK0UC0tgkZ7NiRsmBGwsjqc64BHo20U59UQ= -github.com/ethereum/go-ethereum v1.16.7/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= +github.com/ethereum/go-ethereum v1.17.0 h1:2D+1Fe23CwZ5tQoAS5DfwKFNI1HGcTwi65/kRlAVxes= +github.com/ethereum/go-ethereum v1.17.0/go.mod h1:2W3msvdosS/MCWytpqTcqgFiRYbTH59FxDJzqah120o= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= -nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= +nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= +nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/go/report/example_test.go b/go/report/example_test.go index 8ab9132..1364c8b 100644 --- a/go/report/example_test.go +++ b/go/report/example_test.go @@ -4,9 +4,9 @@ import ( "encoding/hex" "os" - streams "github.com/smartcontractkit/data-streams-sdk/go" - "github.com/smartcontractkit/data-streams-sdk/go/report" - v3 "github.com/smartcontractkit/data-streams-sdk/go/report/v3" + streams "github.com/smartcontractkit/data-streams-sdk/go/v2" + "github.com/smartcontractkit/data-streams-sdk/go/v2/report" + v3 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v3" ) func ExampleDecode() { diff --git a/go/report/report.go b/go/report/report.go index 654e937..6b0f428 100644 --- a/go/report/report.go +++ b/go/report/report.go @@ -5,19 +5,19 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" - v1 "github.com/smartcontractkit/data-streams-sdk/go/report/v1" - v10 "github.com/smartcontractkit/data-streams-sdk/go/report/v10" - v11 "github.com/smartcontractkit/data-streams-sdk/go/report/v11" - v12 "github.com/smartcontractkit/data-streams-sdk/go/report/v12" - v13 "github.com/smartcontractkit/data-streams-sdk/go/report/v13" - v2 "github.com/smartcontractkit/data-streams-sdk/go/report/v2" - v3 "github.com/smartcontractkit/data-streams-sdk/go/report/v3" - v4 "github.com/smartcontractkit/data-streams-sdk/go/report/v4" - v5 "github.com/smartcontractkit/data-streams-sdk/go/report/v5" - v6 "github.com/smartcontractkit/data-streams-sdk/go/report/v6" - v7 "github.com/smartcontractkit/data-streams-sdk/go/report/v7" - v8 "github.com/smartcontractkit/data-streams-sdk/go/report/v8" - v9 "github.com/smartcontractkit/data-streams-sdk/go/report/v9" + v1 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v1" + v10 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v10" + v11 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v11" + v12 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v12" + v13 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v13" + v2 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v2" + v3 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v3" + v4 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v4" + v5 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v5" + v6 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v6" + v7 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v7" + v8 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v8" + v9 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v9" ) // Data represents the actual report data and attributes @@ -48,15 +48,47 @@ func Decode[T Data](fullReport []byte) (r *Report[T], err error) { return nil, fmt.Errorf("report: failed to copy: %s", err) } - dataSchema := r.Data.Schema() - dataValues, err := dataSchema.Unpack(r.ReportBlob) - if err != nil { - return nil, fmt.Errorf("report: failed to unpack data: %s", err) + var data any + switch any(r.Data).(type) { + case v1.Data: + data, err = v1.Decode(r.ReportBlob) + case v2.Data: + data, err = v2.Decode(r.ReportBlob) + case v3.Data: + data, err = v3.Decode(r.ReportBlob) + case v4.Data: + data, err = v4.Decode(r.ReportBlob) + case v5.Data: + data, err = v5.Decode(r.ReportBlob) + case v6.Data: + data, err = v6.Decode(r.ReportBlob) + case v7.Data: + data, err = v7.Decode(r.ReportBlob) + case v8.Data: + data, err = v8.Decode(r.ReportBlob) + case v9.Data: + data, err = v9.Decode(r.ReportBlob) + case v10.Data: + data, err = v10.Decode(r.ReportBlob) + case v11.Data: + data, err = v11.Decode(r.ReportBlob) + case v12.Data: + data, err = v12.Decode(r.ReportBlob) + case v13.Data: + data, err = v13.Decode(r.ReportBlob) + default: + return nil, fmt.Errorf("report: unsupported data type") } - err = dataSchema.Copy(&r.Data, dataValues) if err != nil { - return nil, fmt.Errorf("report: failed to copy data: %s", err) + return nil, fmt.Errorf("report: failed to decode data: %s", err) + } + + // Required to return typed data as T + if d, ok := data.(*T); ok { + r.Data = *d + } else { + return nil, fmt.Errorf("report: could not cast data to T") } return r, nil diff --git a/go/report/report_test.go b/go/report/report_test.go index 9d6099c..ca2b5fc 100644 --- a/go/report/report_test.go +++ b/go/report/report_test.go @@ -9,20 +9,20 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/report/common" - v1 "github.com/smartcontractkit/data-streams-sdk/go/report/v1" - v10 "github.com/smartcontractkit/data-streams-sdk/go/report/v10" - v11 "github.com/smartcontractkit/data-streams-sdk/go/report/v11" - v12 "github.com/smartcontractkit/data-streams-sdk/go/report/v12" - v13 "github.com/smartcontractkit/data-streams-sdk/go/report/v13" - v2 "github.com/smartcontractkit/data-streams-sdk/go/report/v2" - v3 "github.com/smartcontractkit/data-streams-sdk/go/report/v3" - v4 "github.com/smartcontractkit/data-streams-sdk/go/report/v4" - v5 "github.com/smartcontractkit/data-streams-sdk/go/report/v5" - v6 "github.com/smartcontractkit/data-streams-sdk/go/report/v6" - v7 "github.com/smartcontractkit/data-streams-sdk/go/report/v7" - v8 "github.com/smartcontractkit/data-streams-sdk/go/report/v8" - v9 "github.com/smartcontractkit/data-streams-sdk/go/report/v9" + "github.com/smartcontractkit/data-streams-sdk/go/v2/report/common" + v1 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v1" + v10 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v10" + v11 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v11" + v12 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v12" + v13 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v13" + v2 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v2" + v3 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v3" + v4 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v4" + v5 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v5" + v6 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v6" + v7 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v7" + v8 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v8" + v9 "github.com/smartcontractkit/data-streams-sdk/go/v2/report/v9" ) func TestReport(t *testing.T) { @@ -328,33 +328,33 @@ var v13Report = &Report[v13.Data]{ var v1Data = v1.Data{ FeedID: [32]uint8{00, 01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ObservationsTimestamp: uint32(time.Now().Unix()), + ObservationsTimestamp: time.Unix(1700000000, 0), BenchmarkPrice: big.NewInt(100), Bid: big.NewInt(100), Ask: big.NewInt(100), CurrentBlockNum: 100, CurrentBlockHash: [32]uint8{0, 0, 7, 4, 7, 2, 4, 1, 82, 38, 2, 9, 6, 5, 6, 8, 2, 8, 5, 5, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 1}, ValidFromBlockNum: 768986, - CurrentBlockTimestamp: uint64(time.Now().Unix()), + CurrentBlockTimestamp: time.Unix(1700000000, 0), } var v2Data = v2.Data{ FeedID: [32]uint8{00, 02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ObservationsTimestamp: uint32(time.Now().Unix()), + ObservationsTimestamp: time.Unix(1700000000, 0), BenchmarkPrice: big.NewInt(100), - ValidFromTimestamp: uint32(time.Now().Unix()), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ValidFromTimestamp: time.Unix(1700000000, 0), + ExpiresAt: time.Unix(1700000100, 0), LinkFee: big.NewInt(10), NativeFee: big.NewInt(10), } var v3Data = v3.Data{ FeedID: [32]uint8{00, 03, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), BenchmarkPrice: big.NewInt(100), Bid: big.NewInt(100), Ask: big.NewInt(100), @@ -362,34 +362,34 @@ var v3Data = v3.Data{ var v4Data = v4.Data{ FeedID: [32]uint8{00, 04, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), BenchmarkPrice: big.NewInt(100), MarketStatus: common.MarketStatusOpen, } var v5Data = v5.Data{ FeedID: [32]uint8{00, 5, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), Rate: big.NewInt(550), - Timestamp: uint32(time.Now().Unix()), - Duration: uint32(86400), + Timestamp: time.Unix(1700000000, 0), + Duration: 86400 * time.Second, } var v6Data = v6.Data{ FeedID: [32]uint8{00, 6, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), Price: big.NewInt(600), Price2: big.NewInt(601), Price3: big.NewInt(602), @@ -399,64 +399,64 @@ var v6Data = v6.Data{ var v7Data = v7.Data{ FeedID: [32]uint8{00, 7, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), ExchangeRate: big.NewInt(700), } var v8Data = v8.Data{ FeedID: [32]uint8{00, 8, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - LastUpdateTimestamp: uint64(time.Now().UnixNano() - int64(10*time.Second)), + ExpiresAt: time.Unix(1700000100, 0), + LastUpdateTimestamp: time.Unix(0, 1700000000000000000), MidPrice: big.NewInt(100), MarketStatus: 1, } var v9Data = v9.Data{ FeedID: [32]uint8{00, 9, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), NavPerShare: big.NewInt(1100), - NavDate: uint64(time.Now().UnixNano()) - 100, + NavDate: time.Unix(0, 1700000000000000000), Aum: big.NewInt(11009), Ripcord: 108, } var v10Data = v10.Data{ FeedID: [32]uint8{00, 10, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - LastUpdateTimestamp: uint64(time.Now().UnixNano() - int64(10*time.Second)), + ExpiresAt: time.Unix(1700000100, 0), + LastUpdateTimestamp: time.Unix(0, 1700000000000000000), Price: big.NewInt(1000), MarketStatus: 1, CurrentMultiplier: big.NewInt(100), NewMultiplier: big.NewInt(101), - ActivationDateTime: uint32(time.Now().Unix()) + 200, + ActivationDateTime: time.Unix(1700000200, 0), TokenizedPrice: big.NewInt(1001), } var v11Data = v11.Data{ FeedID: [32]uint8{00, 11, 251, 109, 19, 88, 151, 228, 170, 245, 101, 123, 255, 211, 176, 180, 143, 142, 42, 81, 49, 33, 76, 158, 194, 214, 46, 172, 93, 83, 32, 103}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), Mid: big.NewInt(103), - LastSeenTimestampNs: uint64(time.Now().Unix()), + LastSeenTimestampNs: time.Unix(0, 1700000000000000000), Bid: big.NewInt(101), BidVolume: big.NewInt(10002), Ask: big.NewInt(105), @@ -467,24 +467,24 @@ var v11Data = v11.Data{ var v12Data = v12.Data{ FeedID: [32]uint8{00, 12, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), NavPerShare: big.NewInt(1100), NextNavPerShare: big.NewInt(1101), - NavDate: uint64(time.Now().UnixNano()) - 100, + NavDate: time.Unix(0, 1700000000000000000), Ripcord: 108, } var v13Data = v13.Data{ FeedID: [32]uint8{00, 13, 19, 169, 185, 197, 227, 122, 9, 159, 55, 78, 146, 195, 121, 20, 175, 92, 38, 143, 58, 138, 151, 33, 241, 114, 81, 53, 191, 180, 203, 184}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), + ValidFromTimestamp: time.Unix(1700000000, 0), + ObservationsTimestamp: time.Unix(1700000000, 0), NativeFee: big.NewInt(10), LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, + ExpiresAt: time.Unix(1700000100, 0), BestAsk: big.NewInt(75), BestBid: big.NewInt(78), AskVolume: 10000, @@ -501,35 +501,35 @@ func mustPackData(d interface{}) []byte { dataSchema = v1.Schema() args = []interface{}{ v.FeedID, - v.ObservationsTimestamp, + uint64(v.ObservationsTimestamp.Unix()), v.BenchmarkPrice, v.Bid, v.Ask, v.CurrentBlockNum, v.CurrentBlockHash, v.ValidFromBlockNum, - v.CurrentBlockTimestamp, + uint64(v.CurrentBlockTimestamp.Unix()), } case v2.Data: dataSchema = v2.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.BenchmarkPrice, } case v3.Data: dataSchema = v3.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.BenchmarkPrice, v.Bid, v.Ask, @@ -538,11 +538,11 @@ func mustPackData(d interface{}) []byte { dataSchema = v4.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.BenchmarkPrice, v.MarketStatus, } @@ -550,24 +550,24 @@ func mustPackData(d interface{}) []byte { dataSchema = v5.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.Rate, - v.Timestamp, - v.Duration, + uint64(v.Timestamp.Unix()), + uint32(v.Duration.Seconds()), } case v6.Data: dataSchema = v6.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.Price, v.Price2, v.Price3, @@ -578,23 +578,23 @@ func mustPackData(d interface{}) []byte { dataSchema = v7.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.ExchangeRate, } case v8.Data: dataSchema = v8.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, - v.LastUpdateTimestamp, + uint64(v.ExpiresAt.Unix()), + uint64(v.LastUpdateTimestamp.UnixNano()), v.MidPrice, v.MarketStatus, } @@ -602,13 +602,13 @@ func mustPackData(d interface{}) []byte { dataSchema = v9.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.NavPerShare, - v.NavDate, + uint64(v.NavDate.UnixNano()), v.Aum, v.Ripcord, } @@ -616,30 +616,30 @@ func mustPackData(d interface{}) []byte { dataSchema = v10.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, - v.LastUpdateTimestamp, + uint64(v.ExpiresAt.Unix()), + uint64(v.LastUpdateTimestamp.UnixNano()), v.Price, v.MarketStatus, v.CurrentMultiplier, v.NewMultiplier, - v.ActivationDateTime, + uint64(v.ActivationDateTime.Unix()), v.TokenizedPrice, } case v11.Data: dataSchema = v11.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.Mid, - v.LastSeenTimestampNs, + uint64(v.LastSeenTimestampNs.UnixNano()), v.Bid, v.BidVolume, v.Ask, @@ -651,25 +651,25 @@ func mustPackData(d interface{}) []byte { dataSchema = v12.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.NavPerShare, v.NextNavPerShare, - v.NavDate, + uint64(v.NavDate.UnixNano()), v.Ripcord, } case v13.Data: dataSchema = v13.Schema() args = []interface{}{ v.FeedID, - v.ValidFromTimestamp, - v.ObservationsTimestamp, + uint64(v.ValidFromTimestamp.Unix()), + uint64(v.ObservationsTimestamp.Unix()), v.NativeFee, v.LinkFee, - v.ExpiresAt, + uint64(v.ExpiresAt.Unix()), v.BestAsk, v.BestBid, v.AskVolume, diff --git a/go/report/v1/data.go b/go/report/v1/data.go index db0b488..f6aa2d6 100644 --- a/go/report/v1/data.go +++ b/go/report/v1/data.go @@ -3,9 +3,10 @@ package v1 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,7 +22,7 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "benchmarkPrice", Type: mustNewType("int192")}, {Name: "bid", Type: mustNewType("int192")}, {Name: "ask", Type: mustNewType("int192")}, @@ -35,7 +36,20 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 + ObservationsTimestamp time.Time + BenchmarkPrice *big.Int + Bid *big.Int + Ask *big.Int + CurrentBlockNum uint64 + CurrentBlockHash [32]byte + ValidFromBlockNum uint64 + CurrentBlockTimestamp time.Time +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 BenchmarkPrice *big.Int Bid *big.Int Ask *big.Int @@ -56,9 +70,24 @@ func Decode(report []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + BenchmarkPrice: raw.BenchmarkPrice, + Bid: raw.Bid, + Ask: raw.Ask, + CurrentBlockNum: raw.CurrentBlockNum, + CurrentBlockHash: raw.CurrentBlockHash, + ValidFromBlockNum: raw.ValidFromBlockNum, + CurrentBlockTimestamp: feed.ParseTimestamp(raw.CurrentBlockTimestamp, res), + } + return decoded, nil } diff --git a/go/report/v1/data_test.go b/go/report/v1/data_test.go index a5f745f..72f7bb1 100644 --- a/go/report/v1/data_test.go +++ b/go/report/v1/data_test.go @@ -2,46 +2,69 @@ package v1 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ObservationsTimestamp: uint32(time.Now().Unix()), - BenchmarkPrice: big.NewInt(100), - Bid: big.NewInt(100), - Ask: big.NewInt(100), - CurrentBlockNum: 100, - CurrentBlockHash: [32]uint8{0, 0, 7, 4, 7, 2, 4, 1, 82, 38, 2, 9, 6, 5, 6, 8, 2, 8, 5, 5, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 1}, - ValidFromBlockNum: 768986, - CurrentBlockTimestamp: uint64(time.Now().Unix()), - } + // Raw values for packing + feedID := [32]uint8{00, 01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + observationsTS := uint64(time.Now().Unix()) + benchmarkPrice := big.NewInt(100) + bid := big.NewInt(100) + ask := big.NewInt(100) + currentBlockNum := uint64(100) + currentBlockHash := [32]uint8{0, 0, 7, 4, 7, 2, 4, 1, 82, 38, 2, 9, 6, 5, 6, 8, 2, 8, 5, 5, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 1} + validFromBlockNum := uint64(768986) + currentBlockTS := uint64(time.Now().Unix()) b, err := schema.Pack( - r.FeedID, - r.ObservationsTimestamp, - r.BenchmarkPrice, - r.Bid, - r.Ask, - r.CurrentBlockNum, - r.CurrentBlockHash, - r.ValidFromBlockNum, - r.CurrentBlockTimestamp, + feedID, + observationsTS, + benchmarkPrice, + bid, + ask, + currentBlockNum, + currentBlockHash, + validFromBlockNum, + currentBlockTS, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.BenchmarkPrice.Cmp(benchmarkPrice) != 0 { + t.Errorf("BenchmarkPrice mismatch: expected %v, got %v", benchmarkPrice, d.BenchmarkPrice) + } + if d.Bid.Cmp(bid) != 0 { + t.Errorf("Bid mismatch: expected %v, got %v", bid, d.Bid) + } + if d.Ask.Cmp(ask) != 0 { + t.Errorf("Ask mismatch: expected %v, got %v", ask, d.Ask) + } + if d.CurrentBlockNum != currentBlockNum { + t.Errorf("CurrentBlockNum mismatch: expected %d, got %d", currentBlockNum, d.CurrentBlockNum) + } + if d.CurrentBlockHash != currentBlockHash { + t.Errorf("CurrentBlockHash mismatch: expected %v, got %v", currentBlockHash, d.CurrentBlockHash) + } + if d.ValidFromBlockNum != validFromBlockNum { + t.Errorf("ValidFromBlockNum mismatch: expected %d, got %d", validFromBlockNum, d.ValidFromBlockNum) + } + if d.CurrentBlockTimestamp.Unix() != int64(currentBlockTS) { + t.Errorf("CurrentBlockTimestamp mismatch: expected %d, got %d", currentBlockTS, d.CurrentBlockTimestamp.Unix()) } } diff --git a/go/report/v10/data.go b/go/report/v10/data.go index 1884ea0..4c4cc1e 100644 --- a/go/report/v10/data.go +++ b/go/report/v10/data.go @@ -3,9 +3,10 @@ package v10 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,17 +22,17 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "lastUpdateTimestamp", Type: mustNewType("uint64")}, {Name: "price", Type: mustNewType("int192")}, {Name: "marketStatus", Type: mustNewType("uint32")}, {Name: "currentMultiplier", Type: mustNewType("int192")}, {Name: "newMultiplier", Type: mustNewType("int192")}, - {Name: "activationDateTime", Type: mustNewType("uint32")}, + {Name: "activationDateTime", Type: mustNewType("uint64")}, {Name: "tokenizedPrice", Type: mustNewType("int192")}, }) } @@ -39,9 +40,26 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 - ValidFromTimestamp uint32 - ExpiresAt uint32 + ObservationsTimestamp time.Time + ValidFromTimestamp time.Time + ExpiresAt time.Time + LinkFee *big.Int + NativeFee *big.Int + LastUpdateTimestamp time.Time // nanoseconds precision + Price *big.Int + MarketStatus uint32 + CurrentMultiplier *big.Int + NewMultiplier *big.Int + ActivationDateTime time.Time // Always seconds + TokenizedPrice *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 + ValidFromTimestamp uint64 + ExpiresAt uint64 LinkFee *big.Int NativeFee *big.Int LastUpdateTimestamp uint64 @@ -49,7 +67,7 @@ type Data struct { MarketStatus uint32 CurrentMultiplier *big.Int NewMultiplier *big.Int - ActivationDateTime uint32 + ActivationDateTime uint64 TokenizedPrice *big.Int } @@ -64,9 +82,28 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + LastUpdateTimestamp: time.Unix(0, int64(raw.LastUpdateTimestamp)), // Always nanoseconds + Price: raw.Price, + MarketStatus: raw.MarketStatus, + CurrentMultiplier: raw.CurrentMultiplier, + NewMultiplier: raw.NewMultiplier, + ActivationDateTime: time.Unix(int64(raw.ActivationDateTime), 0), // Always seconds + TokenizedPrice: raw.TokenizedPrice, + } + return decoded, nil } diff --git a/go/report/v10/data_test.go b/go/report/v10/data_test.go index 9ad4182..905d6d9 100644 --- a/go/report/v10/data_test.go +++ b/go/report/v10/data_test.go @@ -2,54 +2,89 @@ package v10 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 10, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - LastUpdateTimestamp: uint64(time.Now().UnixNano()) - 100, - Price: big.NewInt(1100), - MarketStatus: 1, - CurrentMultiplier: big.NewInt(1000), - NewMultiplier: big.NewInt(1050), - ActivationDateTime: uint32(time.Now().Unix()) + 200, - TokenizedPrice: big.NewInt(11009), - } + // Raw values for packing + feedID := [32]uint8{00, 10, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + lastUpdateTS := uint64(time.Now().UnixNano()) - 100 + price := big.NewInt(100) + marketStatus := uint32(1) + currentMultiplier := big.NewInt(1) + newMultiplier := big.NewInt(2) + activationDateTime := uint64(time.Now().Unix()) + 200 + tokenizedPrice := big.NewInt(100) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.LastUpdateTimestamp, - r.Price, - r.MarketStatus, - r.CurrentMultiplier, - r.NewMultiplier, - r.ActivationDateTime, - r.TokenizedPrice, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + lastUpdateTS, + price, + marketStatus, + currentMultiplier, + newMultiplier, + activationDateTime, + tokenizedPrice, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.LastUpdateTimestamp.UnixNano() != int64(lastUpdateTS) { + t.Errorf("LastUpdateTimestamp mismatch: expected %d, got %d", lastUpdateTS, d.LastUpdateTimestamp.UnixNano()) + } + if d.Price.Cmp(price) != 0 { + t.Errorf("Price mismatch: expected %v, got %v", price, d.Price) + } + if d.MarketStatus != marketStatus { + t.Errorf("MarketStatus mismatch: expected %d, got %d", marketStatus, d.MarketStatus) + } + if d.CurrentMultiplier.Cmp(currentMultiplier) != 0 { + t.Errorf("CurrentMultiplier mismatch: expected %v, got %v", currentMultiplier, d.CurrentMultiplier) + } + if d.NewMultiplier.Cmp(newMultiplier) != 0 { + t.Errorf("NewMultiplier mismatch: expected %v, got %v", newMultiplier, d.NewMultiplier) + } + if d.ActivationDateTime.Unix() != int64(activationDateTime) { + t.Errorf("ActivationDateTime mismatch: expected %d, got %d", activationDateTime, d.ActivationDateTime.Unix()) + } + if d.TokenizedPrice.Cmp(tokenizedPrice) != 0 { + t.Errorf("TokenizedPrice mismatch: expected %v, got %v", tokenizedPrice, d.TokenizedPrice) } } diff --git a/go/report/v11/data.go b/go/report/v11/data.go index 96f1a99..1922123 100644 --- a/go/report/v11/data.go +++ b/go/report/v11/data.go @@ -3,10 +3,11 @@ package v11 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -22,11 +23,11 @@ func Schema() abi.Arguments { } return []abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "mid", Type: mustNewType("int192")}, {Name: "lastSeenTimestampNs", Type: mustNewType("uint64")}, @@ -42,11 +43,30 @@ func Schema() abi.Arguments { // Data is the container for this schema's attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ValidFromTimestamp uint32 - ObservationsTimestamp uint32 + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time NativeFee *big.Int LinkFee *big.Int - ExpiresAt uint32 + ExpiresAt time.Time + + Mid *big.Int + LastSeenTimestampNs time.Time // nanoseconds precision + Bid *big.Int + BidVolume *big.Int + Ask *big.Int + AskVolume *big.Int + LastTradedPrice *big.Int + MarketStatus uint32 +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ValidFromTimestamp uint64 + ObservationsTimestamp uint64 + NativeFee *big.Int + LinkFee *big.Int + ExpiresAt uint64 Mid *big.Int LastSeenTimestampNs uint64 @@ -69,9 +89,29 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + Mid: raw.Mid, + LastSeenTimestampNs: time.Unix(0, int64(raw.LastSeenTimestampNs)), // Always nanoseconds + Bid: raw.Bid, + BidVolume: raw.BidVolume, + Ask: raw.Ask, + AskVolume: raw.AskVolume, + LastTradedPrice: raw.LastTradedPrice, + MarketStatus: raw.MarketStatus, + } + return decoded, nil } diff --git a/go/report/v11/data_test.go b/go/report/v11/data_test.go index 8b1ae33..d969458 100644 --- a/go/report/v11/data_test.go +++ b/go/report/v11/data_test.go @@ -2,60 +2,96 @@ package v11 import ( "math/big" - "reflect" "testing" "time" - "github.com/smartcontractkit/data-streams-sdk/go/report/common" + "github.com/smartcontractkit/data-streams-sdk/go/v2/report/common" ) func TestData(t *testing.T) { - r := &Data{ - // 0x000bfb6d135897e4aaf5657bffd3b0b48f8e2a5131214c9ec2d62eac5d532067 - FeedID: [32]uint8{0, 11, 251, 109, 19, 88, 151, 228, 170, 245, 101, 123, 255, 211, 176, 180, 143, 142, 42, 81, 49, 33, 76, 158, 194, 214, 46, 172, 93, 83, 32, 103}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - - Mid: big.NewInt(103), - LastSeenTimestampNs: uint64(time.Now().Unix()), - Bid: big.NewInt(101), - BidVolume: big.NewInt(10002), - Ask: big.NewInt(105), - AskVolume: big.NewInt(10001), - LastTradedPrice: big.NewInt(103), - MarketStatus: common.MarketStatusOpen, - } + // Raw values for packing + feedID := [32]uint8{0, 11, 251, 109, 19, 88, 151, 228, 170, 245, 101, 123, 255, 211, 176, 180, 143, 142, 42, 81, 49, 33, 76, 158, 194, 214, 46, 172, 93, 83, 32, 103} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + mid := big.NewInt(103) + lastSeenTimestampNs := uint64(time.Now().UnixNano()) + bid := big.NewInt(101) + bidVolume := big.NewInt(10002) + ask := big.NewInt(105) + askVolume := big.NewInt(10001) + lastTradedPrice := big.NewInt(103) + marketStatus := common.MarketStatusOpen b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.Mid, - r.LastSeenTimestampNs, - r.Bid, - r.BidVolume, - r.Ask, - r.AskVolume, - r.LastTradedPrice, - r.MarketStatus, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + mid, + lastSeenTimestampNs, + bid, + bidVolume, + ask, + askVolume, + lastTradedPrice, + marketStatus, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.Mid.Cmp(mid) != 0 { + t.Errorf("Mid mismatch: expected %v, got %v", mid, d.Mid) + } + if d.LastSeenTimestampNs.UnixNano() != int64(lastSeenTimestampNs) { + t.Errorf("LastSeenTimestampNs mismatch: expected %d, got %d", lastSeenTimestampNs, d.LastSeenTimestampNs.UnixNano()) + } + if d.Bid.Cmp(bid) != 0 { + t.Errorf("Bid mismatch: expected %v, got %v", bid, d.Bid) + } + if d.BidVolume.Cmp(bidVolume) != 0 { + t.Errorf("BidVolume mismatch: expected %v, got %v", bidVolume, d.BidVolume) + } + if d.Ask.Cmp(ask) != 0 { + t.Errorf("Ask mismatch: expected %v, got %v", ask, d.Ask) + } + if d.AskVolume.Cmp(askVolume) != 0 { + t.Errorf("AskVolume mismatch: expected %v, got %v", askVolume, d.AskVolume) + } + if d.LastTradedPrice.Cmp(lastTradedPrice) != 0 { + t.Errorf("LastTradedPrice mismatch: expected %v, got %v", lastTradedPrice, d.LastTradedPrice) + } + if d.MarketStatus != marketStatus { + t.Errorf("MarketStatus mismatch: expected %d, got %d", marketStatus, d.MarketStatus) } } diff --git a/go/report/v12/data.go b/go/report/v12/data.go index a5bc2c9..9590f3d 100644 --- a/go/report/v12/data.go +++ b/go/report/v12/data.go @@ -3,10 +3,11 @@ package v12 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -22,11 +23,11 @@ func Schema() abi.Arguments { } return []abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "navPerShare", Type: mustNewType("int192")}, {Name: "nextNavPerShare", Type: mustNewType("int192")}, @@ -38,11 +39,26 @@ func Schema() abi.Arguments { // Data is the container for this schema's attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ValidFromTimestamp uint32 - ObservationsTimestamp uint32 + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time NativeFee *big.Int LinkFee *big.Int - ExpiresAt uint32 + ExpiresAt time.Time + + NavPerShare *big.Int + NextNavPerShare *big.Int + NavDate time.Time // nanoseconds precision + Ripcord uint32 +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ValidFromTimestamp uint64 + ObservationsTimestamp uint64 + NativeFee *big.Int + LinkFee *big.Int + ExpiresAt uint64 NavPerShare *big.Int NextNavPerShare *big.Int @@ -61,9 +77,25 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + NavPerShare: raw.NavPerShare, + NextNavPerShare: raw.NextNavPerShare, + NavDate: time.Unix(0, int64(raw.NavDate)), // Always nanoseconds + Ripcord: raw.Ripcord, + } + return decoded, nil } diff --git a/go/report/v12/data_test.go b/go/report/v12/data_test.go index a80ae50..fc8ec6e 100644 --- a/go/report/v12/data_test.go +++ b/go/report/v12/data_test.go @@ -2,51 +2,74 @@ package v12 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - // 0x000c6b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472 - FeedID: [32]uint8{00, 12, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - - NavPerShare: big.NewInt(1100), - NextNavPerShare: big.NewInt(1101), - NavDate: uint64(time.Now().UnixNano()) - 100, - Ripcord: 108, - } + // Raw values for packing + feedID := [32]uint8{0, 12, 251, 109, 19, 88, 151, 228, 170, 245, 101, 123, 255, 211, 176, 180, 143, 142, 42, 81, 49, 33, 76, 158, 194, 214, 46, 172, 93, 83, 32, 103} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + navPerShare := big.NewInt(100) + nextNavPerShare := big.NewInt(101) + navDate := uint64(time.Now().UnixNano()) - 100 + ripcord := uint32(1) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - - r.NavPerShare, - r.NextNavPerShare, - r.NavDate, - r.Ripcord, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + navPerShare, + nextNavPerShare, + navDate, + ripcord, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.NavPerShare.Cmp(navPerShare) != 0 { + t.Errorf("NavPerShare mismatch: expected %v, got %v", navPerShare, d.NavPerShare) + } + if d.NextNavPerShare.Cmp(nextNavPerShare) != 0 { + t.Errorf("NextNavPerShare mismatch: expected %v, got %v", nextNavPerShare, d.NextNavPerShare) + } + if d.NavDate.UnixNano() != int64(navDate) { + t.Errorf("NavDate mismatch: expected %d, got %d", navDate, d.NavDate.UnixNano()) + } + if d.Ripcord != ripcord { + t.Errorf("Ripcord mismatch: expected %d, got %d", ripcord, d.Ripcord) } } diff --git a/go/report/v13/data.go b/go/report/v13/data.go index b5970d6..0ce9c98 100644 --- a/go/report/v13/data.go +++ b/go/report/v13/data.go @@ -3,10 +3,11 @@ package v13 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -22,11 +23,11 @@ func Schema() abi.Arguments { } return []abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "bestAsk", Type: mustNewType("int192")}, {Name: "bestBid", Type: mustNewType("int192")}, {Name: "askVolume", Type: mustNewType("uint64")}, @@ -38,11 +39,26 @@ func Schema() abi.Arguments { // Data is the container for this schema's attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ValidFromTimestamp uint32 - ObservationsTimestamp uint32 + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time NativeFee *big.Int LinkFee *big.Int - ExpiresAt uint32 + ExpiresAt time.Time + BestAsk *big.Int + BestBid *big.Int + AskVolume uint64 + BidVolume uint64 + LastTradedPrice *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ValidFromTimestamp uint64 + ObservationsTimestamp uint64 + NativeFee *big.Int + LinkFee *big.Int + ExpiresAt uint64 BestAsk *big.Int BestBid *big.Int AskVolume uint64 @@ -61,9 +77,26 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + BestAsk: raw.BestAsk, + BestBid: raw.BestBid, + AskVolume: raw.AskVolume, + BidVolume: raw.BidVolume, + LastTradedPrice: raw.LastTradedPrice, + } + return decoded, nil } diff --git a/go/report/v13/data_test.go b/go/report/v13/data_test.go index 0fb4396..30cb48a 100644 --- a/go/report/v13/data_test.go +++ b/go/report/v13/data_test.go @@ -2,50 +2,79 @@ package v13 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{0, 13, 19, 169, 185, 197, 227, 122, 9, 159, 55, 78, 146, 195, 121, 20, 175, 92, 38, 143, 58, 138, 151, 33, 241, 114, 81, 53, 191, 180, 203, 184}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - BestAsk: big.NewInt(105), - BestBid: big.NewInt(101), - AskVolume: 10001, - BidVolume: 10002, - LastTradedPrice: big.NewInt(103), - } + // Raw values for packing + feedID := [32]uint8{0, 13, 251, 109, 19, 88, 151, 228, 170, 245, 101, 123, 255, 211, 176, 180, 143, 142, 42, 81, 49, 33, 76, 158, 194, 214, 46, 172, 93, 83, 32, 103} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + bestAsk := big.NewInt(105) + bestBid := big.NewInt(100) + askVolume := uint64(1000) + bidVolume := uint64(2000) + lastTradedPrice := big.NewInt(103) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.BestAsk, - r.BestBid, - r.AskVolume, - r.BidVolume, - r.LastTradedPrice, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + bestAsk, + bestBid, + askVolume, + bidVolume, + lastTradedPrice, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.BestAsk.Cmp(bestAsk) != 0 { + t.Errorf("BestAsk mismatch: expected %v, got %v", bestAsk, d.BestAsk) + } + if d.BestBid.Cmp(bestBid) != 0 { + t.Errorf("BestBid mismatch: expected %v, got %v", bestBid, d.BestBid) + } + if d.AskVolume != askVolume { + t.Errorf("AskVolume mismatch: expected %d, got %d", askVolume, d.AskVolume) + } + if d.BidVolume != bidVolume { + t.Errorf("BidVolume mismatch: expected %d, got %d", bidVolume, d.BidVolume) + } + if d.LastTradedPrice.Cmp(lastTradedPrice) != 0 { + t.Errorf("LastTradedPrice mismatch: expected %v, got %v", lastTradedPrice, d.LastTradedPrice) } } diff --git a/go/report/v2/data.go b/go/report/v2/data.go index fa23ece..72bdb35 100644 --- a/go/report/v2/data.go +++ b/go/report/v2/data.go @@ -3,9 +3,10 @@ package v2 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,11 +22,11 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "benchmarkPrice", Type: mustNewType("int192")}, }) } @@ -33,10 +34,21 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 + ObservationsTimestamp time.Time BenchmarkPrice *big.Int - ValidFromTimestamp uint32 - ExpiresAt uint32 + ValidFromTimestamp time.Time + ExpiresAt time.Time + LinkFee *big.Int + NativeFee *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 + BenchmarkPrice *big.Int + ValidFromTimestamp uint64 + ExpiresAt uint64 LinkFee *big.Int NativeFee *big.Int } @@ -52,9 +64,22 @@ func Decode(report []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + BenchmarkPrice: raw.BenchmarkPrice, + } + return decoded, nil } diff --git a/go/report/v2/data_test.go b/go/report/v2/data_test.go index 44ea752..7da2483 100644 --- a/go/report/v2/data_test.go +++ b/go/report/v2/data_test.go @@ -2,42 +2,59 @@ package v2 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ObservationsTimestamp: uint32(time.Now().Unix()), - BenchmarkPrice: big.NewInt(100), - ValidFromTimestamp: uint32(time.Now().Unix()), - ExpiresAt: uint32(time.Now().Unix()) + 100, - LinkFee: big.NewInt(10), - NativeFee: big.NewInt(10), - } + // Raw values for packing + feedID := [32]uint8{00, 02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + benchmarkPrice := big.NewInt(100) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.BenchmarkPrice, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + benchmarkPrice, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.BenchmarkPrice.Cmp(benchmarkPrice) != 0 { + t.Errorf("BenchmarkPrice mismatch: expected %v, got %v", benchmarkPrice, d.BenchmarkPrice) } } diff --git a/go/report/v3/data.go b/go/report/v3/data.go index a084f90..05d4839 100644 --- a/go/report/v3/data.go +++ b/go/report/v3/data.go @@ -3,9 +3,10 @@ package v3 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,11 +22,11 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "benchmarkPrice", Type: mustNewType("int192")}, {Name: "bid", Type: mustNewType("int192")}, {Name: "ask", Type: mustNewType("int192")}, @@ -35,12 +36,25 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 + ObservationsTimestamp time.Time BenchmarkPrice *big.Int Bid *big.Int Ask *big.Int - ValidFromTimestamp uint32 - ExpiresAt uint32 + ValidFromTimestamp time.Time + ExpiresAt time.Time + LinkFee *big.Int + NativeFee *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 + BenchmarkPrice *big.Int + Bid *big.Int + Ask *big.Int + ValidFromTimestamp uint64 + ExpiresAt uint64 LinkFee *big.Int NativeFee *big.Int } @@ -56,9 +70,24 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + BenchmarkPrice: raw.BenchmarkPrice, + Bid: raw.Bid, + Ask: raw.Ask, + } + return decoded, nil } diff --git a/go/report/v3/data_test.go b/go/report/v3/data_test.go index aa79447..7c753a5 100644 --- a/go/report/v3/data_test.go +++ b/go/report/v3/data_test.go @@ -2,46 +2,69 @@ package v3 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 03, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - BenchmarkPrice: big.NewInt(100), - Bid: big.NewInt(100), - Ask: big.NewInt(100), - } + // Raw values for packing + feedID := [32]uint8{00, 03, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + benchmarkPrice := big.NewInt(100) + bid := big.NewInt(100) + ask := big.NewInt(100) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.BenchmarkPrice, - r.Bid, - r.Ask, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + benchmarkPrice, + bid, + ask, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.BenchmarkPrice.Cmp(benchmarkPrice) != 0 { + t.Errorf("BenchmarkPrice mismatch: expected %v, got %v", benchmarkPrice, d.BenchmarkPrice) + } + if d.Bid.Cmp(bid) != 0 { + t.Errorf("Bid mismatch: expected %v, got %v", bid, d.Bid) + } + if d.Ask.Cmp(ask) != 0 { + t.Errorf("Ask mismatch: expected %v, got %v", ask, d.Ask) } } diff --git a/go/report/v4/data.go b/go/report/v4/data.go index 21dac62..b36d076 100644 --- a/go/report/v4/data.go +++ b/go/report/v4/data.go @@ -3,10 +3,11 @@ package v4 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -22,11 +23,11 @@ func Schema() abi.Arguments { } return []abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "benchmarkPrice", Type: mustNewType("int192")}, {Name: "marketStatus", Type: mustNewType("uint32")}, } @@ -35,11 +36,23 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 + ObservationsTimestamp time.Time BenchmarkPrice *big.Int MarketStatus uint32 - ValidFromTimestamp uint32 - ExpiresAt uint32 + ValidFromTimestamp time.Time + ExpiresAt time.Time + LinkFee *big.Int + NativeFee *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 + BenchmarkPrice *big.Int + MarketStatus uint32 + ValidFromTimestamp uint64 + ExpiresAt uint64 LinkFee *big.Int NativeFee *big.Int } @@ -55,9 +68,23 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + BenchmarkPrice: raw.BenchmarkPrice, + MarketStatus: raw.MarketStatus, + } + return decoded, nil } diff --git a/go/report/v4/data_test.go b/go/report/v4/data_test.go index ea6eb60..de2991a 100644 --- a/go/report/v4/data_test.go +++ b/go/report/v4/data_test.go @@ -2,46 +2,66 @@ package v4 import ( "math/big" - "reflect" "testing" "time" - "github.com/smartcontractkit/data-streams-sdk/go/report/common" + "github.com/smartcontractkit/data-streams-sdk/go/v2/report/common" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 04, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - BenchmarkPrice: big.NewInt(100), - MarketStatus: common.MarketStatusOpen, - } + // Raw values for packing + feedID := [32]uint8{00, 04, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + benchmarkPrice := big.NewInt(100) + marketStatus := common.MarketStatusOpen b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.BenchmarkPrice, - r.MarketStatus, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + benchmarkPrice, + marketStatus, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.BenchmarkPrice.Cmp(benchmarkPrice) != 0 { + t.Errorf("BenchmarkPrice mismatch: expected %v, got %v", benchmarkPrice, d.BenchmarkPrice) + } + if d.MarketStatus != marketStatus { + t.Errorf("MarketStatus mismatch: expected %v, got %v", marketStatus, d.MarketStatus) } } diff --git a/go/report/v5/data.go b/go/report/v5/data.go index 4f8b3cd..a1cf436 100644 --- a/go/report/v5/data.go +++ b/go/report/v5/data.go @@ -3,9 +3,10 @@ package v5 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,13 +22,13 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "rate", Type: mustNewType("int192")}, - {Name: "timestamp", Type: mustNewType("uint32")}, + {Name: "timestamp", Type: mustNewType("uint64")}, {Name: "duration", Type: mustNewType("uint32")}, }) } @@ -35,13 +36,26 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ValidFromTimestamp uint32 - ObservationsTimestamp uint32 + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time NativeFee *big.Int LinkFee *big.Int - ExpiresAt uint32 + ExpiresAt time.Time Rate *big.Int - Timestamp uint32 + Timestamp time.Time // Always seconds + Duration time.Duration // Always seconds +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ValidFromTimestamp uint64 + ObservationsTimestamp uint64 + NativeFee *big.Int + LinkFee *big.Int + ExpiresAt uint64 + Rate *big.Int + Timestamp uint64 Duration uint32 } @@ -56,9 +70,23 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + Rate: raw.Rate, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + Timestamp: time.Unix(int64(raw.Timestamp), 0), // Always seconds + Duration: time.Duration(raw.Duration) * time.Second, // Always seconds + } return decoded, nil } diff --git a/go/report/v5/data_test.go b/go/report/v5/data_test.go index b97b255..80025fb 100644 --- a/go/report/v5/data_test.go +++ b/go/report/v5/data_test.go @@ -2,46 +2,70 @@ package v5 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 05, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - Rate: big.NewInt(100), - Timestamp: uint32(time.Now().Unix()), - Duration: 3600, // 1 hour in seconds - } + // Raw values for packing + feedID := [32]uint8{00, 05, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + rate := big.NewInt(100) + timestamp := uint64(time.Now().Unix()) + duration := uint32(3600) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.Rate, - r.Timestamp, - r.Duration, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + rate, + timestamp, + duration, ) + if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } - // Test data decoding - decoded, err := Decode(b) + d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, decoded) { - t.Errorf("expected: %#v, got %#v", r, decoded) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.Rate.Cmp(rate) != 0 { + t.Errorf("Rate mismatch: expected %v, got %v", rate, d.Rate) + } + if d.Timestamp.Unix() != int64(timestamp) { + t.Errorf("Timestamp mismatch: expected %d, got %d", timestamp, d.Timestamp.Unix()) + } + expectedDuration := time.Duration(duration) * time.Second + if d.Duration != expectedDuration { + t.Errorf("Duration mismatch: expected %v, got %v", expectedDuration, d.Duration) } } diff --git a/go/report/v6/data.go b/go/report/v6/data.go index 4703ca2..999ff5c 100644 --- a/go/report/v6/data.go +++ b/go/report/v6/data.go @@ -3,9 +3,10 @@ package v6 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,11 +22,11 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "price", Type: mustNewType("int192")}, {Name: "price2", Type: mustNewType("int192")}, {Name: "price3", Type: mustNewType("int192")}, @@ -37,11 +38,26 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ValidFromTimestamp uint32 - ObservationsTimestamp uint32 + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time NativeFee *big.Int LinkFee *big.Int - ExpiresAt uint32 + ExpiresAt time.Time + Price *big.Int + Price2 *big.Int + Price3 *big.Int + Price4 *big.Int + Price5 *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ValidFromTimestamp uint64 + ObservationsTimestamp uint64 + NativeFee *big.Int + LinkFee *big.Int + ExpiresAt uint64 Price *big.Int Price2 *big.Int Price3 *big.Int @@ -60,9 +76,26 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + Price: raw.Price, + Price2: raw.Price2, + Price3: raw.Price3, + Price4: raw.Price4, + Price5: raw.Price5, + } + return decoded, nil } diff --git a/go/report/v6/data_test.go b/go/report/v6/data_test.go index 38205d4..fd89e6c 100644 --- a/go/report/v6/data_test.go +++ b/go/report/v6/data_test.go @@ -2,50 +2,79 @@ package v6 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 06, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - Price: big.NewInt(100), - Price2: big.NewInt(110), - Price3: big.NewInt(120), - Price4: big.NewInt(130), - Price5: big.NewInt(140), - } + // Raw values for packing + feedID := [32]uint8{00, 06, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + price := big.NewInt(100) + price2 := big.NewInt(101) + price3 := big.NewInt(102) + price4 := big.NewInt(103) + price5 := big.NewInt(104) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.Price, - r.Price2, - r.Price3, - r.Price4, - r.Price5, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + price, + price2, + price3, + price4, + price5, ) + if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } - // Test data decoding - decoded, err := Decode(b) + d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, decoded) { - t.Errorf("expected: %#v, got %#v", r, decoded) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.Price.Cmp(price) != 0 { + t.Errorf("Price mismatch: expected %v, got %v", price, d.Price) + } + if d.Price2.Cmp(price2) != 0 { + t.Errorf("Price2 mismatch: expected %v, got %v", price2, d.Price2) + } + if d.Price3.Cmp(price3) != 0 { + t.Errorf("Price3 mismatch: expected %v, got %v", price3, d.Price3) + } + if d.Price4.Cmp(price4) != 0 { + t.Errorf("Price4 mismatch: expected %v, got %v", price4, d.Price4) + } + if d.Price5.Cmp(price5) != 0 { + t.Errorf("Price5 mismatch: expected %v, got %v", price5, d.Price5) } } diff --git a/go/report/v7/data.go b/go/report/v7/data.go index 7c3afed..e2c4f3c 100644 --- a/go/report/v7/data.go +++ b/go/report/v7/data.go @@ -3,9 +3,10 @@ package v7 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,11 +22,11 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "exchangeRate", Type: mustNewType("int192")}, }) } @@ -33,11 +34,22 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ValidFromTimestamp uint32 - ObservationsTimestamp uint32 + ValidFromTimestamp time.Time + ObservationsTimestamp time.Time NativeFee *big.Int LinkFee *big.Int - ExpiresAt uint32 + ExpiresAt time.Time + ExchangeRate *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ValidFromTimestamp uint64 + ObservationsTimestamp uint64 + NativeFee *big.Int + LinkFee *big.Int + ExpiresAt uint64 ExchangeRate *big.Int } @@ -52,9 +64,22 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + ExchangeRate: raw.ExchangeRate, + } + return decoded, nil } diff --git a/go/report/v7/data_test.go b/go/report/v7/data_test.go index 10ecc78..0b22a0a 100644 --- a/go/report/v7/data_test.go +++ b/go/report/v7/data_test.go @@ -2,42 +2,59 @@ package v7 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 07, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - ExchangeRate: big.NewInt(100), - } + // Raw values for packing + feedID := [32]uint8{00, 07, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + exchangeRate := big.NewInt(100) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.ExchangeRate, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + exchangeRate, ) + if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } - // Test data decoding - decoded, err := Decode(b) + d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, decoded) { - t.Errorf("expected: %#v, got %#v", r, decoded) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.ExchangeRate.Cmp(exchangeRate) != 0 { + t.Errorf("ExchangeRate mismatch: expected %v, got %v", exchangeRate, d.ExchangeRate) } } diff --git a/go/report/v8/data.go b/go/report/v8/data.go index 9f65a70..73c3fba 100644 --- a/go/report/v8/data.go +++ b/go/report/v8/data.go @@ -3,10 +3,11 @@ package v8 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -22,11 +23,11 @@ func Schema() abi.Arguments { } return []abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "lastUpdateTimestamp", Type: mustNewType("uint64")}, {Name: "midPrice", Type: mustNewType("int192")}, {Name: "marketStatus", Type: mustNewType("uint32")}, @@ -36,12 +37,25 @@ func Schema() abi.Arguments { // Data is the container for this schema's attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 + ObservationsTimestamp time.Time + MidPrice *big.Int + MarketStatus uint32 + LastUpdateTimestamp time.Time // nanoseconds precision + ValidFromTimestamp time.Time + ExpiresAt time.Time + LinkFee *big.Int + NativeFee *big.Int +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 MidPrice *big.Int MarketStatus uint32 LastUpdateTimestamp uint64 - ValidFromTimestamp uint32 - ExpiresAt uint32 + ValidFromTimestamp uint64 + ExpiresAt uint64 LinkFee *big.Int NativeFee *big.Int } @@ -57,9 +71,24 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + LastUpdateTimestamp: time.Unix(0, int64(raw.LastUpdateTimestamp)), // Always nanoseconds + MidPrice: raw.MidPrice, + MarketStatus: raw.MarketStatus, + } + return decoded, nil } diff --git a/go/report/v8/data_test.go b/go/report/v8/data_test.go index 10df7f2..284a12f 100644 --- a/go/report/v8/data_test.go +++ b/go/report/v8/data_test.go @@ -2,48 +2,71 @@ package v8 import ( "math/big" - "reflect" "testing" "time" - "github.com/smartcontractkit/data-streams-sdk/go/report/common" + "github.com/smartcontractkit/data-streams-sdk/go/v2/report/common" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 8, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - LastUpdateTimestamp: uint64(time.Now().UnixNano() - int64(10*time.Second)), - MidPrice: big.NewInt(100), - MarketStatus: common.MarketStatusOpen, - } + // Raw values for packing + feedID := [32]uint8{00, 8, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + lastUpdateTS := uint64(time.Now().UnixNano() - int64(10*time.Second)) + midPrice := big.NewInt(100) + marketStatus := common.MarketStatusOpen b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.LastUpdateTimestamp, - r.MidPrice, - r.MarketStatus, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + lastUpdateTS, + midPrice, + marketStatus, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.LastUpdateTimestamp.UnixNano() != int64(lastUpdateTS) { + t.Errorf("LastUpdateTimestamp mismatch: expected %d, got %d", lastUpdateTS, d.LastUpdateTimestamp.UnixNano()) + } + if d.MidPrice.Cmp(midPrice) != 0 { + t.Errorf("MidPrice mismatch: expected %v, got %v", midPrice, d.MidPrice) + } + if d.MarketStatus != marketStatus { + t.Errorf("MarketStatus mismatch: expected %v, got %v", marketStatus, d.MarketStatus) } } diff --git a/go/report/v9/data.go b/go/report/v9/data.go index 050bbcb..8734f6d 100644 --- a/go/report/v9/data.go +++ b/go/report/v9/data.go @@ -3,9 +3,10 @@ package v9 import ( "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" ) var schema = Schema() @@ -21,11 +22,11 @@ func Schema() abi.Arguments { } return abi.Arguments([]abi.Argument{ {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, + {Name: "validFromTimestamp", Type: mustNewType("uint64")}, + {Name: "observationsTimestamp", Type: mustNewType("uint64")}, {Name: "nativeFee", Type: mustNewType("uint192")}, {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, + {Name: "expiresAt", Type: mustNewType("uint64")}, {Name: "navPerShare", Type: mustNewType("int192")}, {Name: "navDate", Type: mustNewType("uint64")}, {Name: "aum", Type: mustNewType("int192")}, @@ -36,9 +37,23 @@ func Schema() abi.Arguments { // Data is the container for this schema attributes type Data struct { FeedID feed.ID `abi:"feedId"` - ObservationsTimestamp uint32 - ValidFromTimestamp uint32 - ExpiresAt uint32 + ObservationsTimestamp time.Time + ValidFromTimestamp time.Time + ExpiresAt time.Time + LinkFee *big.Int + NativeFee *big.Int + NavPerShare *big.Int + NavDate time.Time // nanoseconds precision + Aum *big.Int + Ripcord uint32 +} + +// rawData is used internally for ABI decoding - types must match ABI schema +type rawData struct { + FeedID feed.ID `abi:"feedId"` + ObservationsTimestamp uint64 + ValidFromTimestamp uint64 + ExpiresAt uint64 LinkFee *big.Int NativeFee *big.Int NavPerShare *big.Int @@ -58,9 +73,25 @@ func Decode(data []byte) (*Data, error) { if err != nil { return nil, fmt.Errorf("failed to decode report: %w", err) } - decoded := new(Data) - if err = schema.Copy(decoded, values); err != nil { + raw := new(rawData) + if err = schema.Copy(raw, values); err != nil { return nil, fmt.Errorf("failed to copy report values to struct: %w", err) } + + res := raw.FeedID.Resolution() + + decoded := &Data{ + FeedID: raw.FeedID, + ValidFromTimestamp: feed.ParseTimestamp(raw.ValidFromTimestamp, res), + ObservationsTimestamp: feed.ParseTimestamp(raw.ObservationsTimestamp, res), + NativeFee: raw.NativeFee, + LinkFee: raw.LinkFee, + ExpiresAt: feed.ParseTimestamp(raw.ExpiresAt, res), + NavPerShare: raw.NavPerShare, + NavDate: time.Unix(0, int64(raw.NavDate)), // Always nanoseconds + Aum: raw.Aum, + Ripcord: raw.Ripcord, + } + return decoded, nil } diff --git a/go/report/v9/data_test.go b/go/report/v9/data_test.go index f62f0c7..6147bfe 100644 --- a/go/report/v9/data_test.go +++ b/go/report/v9/data_test.go @@ -2,48 +2,74 @@ package v9 import ( "math/big" - "reflect" "testing" "time" ) func TestData(t *testing.T) { - r := &Data{ - FeedID: [32]uint8{00, 9, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - ValidFromTimestamp: uint32(time.Now().Unix()), - ObservationsTimestamp: uint32(time.Now().Unix()), - NativeFee: big.NewInt(10), - LinkFee: big.NewInt(10), - ExpiresAt: uint32(time.Now().Unix()) + 100, - NavPerShare: big.NewInt(1100), - NavDate: uint64(time.Now().UnixNano()) - 100, - Aum: big.NewInt(11009), - Ripcord: 108, - } + // Raw values for packing + feedID := [32]uint8{00, 9, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} + validFromTS := uint64(time.Now().Unix()) + observationsTS := uint64(time.Now().Unix()) + nativeFee := big.NewInt(10) + linkFee := big.NewInt(10) + expiresAt := uint64(time.Now().Unix()) + 100 + navPerShare := big.NewInt(100) + navDate := uint64(time.Now().UnixNano()) - 100 + aum := big.NewInt(1000000) + ripcord := uint32(1) b, err := schema.Pack( - r.FeedID, - r.ValidFromTimestamp, - r.ObservationsTimestamp, - r.NativeFee, - r.LinkFee, - r.ExpiresAt, - r.NavPerShare, - r.NavDate, - r.Aum, - r.Ripcord, + feedID, + validFromTS, + observationsTS, + nativeFee, + linkFee, + expiresAt, + navPerShare, + navDate, + aum, + ripcord, ) if err != nil { - t.Errorf("failed to serialize report: %s", err) + t.Fatalf("failed to serialize report: %s", err) } d, err := Decode(b) if err != nil { - t.Errorf("failed to deserialize report: %s", err) + t.Fatalf("failed to deserialize report: %s", err) } - if !reflect.DeepEqual(r, d) { - t.Errorf("expected: %#v, got %#v", r, d) + // Verify decoded values + if d.FeedID != feedID { + t.Errorf("FeedID mismatch: expected %v, got %v", feedID, d.FeedID) + } + if d.ValidFromTimestamp.Unix() != int64(validFromTS) { + t.Errorf("ValidFromTimestamp mismatch: expected %d, got %d", validFromTS, d.ValidFromTimestamp.Unix()) + } + if d.ObservationsTimestamp.Unix() != int64(observationsTS) { + t.Errorf("ObservationsTimestamp mismatch: expected %d, got %d", observationsTS, d.ObservationsTimestamp.Unix()) + } + if d.NativeFee.Cmp(nativeFee) != 0 { + t.Errorf("NativeFee mismatch: expected %v, got %v", nativeFee, d.NativeFee) + } + if d.LinkFee.Cmp(linkFee) != 0 { + t.Errorf("LinkFee mismatch: expected %v, got %v", linkFee, d.LinkFee) + } + if d.ExpiresAt.Unix() != int64(expiresAt) { + t.Errorf("ExpiresAt mismatch: expected %d, got %d", expiresAt, d.ExpiresAt.Unix()) + } + if d.NavPerShare.Cmp(navPerShare) != 0 { + t.Errorf("NavPerShare mismatch: expected %v, got %v", navPerShare, d.NavPerShare) + } + if d.NavDate.UnixNano() != int64(navDate) { + t.Errorf("NavDate mismatch: expected %d, got %d", navDate, d.NavDate.UnixNano()) + } + if d.Aum.Cmp(aum) != 0 { + t.Errorf("Aum mismatch: expected %v, got %v", aum, d.Aum) + } + if d.Ripcord != ripcord { + t.Errorf("Ripcord mismatch: expected %d, got %d", ripcord, d.Ripcord) } } diff --git a/go/stream.go b/go/stream.go index acd1ec5..c06f2e2 100644 --- a/go/stream.go +++ b/go/stream.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "time" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" "nhooyr.io/websocket" ) @@ -83,7 +83,7 @@ type stream struct { connStatusCallback func(isConneccted bool, host string, origin string) waterMarkMu sync.Mutex - waterMark map[string]uint64 + waterMark map[string]time.Time stats struct { accepted atomic.Uint64 @@ -107,7 +107,7 @@ func (c *client) newStream(ctx context.Context, httpClient *http.Client, feedIDs config: c.config, output: make(chan *ReportResponse, 1), feedIDs: feedIDs, - waterMark: make(map[string]uint64), + waterMark: make(map[string]time.Time), streamCtx: streamCtx, streamCtxCancel: streamCtxCancel, } @@ -173,24 +173,30 @@ func (s *stream) pingConn(ctx context.Context, conn *wsConn) { for { select { case <-ctx.Done(): + s.config.logDebug("client: stream websocket %s ping loop exiting: context done", conn.origin) return case <-ticker.C: + pingStart := time.Now() pctx, pcancel := context.WithTimeout(context.Background(), 2*time.Second) err := conn.conn.Ping(pctx) pcancel() + pingDuration := time.Since(pingStart) if s.closed.Load() { + s.config.logDebug("client: stream websocket %s ping loop exiting: stream closed", conn.origin) return } if err != nil { s.config.logInfo( - "client: stream websocket %s ping error: %s, closing client: %s", - conn.origin, err, conn.close(), + "client: stream websocket %s ping FAILED after %v: %s, closing connection: %s", + conn.origin, pingDuration, err, conn.close(), ) return } + + s.config.logDebug("client: stream websocket %s ping OK (took %v)", conn.origin, pingDuration) } } } @@ -355,7 +361,8 @@ func (s *stream) accept(ctx context.Context, m *message) (err error) { id := m.Report.FeedID.String() s.waterMarkMu.Lock() - if s.waterMark[id] >= m.Report.ObservationsTimestamp { + // Skip older reports and reports with the same timestamp + if !m.Report.ObservationsTimestamp.After(s.waterMark[id]) { s.stats.skipped.Add(1) s.waterMarkMu.Unlock() return nil @@ -420,7 +427,7 @@ func (ws *wsConn) replace(c *websocket.Conn) { } func (s *stream) newWSconn(ctx context.Context, origin string) (ws *wsConn, err error) { - reqURL := s.config.wsURL.ResolveReference(&url.URL{Path: apiV1WS}) + reqURL := s.config.wsURL.ResolveReference(&url.URL{Path: apiV2WS}) reqURL.RawQuery = url.Values{"feedIDs": {strings.Join(feedIdsToStringList(s.feedIDs), ",")}}.Encode() headers := http.Header{} diff --git a/go/stream_test.go b/go/stream_test.go index 2406ab0..b214aba 100644 --- a/go/stream_test.go +++ b/go/stream_test.go @@ -6,20 +6,19 @@ import ( "errors" "fmt" "net/http" - "reflect" "sync" "sync/atomic" "testing" "time" - "github.com/smartcontractkit/data-streams-sdk/go/feed" + "github.com/smartcontractkit/data-streams-sdk/go/v2/feed" "nhooyr.io/websocket" ) func TestClient_Subscribe(t *testing.T) { expectedReports := []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 12344}, - {FeedID: feed2, ObservationsTimestamp: 12344}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12344, 0)}, } expectedFeedIdListStr := fmt.Sprintf("%s,%s", feed1.String(), feed2.String()) @@ -28,8 +27,8 @@ func TestClient_Subscribe(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } if r.URL.Query().Get("feedIDs") != expectedFeedIdListStr { @@ -98,7 +97,7 @@ func TestClient_Subscribe(t *testing.T) { reports = append(reports, rep) } - if !reflect.DeepEqual(reports, expectedReports) { + if !reportResponsesEqual(reports, expectedReports) { t.Errorf("Read() = %v, want %v", reports, expectedReports) } @@ -117,8 +116,8 @@ func TestClient_SubscribeWithCallback(t *testing.T) { if r.Method == http.MethodHead { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } _, err := websocket.Accept( w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionContextTakeover}, @@ -223,8 +222,8 @@ func TestClient_SubscribeCanceledContext(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } if r.URL.Query().Get("feedIDs") != expectedFeedIdListStr { @@ -257,8 +256,8 @@ func TestClient_SubscribeCanceledContext(t *testing.T) { func TestClient_StreamReconnectMerge(t *testing.T) { expectedReports := []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 12344}, - {FeedID: feed2, ObservationsTimestamp: 12344}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12344, 0)}, } expectedFeedIdListStr := fmt.Sprintf("%s,%s", feed1.String(), feed2.String()) @@ -268,8 +267,8 @@ func TestClient_StreamReconnectMerge(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } if r.URL.Query().Get("feedIDs") != expectedFeedIdListStr { @@ -351,7 +350,7 @@ func TestClient_StreamReconnectMerge(t *testing.T) { stats := sub.Stats() sub.Close() - if !reflect.DeepEqual(reports, expectedReports) { + if !reportResponsesEqual(reports, expectedReports) { t.Errorf("Read() = %v, want %v", reports, expectedReports) } @@ -366,14 +365,14 @@ func TestClient_StreamReconnectMerge(t *testing.T) { func TestClient_StreamHA(t *testing.T) { expectedReports1 := []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 12344}, - {FeedID: feed2, ObservationsTimestamp: 12344}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12344, 0)}, } expectedReports2 := []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 12344}, - {FeedID: feed2, ObservationsTimestamp: 12344}, - {FeedID: feed1, ObservationsTimestamp: 12345}, - {FeedID: feed2, ObservationsTimestamp: 12346}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12345, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12346, 0)}, } expectedFeedIdListStr := fmt.Sprintf("%s,%s", feed1.String(), feed2.String()) @@ -384,8 +383,8 @@ func TestClient_StreamHA(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } if r.URL.Query().Get("feedIDs") != expectedFeedIdListStr { @@ -465,7 +464,7 @@ func TestClient_StreamHA(t *testing.T) { reports = append(reports, rep) } - if !reflect.DeepEqual(reports, expectedReports2) { + if !reportResponsesEqual(reports, expectedReports2) { t.Errorf("Read() = %v, want %v", reports, expectedReports2) } @@ -479,8 +478,8 @@ func TestClient_StreamHA(t *testing.T) { func TestClient_ReadCancel(t *testing.T) { expectedReports := []*ReportResponse{ - {FeedID: feed1, ObservationsTimestamp: 12344}, - {FeedID: feed2, ObservationsTimestamp: 12344}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(12344, 0)}, + {FeedID: feed2, ObservationsTimestamp: time.Unix(12344, 0)}, } expectedFeedIdListStr := fmt.Sprintf("%s,%s", feed1.String(), feed2.String()) @@ -489,8 +488,8 @@ func TestClient_ReadCancel(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } if r.URL.Query().Get("feedIDs") != expectedFeedIdListStr { @@ -579,7 +578,7 @@ func TestClient_ReadCancel(t *testing.T) { t.Errorf("expected nil report, got %#v", rep) } - if !reflect.DeepEqual(reports, expectedReports) { + if !reportResponsesEqual(reports, expectedReports) { t.Errorf("Read() = %v, want %v", reports, expectedReports) } @@ -601,8 +600,8 @@ func TestClient_StreamHAPartialReconnect(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } conn, err := websocket.Accept( @@ -662,8 +661,8 @@ func TestClient_StreamCustomHeader(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } if r.Header.Get("custom-header") != "custom-value" { @@ -730,8 +729,8 @@ func TestClient_StreamHA_OneOriginDown(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } origin := r.Header.Get(cllOriginHeader) @@ -814,8 +813,8 @@ func TestClient_StreamHA_OneOriginDownRecovery(t *testing.T) { return } - if r.URL.Path != apiV1WS { - t.Errorf("expected path %s, got %s", apiV1WS, r.URL.Path) + if r.URL.Path != apiV2WS { + t.Errorf("expected path %s, got %s", apiV2WS, r.URL.Path) } origin := r.Header.Get(cllOriginHeader) From 07e8891cb9f42dd62277cd74de64ceee3699c97c Mon Sep 17 00:00:00 2001 From: Josh Brown Date: Wed, 4 Mar 2026 16:13:00 -0500 Subject: [PATCH 2/9] correct upgrade path --- go/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index f820158..2f950be 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,4 +1,4 @@ -module github.com/smartcontractkit/data-streams-sdk/go/v2 +module github.com/smartcontractkit/data-streams-sdk/go go 1.24.0 From 4a24435f919b0055027cc3edc6b5efa0f36cbaef Mon Sep 17 00:00:00 2001 From: Josh Brown Date: Wed, 4 Mar 2026 16:26:28 -0500 Subject: [PATCH 3/9] Revert "correct upgrade path" due to tagging issues This reverts commit 07e8891cb9f42dd62277cd74de64ceee3699c97c. --- go/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 2f950be..f820158 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,4 +1,4 @@ -module github.com/smartcontractkit/data-streams-sdk/go +module github.com/smartcontractkit/data-streams-sdk/go/v2 go 1.24.0 From a8bb492747236cccc4160fdf6961225189d00791 Mon Sep 17 00:00:00 2001 From: cal Date: Thu, 9 Apr 2026 15:07:38 +1000 Subject: [PATCH 4/9] feat: add WsAllowOutOfOrder config --- go/README.md | 2 + go/config.go | 1 + go/stream.go | 37 ++++- go/stream_test.go | 157 ++++++++++++++++++ rust/crates/sdk/src/config.rs | 13 ++ rust/crates/sdk/src/stream.rs | 6 + .../sdk/src/stream/monitor_connection.rs | 33 +++- typescript/src/stream/deduplication.ts | 60 +++++-- typescript/src/stream/index.ts | 9 +- typescript/src/stream/stats.ts | 10 ++ typescript/src/types/client.ts | 11 ++ typescript/src/types/metrics.ts | 9 + .../tests/unit/stream/stream-stats.test.ts | 2 + 13 files changed, 320 insertions(+), 30 deletions(-) diff --git a/go/README.md b/go/README.md index 692aab9..03484e7 100644 --- a/go/README.md +++ b/go/README.md @@ -175,6 +175,7 @@ type Config struct { WsURL string // Websocket Api url WsHA bool // Use concurrent connections to multiple Streams servers + WsAllowOutOfOrder bool // Allow out-of-order reports through while still deduplicating HA duplicates WsMaxReconnect int // Maximum number of reconnection attempts for Stream underlying connections LogDebug bool // Log debug information InsecureSkipVerify bool // Skip server certificate chain and host name verification @@ -292,6 +293,7 @@ func (r *ReportResponse) UnmarshalJSON(b []byte) (err error) type Stats struct { Accepted uint64 // Total number of accepted reports Deduplicated uint64 // Total number of deduplicated reports when in HA + OutOfOrder uint64 // Total number of out-of-order reports seen TotalReceived uint64 // Total number of received reports PartialReconnects uint64 // Total number of partial reconnects when in HA FullReconnects uint64 // Total number of full reconnects diff --git a/go/config.go b/go/config.go index bea548c..b74f764 100644 --- a/go/config.go +++ b/go/config.go @@ -15,6 +15,7 @@ type Config struct { WsURL string // Websocket Api url wsURL *url.URL // Websocket Api url WsHA bool // Use concurrent connections to multiple Streams servers + WsAllowOutOfOrder bool // Allow out-of-order reports through while still deduplicating HA duplicates WsMaxReconnect int // Maximum number of reconnection attempts for Stream underlying connections LogDebug bool // Log debug information InsecureSkipVerify bool // Skip server certificate chain and host name verification diff --git a/go/stream.go b/go/stream.go index ecbe347..56faa6a 100644 --- a/go/stream.go +++ b/go/stream.go @@ -54,6 +54,7 @@ type Stream interface { type Stats struct { Accepted uint64 // Total number of accepted reports Deduplicated uint64 // Total number of deduplicated reports when in HA + OutOfOrder uint64 // Total number of out-of-order reports seen TotalReceived uint64 // Total number of received reports PartialReconnects uint64 // Total number of partial reconnects when in HA FullReconnects uint64 // Total number of full reconnects @@ -63,8 +64,8 @@ type Stats struct { func (s Stats) String() (st string) { return fmt.Sprintf( - "accepted: %d, deduplicated: %d, total_received %d, partial_reconnects: %d, full_reconnects: %d, configured_connections: %d, active_connections %d", - s.Accepted, s.Deduplicated, + "accepted: %d, deduplicated: %d, out_of_order: %d, total_received %d, partial_reconnects: %d, full_reconnects: %d, configured_connections: %d, active_connections %d", + s.Accepted, s.Deduplicated, s.OutOfOrder, s.TotalReceived, s.PartialReconnects, s.FullReconnects, s.ConfiguredConnections, s.ActiveConnections, ) @@ -88,6 +89,7 @@ type stream struct { stats struct { accepted atomic.Uint64 skipped atomic.Uint64 + outOfOrder atomic.Uint64 partialReconnects atomic.Uint64 fullReconnects atomic.Uint64 activeConnections atomic.Uint64 @@ -310,6 +312,7 @@ func (s *stream) newWSconnWithRetry(origin string) (conn *wsConn, err error) { func (s *stream) Stats() (st Stats) { st.Accepted = s.stats.accepted.Load() st.Deduplicated = s.stats.skipped.Load() + st.OutOfOrder = s.stats.outOfOrder.Load() st.TotalReceived = st.Accepted + st.Deduplicated st.PartialReconnects = s.stats.partialReconnects.Load() st.FullReconnects = s.stats.fullReconnects.Load() @@ -359,17 +362,35 @@ func (s *stream) Close() (err error) { func (s *stream) accept(ctx context.Context, m *message) (err error) { id := m.Report.FeedID.String() + ts := m.Report.ObservationsTimestamp s.waterMarkMu.Lock() - // Skip older reports and reports with the same timestamp - if !m.Report.ObservationsTimestamp.After(s.waterMark[id]) { - s.stats.skipped.Add(1) - s.waterMarkMu.Unlock() - return nil + wm := s.waterMark[id] + + if s.config.WsAllowOutOfOrder { + if ts.Equal(wm) { + s.stats.skipped.Add(1) + s.waterMarkMu.Unlock() + return nil + } + if ts.Before(wm) { + s.stats.outOfOrder.Add(1) + } else { + s.waterMark[id] = ts + } + } else { + if !ts.After(wm) { + s.stats.skipped.Add(1) + if ts.Before(wm) { + s.stats.outOfOrder.Add(1) + } + s.waterMarkMu.Unlock() + return nil + } + s.waterMark[id] = ts } s.stats.accepted.Add(1) - s.waterMark[id] = m.Report.ObservationsTimestamp s.waterMarkMu.Unlock() select { diff --git a/go/stream_test.go b/go/stream_test.go index b214aba..071d389 100644 --- a/go/stream_test.go +++ b/go/stream_test.go @@ -801,6 +801,163 @@ func TestClient_StreamHA_OneOriginDown(t *testing.T) { } +func TestClient_StreamOutOfOrder(t *testing.T) { + reports := []*ReportResponse{ + {FeedID: feed1, ObservationsTimestamp: time.Unix(100, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(102, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(101, 0)}, // out-of-order + } + + ms := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + return + } + + conn, err := websocket.Accept( + w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionContextTakeover}, + ) + if err != nil { + t.Fatalf("error accepting connection: %s", err) + } + defer func() { _ = conn.CloseNow() }() + + for _, rpt := range reports { + b, err := json.Marshal(&message{rpt}) + if err != nil { + t.Errorf("failed to serialize message: %s", err) + } + if err := conn.Write(context.Background(), websocket.MessageBinary, b); err != nil { + t.Errorf("failed to write message: %s", err) + } + } + + for conn.Ping(context.Background()) == nil { + time.Sleep(100 * time.Millisecond) + } + }) + defer ms.Close() + + streamsClient, err := ms.Client() + if err != nil { + t.Fatalf("error creating client %s", err) + } + + cc := streamsClient.(*client) + cc.config.Logger = LogPrintf + cc.config.LogDebug = true + cc.config.WsAllowOutOfOrder = true + + sub, err := streamsClient.Stream(context.Background(), []feed.ID{feed1}) + if err != nil { + t.Fatalf("error subscribing %s", err) + } + defer sub.Close() + + var received []*ReportResponse + for i := 0; i < len(reports); i++ { + rep, err := sub.Read(context.Background()) + if err != nil { + t.Fatalf("error reading report %s", err) + } + received = append(received, rep) + } + + if !reportResponsesEqual(received, reports) { + t.Errorf("Read() = %v, want %v", received, reports) + } + + stats := sub.Stats() + if stats.Accepted != 3 { + t.Errorf("stats.Accepted = %d, want 3", stats.Accepted) + } + if stats.OutOfOrder != 1 { + t.Errorf("stats.OutOfOrder = %d, want 1", stats.OutOfOrder) + } + if stats.Deduplicated != 0 { + t.Errorf("stats.Deduplicated = %d, want 0", stats.Deduplicated) + } +} + +func TestClient_StreamOutOfOrder_DefaultDrop(t *testing.T) { + reports := []*ReportResponse{ + {FeedID: feed1, ObservationsTimestamp: time.Unix(100, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(102, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(101, 0)}, // out-of-order, should be dropped + } + + ms := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + return + } + + conn, err := websocket.Accept( + w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionContextTakeover}, + ) + if err != nil { + t.Fatalf("error accepting connection: %s", err) + } + defer func() { _ = conn.CloseNow() }() + + for _, rpt := range reports { + b, err := json.Marshal(&message{rpt}) + if err != nil { + t.Errorf("failed to serialize message: %s", err) + } + if err := conn.Write(context.Background(), websocket.MessageBinary, b); err != nil { + t.Errorf("failed to write message: %s", err) + } + } + + for conn.Ping(context.Background()) == nil { + time.Sleep(100 * time.Millisecond) + } + }) + defer ms.Close() + + streamsClient, err := ms.Client() + if err != nil { + t.Fatalf("error creating client %s", err) + } + + cc := streamsClient.(*client) + cc.config.Logger = LogPrintf + cc.config.LogDebug = true + + sub, err := streamsClient.Stream(context.Background(), []feed.ID{feed1}) + if err != nil { + t.Fatalf("error subscribing %s", err) + } + defer sub.Close() + + var received []*ReportResponse + for i := 0; i < 2; i++ { + rep, err := sub.Read(context.Background()) + if err != nil { + t.Fatalf("error reading report %s", err) + } + received = append(received, rep) + } + + // Wait for the third message to be processed by accept (it should be dropped) + time.Sleep(50 * time.Millisecond) + + expectedDelivered := reports[:2] + if !reportResponsesEqual(received, expectedDelivered) { + t.Errorf("Read() = %v, want %v", received, expectedDelivered) + } + + stats := sub.Stats() + if stats.Accepted != 2 { + t.Errorf("stats.Accepted = %d, want 2", stats.Accepted) + } + if stats.Deduplicated != 1 { + t.Errorf("stats.Deduplicated = %d, want 1", stats.Deduplicated) + } + if stats.OutOfOrder != 1 { + t.Errorf("stats.OutOfOrder = %d, want 1", stats.OutOfOrder) + } +} + // Tests that when in HA mode both origins are up after a recovery period even if one origin is down on initial connection func TestClient_StreamHA_OneOriginDownRecovery(t *testing.T) { connectAttempts := &atomic.Uint64{} diff --git a/rust/crates/sdk/src/config.rs b/rust/crates/sdk/src/config.rs index e55a551..627bfe9 100644 --- a/rust/crates/sdk/src/config.rs +++ b/rust/crates/sdk/src/config.rs @@ -51,6 +51,9 @@ pub struct Config { /// High Availability Mode: Use concurrent connections to multiple Streams servers pub ws_ha: WebSocketHighAvailability, + /// Allow out-of-order reports through while still deduplicating HA duplicates + pub ws_allow_out_of_order: bool, + /// Maximum number of reconnection attempts for underlying WebSocket connections pub ws_max_reconnect: usize, @@ -65,6 +68,7 @@ pub struct Config { impl Config { const DEFAULT_WS_MAX_RECONNECT: usize = 5; const DEFAULT_WS_HA: WebSocketHighAvailability = WebSocketHighAvailability::Disabled; + const DEFAULT_WS_ALLOW_OUT_OF_ORDER: bool = false; const DEFAULT_INSECURE_SKIP_VERIFY: InsecureSkipVerify = InsecureSkipVerify::Disabled; const DEFAULT_INSPECT_HTTP_RESPONSE: Option = None; @@ -140,6 +144,7 @@ impl Config { rest_url, ws_url, ws_ha: Self::DEFAULT_WS_HA, + ws_allow_out_of_order: Self::DEFAULT_WS_ALLOW_OUT_OF_ORDER, ws_max_reconnect: Self::DEFAULT_WS_MAX_RECONNECT, insecure_skip_verify: Self::DEFAULT_INSECURE_SKIP_VERIFY, inspect_http_response: Self::DEFAULT_INSPECT_HTTP_RESPONSE, @@ -160,6 +165,7 @@ pub struct ConfigBuilder { rest_url: String, ws_url: String, ws_ha: WebSocketHighAvailability, + ws_allow_out_of_order: bool, ws_max_reconnect: usize, insecure_skip_verify: InsecureSkipVerify, inspect_http_response: Option, @@ -172,6 +178,12 @@ impl ConfigBuilder { self } + /// Sets the `ws_allow_out_of_order` parameter. + pub fn with_ws_allow_out_of_order(mut self, ws_allow_out_of_order: bool) -> Self { + self.ws_allow_out_of_order = ws_allow_out_of_order; + self + } + // Sets the `ws_max_reconnect` parameter. pub fn with_ws_max_reconnect(mut self, ws_max_reconnect: usize) -> Self { self.ws_max_reconnect = ws_max_reconnect; @@ -206,6 +218,7 @@ impl ConfigBuilder { rest_url: self.rest_url, ws_url: self.ws_url, ws_ha: self.ws_ha, + ws_allow_out_of_order: self.ws_allow_out_of_order, ws_max_reconnect: self.ws_max_reconnect, insecure_skip_verify: self.insecure_skip_verify, inspect_http_response: self.inspect_http_response, diff --git a/rust/crates/sdk/src/stream.rs b/rust/crates/sdk/src/stream.rs index 4c4006d..7d7a955 100644 --- a/rust/crates/sdk/src/stream.rs +++ b/rust/crates/sdk/src/stream.rs @@ -57,6 +57,8 @@ struct Stats { accepted: AtomicUsize, /// Total number of deduplicated reports when in HA deduplicated: AtomicUsize, + /// Total number of out-of-order reports seen + out_of_order: AtomicUsize, /// Total number of partial reconnects when in HA partial_reconnects: AtomicUsize, /// Total number of full reconnects @@ -135,6 +137,7 @@ impl Stream { let stats = Arc::new(Stats { accepted: AtomicUsize::new(0), deduplicated: AtomicUsize::new(0), + out_of_order: AtomicUsize::new(0), partial_reconnects: AtomicUsize::new(0), full_reconnects: AtomicUsize::new(0), configured_connections: AtomicUsize::new(0), @@ -257,6 +260,7 @@ impl Stream { StatsSnapshot { accepted, deduplicated, + out_of_order: self.stats.out_of_order.load(Ordering::SeqCst), total_received: accepted + deduplicated, partial_reconnects: self.stats.partial_reconnects.load(Ordering::SeqCst), full_reconnects: self.stats.full_reconnects.load(Ordering::SeqCst), @@ -273,6 +277,8 @@ pub struct StatsSnapshot { pub accepted: usize, /// Total number of deduplicated reports when in HA pub deduplicated: usize, + /// Total number of out-of-order reports seen + pub out_of_order: usize, /// Total number of received reports pub total_received: usize, /// Total number of partial reconnects when in HA diff --git a/rust/crates/sdk/src/stream/monitor_connection.rs b/rust/crates/sdk/src/stream/monitor_connection.rs index 1cd497b..a2c486f 100644 --- a/rust/crates/sdk/src/stream/monitor_connection.rs +++ b/rust/crates/sdk/src/stream/monitor_connection.rs @@ -48,16 +48,41 @@ pub(crate) async fn run_stream( let feed_id = report.report.feed_id.to_hex_string(); let observations_timestamp = report.report.observations_timestamp; - if water_mark.lock().await.contains_key(&feed_id) && water_mark.lock().await[&feed_id] >= observations_timestamp { - stats.deduplicated.fetch_add(1, Ordering::SeqCst); - continue; + let mut wm = water_mark.lock().await; + let current_wm = wm.get(&feed_id).copied(); + + if let Some(current) = current_wm { + if config.ws_allow_out_of_order { + if observations_timestamp == current { + stats.deduplicated.fetch_add(1, Ordering::SeqCst); + drop(wm); + continue; + } + if observations_timestamp < current { + stats.out_of_order.fetch_add(1, Ordering::SeqCst); + } else { + wm.insert(feed_id.clone(), observations_timestamp); + } + } else { + if observations_timestamp <= current { + stats.deduplicated.fetch_add(1, Ordering::SeqCst); + if observations_timestamp < current { + stats.out_of_order.fetch_add(1, Ordering::SeqCst); + } + drop(wm); + continue; + } + wm.insert(feed_id.clone(), observations_timestamp); + } + } else { + wm.insert(feed_id.clone(), observations_timestamp); } + drop(wm); report_sender.send(report).await.map_err(|e| { StreamError::ConnectionError(format!("Failed to send report: {}", e)) })?; - water_mark.lock().await.insert(feed_id, observations_timestamp); stats.accepted.fetch_add(1, Ordering::SeqCst); } else { diff --git a/typescript/src/stream/deduplication.ts b/typescript/src/stream/deduplication.ts index bed17ce..660e6ce 100644 --- a/typescript/src/stream/deduplication.ts +++ b/typescript/src/stream/deduplication.ts @@ -12,12 +12,14 @@ export interface ReportMetadata { export interface DeduplicationResult { isAccepted: boolean; isDuplicate: boolean; + isOutOfOrder: boolean; reason?: string; } export interface DeduplicationStats { accepted: number; deduplicated: number; + outOfOrder: number; totalReceived: number; watermarkCount: number; } @@ -29,20 +31,24 @@ export class ReportDeduplicator { private waterMark: Map = new Map(); private acceptedCount = 0; private deduplicatedCount = 0; + private outOfOrderCount = 0; private cleanupInterval: NodeJS.Timeout | null = null; // Configuration private readonly maxWatermarkAge: number; private readonly cleanupIntervalMs: number; + private readonly allowOutOfOrder: boolean; constructor( options: { maxWatermarkAge?: number; // How long to keep watermarks (default: 1 hour) cleanupIntervalMs?: number; // How often to clean old watermarks (default: 5 minutes) + allowOutOfOrder?: boolean; // Allow out-of-order reports through (default: false) } = {} ) { this.maxWatermarkAge = options.maxWatermarkAge ?? 60 * 60 * 1000; // 1 hour this.cleanupIntervalMs = options.cleanupIntervalMs ?? 5 * 60 * 1000; // 5 minutes + this.allowOutOfOrder = options.allowOutOfOrder ?? false; // Start periodic cleanup this.startCleanup(); @@ -55,27 +61,47 @@ export class ReportDeduplicator { const feedId = report.feedID; const observationsTimestamp = report.observationsTimestamp; - // Get current watermark for this feed const currentWatermark = this.waterMark.get(feedId); - // Check if this report is older than or equal to the watermark - if (currentWatermark !== undefined && currentWatermark >= observationsTimestamp) { - this.deduplicatedCount++; - return { - isAccepted: false, - isDuplicate: true, - reason: `Report timestamp ${observationsTimestamp} <= watermark ${currentWatermark} for feed ${feedId}`, - }; + if (currentWatermark !== undefined) { + if (this.allowOutOfOrder) { + if (observationsTimestamp === currentWatermark) { + this.deduplicatedCount++; + return { + isAccepted: false, + isDuplicate: true, + isOutOfOrder: false, + reason: `Duplicate timestamp ${observationsTimestamp} for feed ${feedId}`, + }; + } + if (observationsTimestamp < currentWatermark) { + this.outOfOrderCount++; + this.acceptedCount++; + return { isAccepted: true, isDuplicate: false, isOutOfOrder: true }; + } + this.waterMark.set(feedId, observationsTimestamp); + } else { + if (observationsTimestamp <= currentWatermark) { + this.deduplicatedCount++; + const isOOO = observationsTimestamp < currentWatermark; + if (isOOO) { + this.outOfOrderCount++; + } + return { + isAccepted: false, + isDuplicate: true, + isOutOfOrder: isOOO, + reason: `Report timestamp ${observationsTimestamp} <= watermark ${currentWatermark} for feed ${feedId}`, + }; + } + this.waterMark.set(feedId, observationsTimestamp); + } + } else { + this.waterMark.set(feedId, observationsTimestamp); } - // Accept the report and update watermark - this.waterMark.set(feedId, observationsTimestamp); this.acceptedCount++; - - return { - isAccepted: true, - isDuplicate: false, - }; + return { isAccepted: true, isDuplicate: false, isOutOfOrder: false }; } /** @@ -85,6 +111,7 @@ export class ReportDeduplicator { return { accepted: this.acceptedCount, deduplicated: this.deduplicatedCount, + outOfOrder: this.outOfOrderCount, totalReceived: this.acceptedCount + this.deduplicatedCount, watermarkCount: this.waterMark.size, }; @@ -135,6 +162,7 @@ export class ReportDeduplicator { reset(): void { this.acceptedCount = 0; this.deduplicatedCount = 0; + this.outOfOrderCount = 0; this.waterMark.clear(); } diff --git a/typescript/src/stream/index.ts b/typescript/src/stream/index.ts index 232441c..c4859ff 100644 --- a/typescript/src/stream/index.ts +++ b/typescript/src/stream/index.ts @@ -132,8 +132,9 @@ export class Stream extends EventEmitter { this.connectionManager = new ConnectionManager(config, managerConfig); - // Initialize deduplicator for HA mode (single mode can also benefit from deduplication) - this.deduplicator = new ReportDeduplicator(); + this.deduplicator = new ReportDeduplicator({ + allowOutOfOrder: config.wsAllowOutOfOrder, + }); // Inject StreamStats into ConnectionManager for unified metrics this.connectionManager.setStreamStats(this.stats); @@ -234,6 +235,10 @@ export class Stream extends EventEmitter { const originInfo = origin ? ` from ${new URL(origin).host}` : ""; + if (result.isOutOfOrder) { + this.stats.incrementOutOfOrder(); + } + if (result.isAccepted) { this.stats.incrementAccepted(); this.logger.debug( diff --git a/typescript/src/stream/stats.ts b/typescript/src/stream/stats.ts index 15da3ef..d3f6cc4 100644 --- a/typescript/src/stream/stats.ts +++ b/typescript/src/stream/stats.ts @@ -6,6 +6,7 @@ import { MetricsSnapshot, ConnectionStatus } from "../types/metrics"; export class StreamStats { private _accepted = 0; private _deduplicated = 0; + private _outOfOrder = 0; private _partialReconnects = 0; private _fullReconnects = 0; private _configuredConnections = 0; @@ -33,6 +34,13 @@ export class StreamStats { this._totalReceived++; } + /** + * Increment the number of out-of-order reports seen + */ + incrementOutOfOrder(): void { + this._outOfOrder++; + } + /** * Increment the number of partial reconnects (some connections lost but not all) */ @@ -102,6 +110,7 @@ export class StreamStats { reset(): void { this._accepted = 0; this._deduplicated = 0; + this._outOfOrder = 0; this._partialReconnects = 0; this._fullReconnects = 0; this._totalReceived = 0; @@ -121,6 +130,7 @@ export class StreamStats { return { accepted: this._accepted, deduplicated: this._deduplicated, + outOfOrder: this._outOfOrder, partialReconnects: this._partialReconnects, fullReconnects: this._fullReconnects, configuredConnections: this._configuredConnections, diff --git a/typescript/src/types/client.ts b/typescript/src/types/client.ts index 6904062..6836d14 100644 --- a/typescript/src/types/client.ts +++ b/typescript/src/types/client.ts @@ -128,6 +128,17 @@ export interface Config { */ haMode?: boolean; + /** + * Allow out-of-order reports through while still deduplicating HA duplicates. + * + * When false (default), any report with timestamp <= watermark is dropped. + * When true, only exact-timestamp matches are deduplicated; out-of-order + * reports (older but distinct timestamps) are delivered. + * + * @default false + */ + wsAllowOutOfOrder?: boolean; + /** * Connection timeout for individual WebSocket connections in HA mode (milliseconds). * diff --git a/typescript/src/types/metrics.ts b/typescript/src/types/metrics.ts index a8240e2..3d5aded 100644 --- a/typescript/src/types/metrics.ts +++ b/typescript/src/types/metrics.ts @@ -50,6 +50,15 @@ export interface MetricsSnapshot { */ readonly deduplicated: number; + /** + * Total number of out-of-order reports seen. + * + * Tracks reports received with a timestamp older than the current watermark. + * When wsAllowOutOfOrder is false, these reports are dropped (included in deduplicated count). + * When wsAllowOutOfOrder is true, these reports are delivered (included in accepted count). + */ + readonly outOfOrder: number; + /** * Total number of reports received across all connections. * diff --git a/typescript/tests/unit/stream/stream-stats.test.ts b/typescript/tests/unit/stream/stream-stats.test.ts index c7241d0..8d37ebc 100644 --- a/typescript/tests/unit/stream/stream-stats.test.ts +++ b/typescript/tests/unit/stream/stream-stats.test.ts @@ -31,6 +31,7 @@ describe("StreamStats Tests", () => { expect(initialStats).toEqual({ accepted: 0, deduplicated: 0, + outOfOrder: 0, partialReconnects: 0, fullReconnects: 0, configuredConnections: 1, // Default is 1 @@ -147,6 +148,7 @@ describe("StreamStats Tests", () => { expect(finalStats).toEqual({ accepted: 4, deduplicated: 2, + outOfOrder: 0, partialReconnects: 1, fullReconnects: 1, configuredConnections: 2, From 09cfb0341f1c0f386d0c5518407bb3376bd3fd84 Mon Sep 17 00:00:00 2001 From: cal Date: Fri, 10 Apr 2026 13:25:26 +1000 Subject: [PATCH 5/9] refactor: extract deduplication into FeedDeduplicator --- go/dedup.go | 59 +++ go/dedup_test.go | 109 +++++ go/stream.go | 39 +- rust/crates/sdk/src/stream.rs | 27 +- rust/crates/sdk/src/stream/dedup.rs | 158 +++++++ .../sdk/src/stream/monitor_connection.rs | 50 +-- typescript/src/stream/deduplication.ts | 229 ++++------ .../tests/unit/stream/deduplication.test.ts | 425 +++++++++--------- 8 files changed, 660 insertions(+), 436 deletions(-) create mode 100644 go/dedup.go create mode 100644 go/dedup_test.go create mode 100644 rust/crates/sdk/src/stream/dedup.rs diff --git a/go/dedup.go b/go/dedup.go new file mode 100644 index 0000000..fc9bda8 --- /dev/null +++ b/go/dedup.go @@ -0,0 +1,59 @@ +package streams + +const seenBufferSize = 32 + +type Verdict int + +const ( + Accept Verdict = iota + Duplicate + OutOfOrder +) + +type feedState struct { + watermark int64 + ring [seenBufferSize]int64 + set map[int64]struct{} + cursor int + count int +} + +type FeedDeduplicator struct { + feeds map[string]*feedState +} + +func NewFeedDeduplicator() *FeedDeduplicator { + return &FeedDeduplicator{feeds: make(map[string]*feedState)} +} + +func (d *FeedDeduplicator) Check(feedID string, ts int64) Verdict { + fs := d.feeds[feedID] + if fs == nil { + fs = &feedState{set: make(map[int64]struct{}, seenBufferSize)} + d.feeds[feedID] = fs + } + + if _, dup := fs.set[ts]; dup { + return Duplicate + } + + if fs.count == seenBufferSize { + evict := fs.ring[fs.cursor] + delete(fs.set, evict) + } else { + fs.count++ + } + fs.ring[fs.cursor] = ts + fs.set[ts] = struct{}{} + fs.cursor = (fs.cursor + 1) % seenBufferSize + + isOutOfOrder := fs.watermark > 0 && ts < fs.watermark + if ts > fs.watermark { + fs.watermark = ts + } + + if isOutOfOrder { + return OutOfOrder + } + return Accept +} diff --git a/go/dedup_test.go b/go/dedup_test.go new file mode 100644 index 0000000..2cda469 --- /dev/null +++ b/go/dedup_test.go @@ -0,0 +1,109 @@ +package streams + +import "testing" + +func TestFeedDeduplicator_Accept(t *testing.T) { + d := NewFeedDeduplicator() + if v := d.Check("feed-1", 100); v != Accept { + t.Fatalf("expected Accept, got %d", v) + } +} + +func TestFeedDeduplicator_Duplicate(t *testing.T) { + d := NewFeedDeduplicator() + d.Check("feed-1", 100) + if v := d.Check("feed-1", 100); v != Duplicate { + t.Fatalf("expected Duplicate, got %d", v) + } +} + +func TestFeedDeduplicator_OutOfOrder(t *testing.T) { + d := NewFeedDeduplicator() + d.Check("feed-1", 200) + if v := d.Check("feed-1", 100); v != OutOfOrder { + t.Fatalf("expected OutOfOrder, got %d", v) + } +} + +func TestFeedDeduplicator_OutOfOrderNotDuplicate(t *testing.T) { + d := NewFeedDeduplicator() + d.Check("feed-1", 200) + v := d.Check("feed-1", 100) + if v != OutOfOrder { + t.Fatalf("expected OutOfOrder for first OOO delivery, got %d", v) + } + if v := d.Check("feed-1", 100); v != Duplicate { + t.Fatalf("expected Duplicate for second OOO delivery, got %d", v) + } +} + +func TestFeedDeduplicator_FIFOEviction(t *testing.T) { + d := NewFeedDeduplicator() + for i := int64(1); i <= seenBufferSize; i++ { + d.Check("feed-1", i) + } + d.Check("feed-1", 33) + // ts=2 (second inserted) should still be in the buffer + if v := d.Check("feed-1", 2); v != Duplicate { + t.Fatalf("expected ts=2 still in buffer, got %d", v) + } + // ts=1 (first inserted) was evicted by ts=33 + if v := d.Check("feed-1", 1); v == Duplicate { + t.Fatal("expected ts=1 to be evicted (FIFO), but got Duplicate") + } +} + +func TestFeedDeduplicator_FIFOEvictsOldestInsertedNotSmallest(t *testing.T) { + d := NewFeedDeduplicator() + // Insert out of order: 100, 1, 2, 3, ..., 31 (total 32 entries) + d.Check("feed-1", 100) + for i := int64(1); i <= seenBufferSize-1; i++ { + d.Check("feed-1", i) + } + // Buffer is full. ts=100 was inserted first (oldest by insertion). + // Adding ts=999 should evict ts=100, NOT ts=1 (the smallest value). + d.Check("feed-1", 999) + // ts=1 should still be present (smallest value, but NOT oldest inserted) + if v := d.Check("feed-1", 1); v != Duplicate { + t.Fatalf("expected ts=1 (smallest value, but not oldest inserted) to remain, got %d", v) + } + // ts=100 should have been evicted (oldest inserted) + if v := d.Check("feed-1", 100); v == Duplicate { + t.Fatal("expected ts=100 (oldest inserted) to be evicted, but got Duplicate") + } +} + +func TestFeedDeduplicator_IndependentFeeds(t *testing.T) { + d := NewFeedDeduplicator() + d.Check("feed-a", 100) + d.Check("feed-b", 100) + + if v := d.Check("feed-a", 100); v != Duplicate { + t.Fatalf("expected Duplicate for feed-a, got %d", v) + } + if v := d.Check("feed-b", 100); v != Duplicate { + t.Fatalf("expected Duplicate for feed-b, got %d", v) + } + // Different feed, same ts is not a duplicate + if v := d.Check("feed-c", 100); v != Accept { + t.Fatalf("expected Accept for new feed-c, got %d", v) + } +} + +func TestFeedDeduplicator_WatermarkZeroNotOutOfOrder(t *testing.T) { + d := NewFeedDeduplicator() + // First report at ts=0 should be Accept, not OutOfOrder + if v := d.Check("feed-1", 0); v != Accept { + t.Fatalf("expected Accept for first report at ts=0, got %d", v) + } +} + +func TestFeedDeduplicator_HADuplicateAfterWatermarkAdvance(t *testing.T) { + d := NewFeedDeduplicator() + d.Check("feed-1", 100) // Accept + d.Check("feed-1", 200) // Accept, watermark -> 200 + // HA duplicate of ts=100 arrives from second connection + if v := d.Check("feed-1", 100); v != Duplicate { + t.Fatalf("expected Duplicate for HA retransmit, got %d", v) + } +} diff --git a/go/stream.go b/go/stream.go index 56faa6a..81fe025 100644 --- a/go/stream.go +++ b/go/stream.go @@ -83,8 +83,8 @@ type stream struct { closeError atomic.Value connStatusCallback func(isConneccted bool, host string, origin string) - waterMarkMu sync.Mutex - waterMark map[string]time.Time + feedsMu sync.Mutex + dedup *FeedDeduplicator stats struct { accepted atomic.Uint64 @@ -109,7 +109,7 @@ func (c *client) newStream(ctx context.Context, httpClient *http.Client, feedIDs config: c.config, output: make(chan *ReportResponse, 1), feedIDs: feedIDs, - waterMark: make(map[string]time.Time), + dedup: NewFeedDeduplicator(), streamCtx: streamCtx, streamCtxCancel: streamCtxCancel, } @@ -362,36 +362,25 @@ func (s *stream) Close() (err error) { func (s *stream) accept(ctx context.Context, m *message) (err error) { id := m.Report.FeedID.String() - ts := m.Report.ObservationsTimestamp + ts := m.Report.ObservationsTimestamp.UnixMilli() - s.waterMarkMu.Lock() - wm := s.waterMark[id] + s.feedsMu.Lock() + verdict := s.dedup.Check(id, ts) + s.feedsMu.Unlock() - if s.config.WsAllowOutOfOrder { - if ts.Equal(wm) { - s.stats.skipped.Add(1) - s.waterMarkMu.Unlock() - return nil - } - if ts.Before(wm) { - s.stats.outOfOrder.Add(1) - } else { - s.waterMark[id] = ts - } - } else { - if !ts.After(wm) { + switch verdict { + case Duplicate: + s.stats.skipped.Add(1) + return nil + case OutOfOrder: + s.stats.outOfOrder.Add(1) + if !s.config.WsAllowOutOfOrder { s.stats.skipped.Add(1) - if ts.Before(wm) { - s.stats.outOfOrder.Add(1) - } - s.waterMarkMu.Unlock() return nil } - s.waterMark[id] = ts } s.stats.accepted.Add(1) - s.waterMarkMu.Unlock() select { case <-ctx.Done(): diff --git a/rust/crates/sdk/src/stream.rs b/rust/crates/sdk/src/stream.rs index 7d7a955..4b478ac 100644 --- a/rust/crates/sdk/src/stream.rs +++ b/rust/crates/sdk/src/stream.rs @@ -1,6 +1,8 @@ +mod dedup; mod establish_connection; mod monitor_connection; +use dedup::FeedDeduplicator; use establish_connection::connect; use monitor_connection::run_stream; @@ -10,12 +12,9 @@ use chainlink_data_streams_report::feed_id::ID; use chainlink_data_streams_report::report::Report; use serde::{Deserialize, Serialize}; -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, }; use tokio::{ net::TcpStream, @@ -23,7 +22,7 @@ use tokio::{ time::{sleep, Duration}, }; use tokio_tungstenite::{MaybeTlsStream, WebSocketStream as TungsteniteWebSocketStream}; -use tracing::{debug, error, info}; +use tracing::{debug, info}; pub const DEFAULT_WS_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); pub const MIN_WS_RECONNECT_INTERVAL: Duration = Duration::from_millis(1000); @@ -87,7 +86,7 @@ pub struct Stream { report_receiver: mpsc::Receiver, shutdown_sender: broadcast::Sender<()>, stats: Arc, - water_mark: Arc>>, + dedup: Arc>, } impl Stream { @@ -146,7 +145,7 @@ impl Stream { let conn = connect(config, &feed_ids, stats.clone()).await?; - let water_mark = Arc::new(Mutex::new(HashMap::new())); + let dedup = Arc::new(Mutex::new(FeedDeduplicator::new())); Ok(Stream { config: config.clone(), @@ -156,7 +155,7 @@ impl Stream { report_receiver, shutdown_sender, stats, - water_mark, + dedup, }) } @@ -173,7 +172,7 @@ impl Stream { let report_sender = self.report_sender.clone(); let shutdown_receiver = self.shutdown_sender.subscribe(); let stats = self.stats.clone(); - let water_mark = self.water_mark.clone(); + let dedup = self.dedup.clone(); let config = self.config.clone(); let feed_ids = self.feed_ids.clone(); @@ -182,7 +181,7 @@ impl Stream { report_sender, shutdown_receiver, stats, - water_mark, + dedup, config, feed_ids, )); @@ -192,7 +191,7 @@ impl Stream { let report_sender = self.report_sender.clone(); let shutdown_receiver = self.shutdown_sender.subscribe(); let stats = self.stats.clone(); - let water_mark = self.water_mark.clone(); + let dedup = self.dedup.clone(); let config = self.config.clone(); let feed_ids = self.feed_ids.clone(); @@ -201,7 +200,7 @@ impl Stream { report_sender, shutdown_receiver, stats, - water_mark, + dedup, config, feed_ids, )); diff --git a/rust/crates/sdk/src/stream/dedup.rs b/rust/crates/sdk/src/stream/dedup.rs new file mode 100644 index 0000000..2c16a71 --- /dev/null +++ b/rust/crates/sdk/src/stream/dedup.rs @@ -0,0 +1,158 @@ +use std::collections::{HashMap, HashSet}; + +const SEEN_BUFFER_SIZE: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Verdict { + Accept, + Duplicate, + OutOfOrder, +} + +struct FeedState { + watermark: u64, + ring: [u64; SEEN_BUFFER_SIZE], + set: HashSet, + cursor: usize, + count: usize, +} + +impl FeedState { + fn new() -> Self { + Self { + watermark: 0, + ring: [0; SEEN_BUFFER_SIZE], + set: HashSet::with_capacity(SEEN_BUFFER_SIZE), + cursor: 0, + count: 0, + } + } +} + +pub(crate) struct FeedDeduplicator { + feeds: HashMap, +} + +impl FeedDeduplicator { + pub fn new() -> Self { + Self { + feeds: HashMap::new(), + } + } + + pub fn check(&mut self, feed_id: &str, ts: u64) -> Verdict { + let fs = self + .feeds + .entry(feed_id.to_owned()) + .or_insert_with(FeedState::new); + + if fs.set.contains(&ts) { + return Verdict::Duplicate; + } + + if fs.count == SEEN_BUFFER_SIZE { + let evict = fs.ring[fs.cursor]; + fs.set.remove(&evict); + } else { + fs.count += 1; + } + fs.ring[fs.cursor] = ts; + fs.set.insert(ts); + fs.cursor = (fs.cursor + 1) % SEEN_BUFFER_SIZE; + + let is_out_of_order = fs.watermark > 0 && ts < fs.watermark; + if ts > fs.watermark { + fs.watermark = ts; + } + + if is_out_of_order { + Verdict::OutOfOrder + } else { + Verdict::Accept + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accept() { + let mut d = FeedDeduplicator::new(); + assert_eq!(d.check("feed-1", 100), Verdict::Accept); + } + + #[test] + fn duplicate() { + let mut d = FeedDeduplicator::new(); + d.check("feed-1", 100); + assert_eq!(d.check("feed-1", 100), Verdict::Duplicate); + } + + #[test] + fn out_of_order() { + let mut d = FeedDeduplicator::new(); + d.check("feed-1", 200); + assert_eq!(d.check("feed-1", 100), Verdict::OutOfOrder); + } + + #[test] + fn out_of_order_then_duplicate() { + let mut d = FeedDeduplicator::new(); + d.check("feed-1", 200); + assert_eq!(d.check("feed-1", 100), Verdict::OutOfOrder); + assert_eq!(d.check("feed-1", 100), Verdict::Duplicate); + } + + #[test] + fn fifo_eviction() { + let mut d = FeedDeduplicator::new(); + for i in 1..=SEEN_BUFFER_SIZE as u64 { + d.check("feed-1", i); + } + d.check("feed-1", 33); + // ts=2 still present + assert_eq!(d.check("feed-1", 2), Verdict::Duplicate); + // ts=1 was evicted (FIFO oldest inserted) + assert_ne!(d.check("feed-1", 1), Verdict::Duplicate); + } + + #[test] + fn fifo_evicts_oldest_inserted_not_smallest_value() { + let mut d = FeedDeduplicator::new(); + d.check("feed-1", 100); + for i in 1..SEEN_BUFFER_SIZE as u64 { + d.check("feed-1", i); + } + d.check("feed-1", 999); + // ts=1 (smallest value) should still be present + assert_eq!(d.check("feed-1", 1), Verdict::Duplicate); + // ts=100 (oldest inserted) should be evicted + assert_ne!(d.check("feed-1", 100), Verdict::Duplicate); + } + + #[test] + fn independent_feeds() { + let mut d = FeedDeduplicator::new(); + d.check("feed-a", 100); + d.check("feed-b", 100); + assert_eq!(d.check("feed-a", 100), Verdict::Duplicate); + assert_eq!(d.check("feed-b", 100), Verdict::Duplicate); + assert_eq!(d.check("feed-c", 100), Verdict::Accept); + } + + #[test] + fn watermark_zero_not_out_of_order() { + let mut d = FeedDeduplicator::new(); + assert_eq!(d.check("feed-1", 0), Verdict::Accept); + } + + #[test] + fn ha_duplicate_after_watermark_advance() { + let mut d = FeedDeduplicator::new(); + d.check("feed-1", 100); + d.check("feed-1", 200); + assert_eq!(d.check("feed-1", 100), Verdict::Duplicate); + } +} diff --git a/rust/crates/sdk/src/stream/monitor_connection.rs b/rust/crates/sdk/src/stream/monitor_connection.rs index a2c486f..9ba8421 100644 --- a/rust/crates/sdk/src/stream/monitor_connection.rs +++ b/rust/crates/sdk/src/stream/monitor_connection.rs @@ -1,4 +1,4 @@ -use super::{Stats, StreamError, WebSocketReport}; +use super::{dedup::{FeedDeduplicator, Verdict}, Stats, StreamError, WebSocketReport}; use crate::{config::Config, stream::establish_connection::try_to_reconnect}; @@ -6,12 +6,9 @@ use chainlink_data_streams_report::feed_id::ID; use futures::SinkExt; use futures_util::StreamExt; -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, }; use tokio::{ net::TcpStream, @@ -27,7 +24,7 @@ pub(crate) async fn run_stream( report_sender: mpsc::Sender, mut shutdown_receiver: broadcast::Receiver<()>, stats: Arc, - water_mark: Arc>>, + dedup: Arc>, config: Config, feed_ids: Vec, ) -> Result<(), StreamError> { @@ -46,45 +43,30 @@ pub(crate) async fn run_stream( info!("Received new report from Data Streams Endpoint."); if let Ok(report) = serde_json::from_slice::(&data) { let feed_id = report.report.feed_id.to_hex_string(); - let observations_timestamp = report.report.observations_timestamp; + let ts = report.report.observations_timestamp as u64; - let mut wm = water_mark.lock().await; - let current_wm = wm.get(&feed_id).copied(); + let verdict = dedup.lock().await.check(&feed_id, ts); - if let Some(current) = current_wm { - if config.ws_allow_out_of_order { - if observations_timestamp == current { - stats.deduplicated.fetch_add(1, Ordering::SeqCst); - drop(wm); - continue; - } - if observations_timestamp < current { - stats.out_of_order.fetch_add(1, Ordering::SeqCst); - } else { - wm.insert(feed_id.clone(), observations_timestamp); - } - } else { - if observations_timestamp <= current { + match verdict { + Verdict::Duplicate => { + stats.deduplicated.fetch_add(1, Ordering::SeqCst); + continue; + } + Verdict::OutOfOrder => { + stats.out_of_order.fetch_add(1, Ordering::SeqCst); + if !config.ws_allow_out_of_order { stats.deduplicated.fetch_add(1, Ordering::SeqCst); - if observations_timestamp < current { - stats.out_of_order.fetch_add(1, Ordering::SeqCst); - } - drop(wm); continue; } - wm.insert(feed_id.clone(), observations_timestamp); } - } else { - wm.insert(feed_id.clone(), observations_timestamp); + Verdict::Accept => {} } - drop(wm); report_sender.send(report).await.map_err(|e| { StreamError::ConnectionError(format!("Failed to send report: {}", e)) })?; stats.accepted.fetch_add(1, Ordering::SeqCst); - } else { error!("Failed to parse binary message."); } diff --git a/typescript/src/stream/deduplication.ts b/typescript/src/stream/deduplication.ts index 660e6ce..0060e1a 100644 --- a/typescript/src/stream/deduplication.ts +++ b/typescript/src/stream/deduplication.ts @@ -1,7 +1,12 @@ /** - * Report deduplication using watermark timestamps + * Report deduplication using a bounded set of recently seen timestamps per feed. + * Each feed tracks a watermark (highest timestamp) for ordering decisions and a + * set of recently seen timestamps for deduplication, allowing correct dedup of + * both in-order and out-of-order HA duplicates. */ +const SEEN_BUFFER_SIZE = 32; + export interface ReportMetadata { feedID: string; observationsTimestamp: number; @@ -24,11 +29,20 @@ export interface DeduplicationStats { watermarkCount: number; } -/** - * Manages report deduplication using watermark timestamps - */ +enum Verdict { + Accept, + Duplicate, + OutOfOrder, +} + +interface FeedState { + watermark: number; + seen: Set; +} + +// ReportDeduplicator manages deduplication of reports for a set of feeds. export class ReportDeduplicator { - private waterMark: Map = new Map(); + private feedState: Map = new Map(); private acceptedCount = 0; private deduplicatedCount = 0; private outOfOrderCount = 0; @@ -54,193 +68,116 @@ export class ReportDeduplicator { this.startCleanup(); } - /** - * Process a report and determine if it should be accepted or deduplicated - */ - processReport(report: ReportMetadata): DeduplicationResult { - const feedId = report.feedID; - const observationsTimestamp = report.observationsTimestamp; + private check(feedId: string, ts: number): Verdict { + let state = this.feedState.get(feedId); + if (!state) { + state = { watermark: 0, seen: new Set() }; + this.feedState.set(feedId, state); + } - const currentWatermark = this.waterMark.get(feedId); + if (state.seen.has(ts)) { + return Verdict.Duplicate; + } - if (currentWatermark !== undefined) { - if (this.allowOutOfOrder) { - if (observationsTimestamp === currentWatermark) { - this.deduplicatedCount++; - return { - isAccepted: false, - isDuplicate: true, - isOutOfOrder: false, - reason: `Duplicate timestamp ${observationsTimestamp} for feed ${feedId}`, - }; - } - if (observationsTimestamp < currentWatermark) { - this.outOfOrderCount++; - this.acceptedCount++; - return { isAccepted: true, isDuplicate: false, isOutOfOrder: true }; - } - this.waterMark.set(feedId, observationsTimestamp); - } else { - if (observationsTimestamp <= currentWatermark) { + if (state.seen.size >= SEEN_BUFFER_SIZE) { + const oldest = state.seen.values().next().value!; + state.seen.delete(oldest); + } + state.seen.add(ts); + + const isOutOfOrder = state.watermark > 0 && ts < state.watermark; + if (ts > state.watermark) { + state.watermark = ts; + } + + if (isOutOfOrder) { + return Verdict.OutOfOrder; + } + return Verdict.Accept; + } + + // Process a report and return a verdict on whether it is accepted, duplicated, or out-of-order. + processReport(report: ReportMetadata): DeduplicationResult { + const feedId = report.feedID; + const ts = report.observationsTimestamp; + const verdict = this.check(feedId, ts); + + switch (verdict) { + case Verdict.Duplicate: + this.deduplicatedCount++; + return { + isAccepted: false, + isDuplicate: true, + isOutOfOrder: false, + reason: `Duplicate timestamp ${ts} already seen for feed ${feedId}`, + }; + + case Verdict.OutOfOrder: { + this.outOfOrderCount++; + if (!this.allowOutOfOrder) { this.deduplicatedCount++; - const isOOO = observationsTimestamp < currentWatermark; - if (isOOO) { - this.outOfOrderCount++; - } return { isAccepted: false, - isDuplicate: true, - isOutOfOrder: isOOO, - reason: `Report timestamp ${observationsTimestamp} <= watermark ${currentWatermark} for feed ${feedId}`, + isDuplicate: false, + isOutOfOrder: true, + reason: `Out-of-order timestamp ${ts} < watermark ${this.feedState.get(feedId)!.watermark} for feed ${feedId}`, }; } - this.waterMark.set(feedId, observationsTimestamp); + this.acceptedCount++; + return { isAccepted: true, isDuplicate: false, isOutOfOrder: true }; } - } else { - this.waterMark.set(feedId, observationsTimestamp); - } - this.acceptedCount++; - return { isAccepted: true, isDuplicate: false, isOutOfOrder: false }; + case Verdict.Accept: + this.acceptedCount++; + return { isAccepted: true, isDuplicate: false, isOutOfOrder: false }; + } } - /** - * Get current deduplication statistics - */ + // Get statistics on deduplication performance. getStats(): DeduplicationStats { return { accepted: this.acceptedCount, deduplicated: this.deduplicatedCount, outOfOrder: this.outOfOrderCount, totalReceived: this.acceptedCount + this.deduplicatedCount, - watermarkCount: this.waterMark.size, + watermarkCount: this.feedState.size, }; } - /** - * Get watermark for a specific feed ID - */ + // Get the watermark for a feed. getWatermark(feedId: string): number | undefined { - return this.waterMark.get(feedId); - } - - /** - * Get all current watermarks (for debugging/monitoring) - */ - getAllWatermarks(): Record { - const watermarks: Record = {}; - for (const [feedId, timestamp] of this.waterMark) { - watermarks[feedId] = timestamp; - } - return watermarks; - } - - /** - * Manually set watermark for a feed (useful for initialization) - */ - setWatermark(feedId: string, timestamp: number): void { - this.waterMark.set(feedId, timestamp); - } - - /** - * Clear watermark for a specific feed - */ - clearWatermark(feedId: string): boolean { - return this.waterMark.delete(feedId); - } - - /** - * Clear all watermarks - */ - clearAllWatermarks(): void { - this.waterMark.clear(); + return this.feedState.get(feedId)?.watermark; } - /** - * Reset all counters and watermarks - */ reset(): void { this.acceptedCount = 0; this.deduplicatedCount = 0; this.outOfOrderCount = 0; - this.waterMark.clear(); + this.feedState.clear(); } - /** - * Start periodic cleanup of old watermarks - * This prevents memory leaks for feeds that are no longer active - */ private startCleanup(): void { this.cleanupInterval = setInterval(() => { this.cleanupOldWatermarks(); }, this.cleanupIntervalMs); } - /** - * Clean up watermarks that are too old - * This is a safety mechanism to prevent unbounded memory growth - */ + // Clean up old watermarks to keep memory usage under control. private cleanupOldWatermarks(): void { const now = Date.now(); - const cutoffTime = now - this.maxWatermarkAge; + const cutoffTimestamp = Math.floor((now - this.maxWatermarkAge) / 1000); - // Convert cutoff time to seconds (like the timestamps in reports) - const cutoffTimestamp = Math.floor(cutoffTime / 1000); - - let _removedCount = 0; - for (const [feedId, timestamp] of this.waterMark) { - if (timestamp < cutoffTimestamp) { - this.waterMark.delete(feedId); - _removedCount++; + for (const [feedId, state] of this.feedState) { + if (state.watermark < cutoffTimestamp) { + this.feedState.delete(feedId); } } } - /** - * Stop the deduplicator and clean up resources - */ stop(): void { if (this.cleanupInterval) { clearInterval(this.cleanupInterval); this.cleanupInterval = null; } } - - /** - * Get memory usage information - */ - getMemoryInfo(): { - watermarkCount: number; - estimatedMemoryBytes: number; - } { - const watermarkCount = this.waterMark.size; - - // Rough estimation: each entry has a string key (~64 chars) + number value - // String: ~64 bytes (feed ID) + Number: 8 bytes + Map overhead: ~32 bytes - const estimatedMemoryBytes = watermarkCount * (64 + 8 + 32); - - return { - watermarkCount, - estimatedMemoryBytes, - }; - } - - /** - * Export watermarks for persistence/debugging - */ - exportWatermarks(): Array<{ feedId: string; timestamp: number }> { - return Array.from(this.waterMark.entries()).map(([feedId, timestamp]) => ({ - feedId, - timestamp, - })); - } - - /** - * Import watermarks from external source - */ - importWatermarks(watermarks: Array<{ feedId: string; timestamp: number }>): void { - for (const { feedId, timestamp } of watermarks) { - this.waterMark.set(feedId, timestamp); - } - } } diff --git a/typescript/tests/unit/stream/deduplication.test.ts b/typescript/tests/unit/stream/deduplication.test.ts index 2ca11b5..824c77d 100644 --- a/typescript/tests/unit/stream/deduplication.test.ts +++ b/typescript/tests/unit/stream/deduplication.test.ts @@ -33,16 +33,15 @@ describe("ReportDeduplicator", () => { validFromTimestamp: 900, }; - // First report should be accepted const result1 = deduplicator.processReport(report); expect(result1.isAccepted).toBe(true); expect(result1.isDuplicate).toBe(false); - // Duplicate should be rejected const result2 = deduplicator.processReport(report); expect(result2.isAccepted).toBe(false); expect(result2.isDuplicate).toBe(true); - expect(result2.reason).toContain("watermark"); + expect(result2.isOutOfOrder).toBe(false); + expect(result2.reason).toContain("already seen"); }); it("should reject reports with older timestamps", () => { @@ -60,14 +59,12 @@ describe("ReportDeduplicator", () => { validFromTimestamp: 900, }; - // Accept newer report first const result1 = deduplicator.processReport(newerReport); expect(result1.isAccepted).toBe(true); - // Reject older report const result2 = deduplicator.processReport(olderReport); expect(result2.isAccepted).toBe(false); - expect(result2.isDuplicate).toBe(true); + expect(result2.isOutOfOrder).toBe(true); }); it("should accept reports with newer timestamps", () => { @@ -85,17 +82,71 @@ describe("ReportDeduplicator", () => { validFromTimestamp: 1900, }; - // Accept older report first const result1 = deduplicator.processReport(olderReport); expect(result1.isAccepted).toBe(true); - // Accept newer report const result2 = deduplicator.processReport(newerReport); expect(result2.isAccepted).toBe(true); expect(result2.isDuplicate).toBe(false); }); }); + describe("out-of-order with allowOutOfOrder", () => { + it("should accept out-of-order reports when allowOutOfOrder is true", () => { + const dedup = new ReportDeduplicator({ allowOutOfOrder: true }); + + dedup.processReport({ + feedID: "0x123", + observationsTimestamp: 2000, + fullReport: "newer", + validFromTimestamp: 1900, + }); + + const result = dedup.processReport({ + feedID: "0x123", + observationsTimestamp: 1000, + fullReport: "older", + validFromTimestamp: 900, + }); + + expect(result.isAccepted).toBe(true); + expect(result.isOutOfOrder).toBe(true); + expect(result.isDuplicate).toBe(false); + dedup.stop(); + }); + + it("should distinguish out-of-order from duplicate", () => { + const dedup = new ReportDeduplicator({ allowOutOfOrder: true }); + + dedup.processReport({ + feedID: "0x123", + observationsTimestamp: 200, + fullReport: "r", + validFromTimestamp: 100, + }); + + const ooo = dedup.processReport({ + feedID: "0x123", + observationsTimestamp: 100, + fullReport: "r", + validFromTimestamp: 50, + }); + expect(ooo.isAccepted).toBe(true); + expect(ooo.isOutOfOrder).toBe(true); + + const dup = dedup.processReport({ + feedID: "0x123", + observationsTimestamp: 100, + fullReport: "r", + validFromTimestamp: 50, + }); + expect(dup.isAccepted).toBe(false); + expect(dup.isDuplicate).toBe(true); + expect(dup.isOutOfOrder).toBe(false); + dedup.stop(); + }); + }); + describe("multi-feed handling", () => { it("should handle multiple feeds independently", () => { const report1: ReportMetadata = { @@ -107,19 +158,17 @@ describe("ReportDeduplicator", () => { const report2: ReportMetadata = { feedID: "0x456", - observationsTimestamp: 1000, // Same timestamp, different feed + observationsTimestamp: 1000, fullReport: "report2", validFromTimestamp: 900, }; - // Both should be accepted since they're for different feeds const result1 = deduplicator.processReport(report1); expect(result1.isAccepted).toBe(true); const result2 = deduplicator.processReport(report2); expect(result2.isAccepted).toBe(true); - // Duplicates should be rejected const result3 = deduplicator.processReport(report1); expect(result3.isAccepted).toBe(false); @@ -128,36 +177,27 @@ describe("ReportDeduplicator", () => { }); it("should track watermarks per feed independently", () => { - const feed1Report1: ReportMetadata = { + deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 1000, fullReport: "report1", validFromTimestamp: 900, - }; + }); - const feed2Report1: ReportMetadata = { + deduplicator.processReport({ feedID: "0x456", observationsTimestamp: 2000, fullReport: "report2", validFromTimestamp: 1900, - }; + }); - const feed1Report2: ReportMetadata = { + deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 1500, fullReport: "report3", validFromTimestamp: 1400, - }; - - // Accept initial reports - deduplicator.processReport(feed1Report1); - deduplicator.processReport(feed2Report1); - - // Accept newer report for feed1 - const result = deduplicator.processReport(feed1Report2); - expect(result.isAccepted).toBe(true); + }); - // Verify watermarks are independent expect(deduplicator.getWatermark("0x123")).toBe(1500); expect(deduplicator.getWatermark("0x456")).toBe(2000); }); @@ -169,114 +209,170 @@ describe("ReportDeduplicator", () => { }); it("should update watermarks correctly", () => { - const report: ReportMetadata = { + expect(deduplicator.getWatermark("0x123")).toBeUndefined(); + + deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 1500, fullReport: "report", validFromTimestamp: 1400, - }; - - expect(deduplicator.getWatermark("0x123")).toBeUndefined(); - - deduplicator.processReport(report); + }); expect(deduplicator.getWatermark("0x123")).toBe(1500); }); - it("should not update watermark for rejected reports", () => { - const report1: ReportMetadata = { + it("should not update watermark for out-of-order reports", () => { + deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 2000, fullReport: "report1", validFromTimestamp: 1900, - }; + }); + expect(deduplicator.getWatermark("0x123")).toBe(2000); - const report2: ReportMetadata = { + deduplicator.processReport({ feedID: "0x123", - observationsTimestamp: 1000, // Older + observationsTimestamp: 1000, fullReport: "report2", validFromTimestamp: 900, - }; - - // Accept newer report - deduplicator.processReport(report1); + }); expect(deduplicator.getWatermark("0x123")).toBe(2000); - - // Reject older report - const result = deduplicator.processReport(report2); - expect(result.isAccepted).toBe(false); - expect(deduplicator.getWatermark("0x123")).toBe(2000); // Should remain unchanged }); + }); - it("should allow manual watermark setting", () => { - deduplicator.setWatermark("0x123", 5000); - expect(deduplicator.getWatermark("0x123")).toBe(5000); + describe("FIFO eviction", () => { + it("should evict oldest-inserted entry when buffer is full", () => { + for (let i = 1; i <= 32; i++) { + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: i, + fullReport: "r", + validFromTimestamp: i - 1, + }); + } - const report: ReportMetadata = { + // ts=2 (second inserted) should still be in the buffer + const stillPresent = deduplicator.processReport({ feedID: "0x123", - observationsTimestamp: 3000, // Lower than manual watermark - fullReport: "report", - validFromTimestamp: 2900, - }; + observationsTimestamp: 2, + fullReport: "r", + validFromTimestamp: 1, + }); + expect(stillPresent.isDuplicate).toBe(true); - const result = deduplicator.processReport(report); - expect(result.isAccepted).toBe(false); + // Adding ts=33 evicts ts=1 (oldest inserted) + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 33, + fullReport: "r", + validFromTimestamp: 32, + }); + + // ts=1 was evicted, so it's no longer a duplicate + // (calling processReport re-inserts it, which evicts ts=3 as next oldest) + const evicted = deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 1, + fullReport: "r", + validFromTimestamp: 0, + }); + expect(evicted.isDuplicate).toBe(false); }); - it("should clear specific watermarks", () => { - deduplicator.setWatermark("0x123", 1000); - deduplicator.setWatermark("0x456", 2000); + it("should evict by insertion order, not by smallest value", () => { + // Insert 100 first, then 1..31 (total 32 entries) + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 100, + fullReport: "r", + validFromTimestamp: 99, + }); + for (let i = 1; i <= 31; i++) { + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: i, + fullReport: "r", + validFromTimestamp: i - 1, + }); + } - expect(deduplicator.getWatermark("0x123")).toBe(1000); - expect(deduplicator.getWatermark("0x456")).toBe(2000); + // Add 999 -> should evict 100 (oldest inserted), NOT 1 (smallest value) + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 999, + fullReport: "r", + validFromTimestamp: 998, + }); - const cleared = deduplicator.clearWatermark("0x123"); - expect(cleared).toBe(true); - expect(deduplicator.getWatermark("0x123")).toBeUndefined(); - expect(deduplicator.getWatermark("0x456")).toBe(2000); + // ts=1 should still be present + const smallestStillPresent = deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 1, + fullReport: "r", + validFromTimestamp: 0, + }); + expect(smallestStillPresent.isDuplicate).toBe(true); - const alreadyCleared = deduplicator.clearWatermark("0x123"); - expect(alreadyCleared).toBe(false); + // ts=100 should have been evicted + const oldestEvicted = deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 100, + fullReport: "r", + validFromTimestamp: 99, + }); + expect(oldestEvicted.isDuplicate).toBe(false); }); + }); - it("should clear all watermarks", () => { - deduplicator.setWatermark("0x123", 1000); - deduplicator.setWatermark("0x456", 2000); + describe("HA duplicate detection", () => { + it("should detect HA duplicate after watermark advance", () => { + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 100, + fullReport: "r", + validFromTimestamp: 99, + }); - deduplicator.clearAllWatermarks(); + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 200, + fullReport: "r", + validFromTimestamp: 199, + }); - expect(deduplicator.getWatermark("0x123")).toBeUndefined(); - expect(deduplicator.getWatermark("0x456")).toBeUndefined(); + // HA duplicate of ts=100 from second connection + const result = deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 100, + fullReport: "r", + validFromTimestamp: 99, + }); + expect(result.isDuplicate).toBe(true); + expect(result.isOutOfOrder).toBe(false); + expect(result.isAccepted).toBe(false); }); }); describe("statistics tracking", () => { it("should track statistics correctly", () => { - const report1: ReportMetadata = { + deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 1000, fullReport: "report1", validFromTimestamp: 900, - }; - - const report2: ReportMetadata = { + }); + deduplicator.processReport({ feedID: "0x123", - observationsTimestamp: 1000, // Duplicate + observationsTimestamp: 1000, fullReport: "report2", validFromTimestamp: 900, - }; - - const report3: ReportMetadata = { + }); + deduplicator.processReport({ feedID: "0x456", observationsTimestamp: 2000, fullReport: "report3", validFromTimestamp: 1900, - }; - - // Process reports - deduplicator.processReport(report1); // Accepted - deduplicator.processReport(report2); // Deduplicated - deduplicator.processReport(report3); // Accepted + }); const stats = deduplicator.getStats(); expect(stats.accepted).toBe(2); @@ -286,15 +382,18 @@ describe("ReportDeduplicator", () => { }); it("should reset statistics", () => { - const report: ReportMetadata = { + deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 1000, fullReport: "report", validFromTimestamp: 900, - }; - - deduplicator.processReport(report); - deduplicator.processReport(report); // Duplicate + }); + deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 1000, + fullReport: "report", + validFromTimestamp: 900, + }); let stats = deduplicator.getStats(); expect(stats.accepted).toBe(1); @@ -312,28 +411,23 @@ describe("ReportDeduplicator", () => { describe("memory management", () => { it("should handle large numbers of feeds efficiently", () => { - const feedCount = 1000; // Reduced for test performance + const feedCount = 1000; const feeds: string[] = []; - // Generate many unique feed IDs for (let i = 0; i < feedCount; i++) { feeds.push(`0x${i.toString(16).padStart(64, "0")}`); } - // Add reports for all feeds feeds.forEach((feedID, index) => { - const report: ReportMetadata = { + const result = deduplicator.processReport({ feedID, observationsTimestamp: index + 1000, fullReport: `report-${index}`, validFromTimestamp: index + 900, - }; - - const result = deduplicator.processReport(report); + }); expect(result.isAccepted).toBe(true); }); - // Verify all watermarks are set correctly feeds.forEach((feedID, index) => { expect(deduplicator.getWatermark(feedID)).toBe(index + 1000); }); @@ -341,167 +435,64 @@ describe("ReportDeduplicator", () => { const stats = deduplicator.getStats(); expect(stats.watermarkCount).toBe(feedCount); }); - - it("should provide memory usage information", () => { - const report: ReportMetadata = { - feedID: "0x123", - observationsTimestamp: 1000, - fullReport: "report", - validFromTimestamp: 900, - }; - - deduplicator.processReport(report); - - const memoryInfo = deduplicator.getMemoryInfo(); - expect(memoryInfo.watermarkCount).toBe(1); - expect(memoryInfo.estimatedMemoryBytes).toBeGreaterThan(0); - }); }); describe("edge cases", () => { it("should handle zero timestamp", () => { - const report: ReportMetadata = { + const result = deduplicator.processReport({ feedID: "0x123", observationsTimestamp: 0, fullReport: "report", validFromTimestamp: 0, - }; - - const result = deduplicator.processReport(report); + }); expect(result.isAccepted).toBe(true); expect(deduplicator.getWatermark("0x123")).toBe(0); - // Should reject duplicate with same zero timestamp - const result2 = deduplicator.processReport(report); + const result2 = deduplicator.processReport({ + feedID: "0x123", + observationsTimestamp: 0, + fullReport: "report", + validFromTimestamp: 0, + }); expect(result2.isAccepted).toBe(false); }); it("should handle very large timestamps", () => { const largeTimestamp = Number.MAX_SAFE_INTEGER; - const report: ReportMetadata = { + const result = deduplicator.processReport({ feedID: "0x123", observationsTimestamp: largeTimestamp, fullReport: "report", validFromTimestamp: largeTimestamp - 1, - }; - - const result = deduplicator.processReport(report); + }); expect(result.isAccepted).toBe(true); expect(deduplicator.getWatermark("0x123")).toBe(largeTimestamp); }); it("should handle empty feed ID", () => { - const report: ReportMetadata = { + const result = deduplicator.processReport({ feedID: "", observationsTimestamp: 1000, fullReport: "report", validFromTimestamp: 900, - }; - - const result = deduplicator.processReport(report); + }); expect(result.isAccepted).toBe(true); expect(deduplicator.getWatermark("")).toBe(1000); }); it("should handle special characters in feed ID", () => { const specialFeedId = "0x!@#$%^&*()_+-=[]{}|;:,.<>?"; - const report: ReportMetadata = { + const result = deduplicator.processReport({ feedID: specialFeedId, observationsTimestamp: 1000, fullReport: "report", validFromTimestamp: 900, - }; - - const result = deduplicator.processReport(report); + }); expect(result.isAccepted).toBe(true); expect(deduplicator.getWatermark(specialFeedId)).toBe(1000); }); }); - describe("export/import functionality", () => { - it("should export watermarks correctly", () => { - const reports = [ - { - feedID: "0x123", - observationsTimestamp: 1000, - fullReport: "report1", - validFromTimestamp: 900, - }, - { - feedID: "0x456", - observationsTimestamp: 2000, - fullReport: "report2", - validFromTimestamp: 1900, - }, - ]; - - reports.forEach(report => { - deduplicator.processReport(report as ReportMetadata); - }); - - const exported = deduplicator.exportWatermarks(); - expect(exported).toHaveLength(2); - expect(exported).toContainEqual({ feedId: "0x123", timestamp: 1000 }); - expect(exported).toContainEqual({ feedId: "0x456", timestamp: 2000 }); - }); - - it("should import watermarks correctly", () => { - const watermarks = [ - { feedId: "0x123", timestamp: 1500 }, - { feedId: "0x456", timestamp: 2500 }, - { feedId: "0x789", timestamp: 3500 }, - ]; - - deduplicator.importWatermarks(watermarks); - - expect(deduplicator.getWatermark("0x123")).toBe(1500); - expect(deduplicator.getWatermark("0x456")).toBe(2500); - expect(deduplicator.getWatermark("0x789")).toBe(3500); - - const stats = deduplicator.getStats(); - expect(stats.watermarkCount).toBe(3); - }); - - it("should handle empty export", () => { - const exported = deduplicator.exportWatermarks(); - expect(exported).toEqual([]); - }); - - it("should handle empty import", () => { - deduplicator.importWatermarks([]); - const stats = deduplicator.getStats(); - expect(stats.watermarkCount).toBe(0); - }); - - it("should overwrite existing watermarks on import", () => { - // Set initial watermark - deduplicator.setWatermark("0x123", 1000); - expect(deduplicator.getWatermark("0x123")).toBe(1000); - - // Import should overwrite - deduplicator.importWatermarks([{ feedId: "0x123", timestamp: 2000 }]); - expect(deduplicator.getWatermark("0x123")).toBe(2000); - }); - }); - - describe("watermark access", () => { - it("should get all watermarks", () => { - deduplicator.setWatermark("0x123", 1000); - deduplicator.setWatermark("0x456", 2000); - - const allWatermarks = deduplicator.getAllWatermarks(); - expect(allWatermarks).toEqual({ - "0x123": 1000, - "0x456": 2000, - }); - }); - - it("should return empty object when no watermarks exist", () => { - const allWatermarks = deduplicator.getAllWatermarks(); - expect(allWatermarks).toEqual({}); - }); - }); - describe("cleanup functionality", () => { it("should initialize with cleanup enabled", () => { const dedup = new ReportDeduplicator({ From 00916e8590101fedb5492e3896174ab6a33eb66b Mon Sep 17 00:00:00 2001 From: cal Date: Fri, 10 Apr 2026 13:32:34 +1000 Subject: [PATCH 6/9] add todo comment for ms support in s sdk --- typescript/src/stream/deduplication.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/typescript/src/stream/deduplication.ts b/typescript/src/stream/deduplication.ts index 0060e1a..97838e8 100644 --- a/typescript/src/stream/deduplication.ts +++ b/typescript/src/stream/deduplication.ts @@ -165,6 +165,8 @@ export class ReportDeduplicator { // Clean up old watermarks to keep memory usage under control. private cleanupOldWatermarks(): void { const now = Date.now(); + // todo: this assumes that the timestamp is in seconds, introduce millisecond support when sdk is + // updated to handle millisecond timestamps. const cutoffTimestamp = Math.floor((now - this.maxWatermarkAge) / 1000); for (const [feedId, state] of this.feedState) { From 2ee318d23e42aa352cfa2fc597a7f36f836ce1a7 Mon Sep 17 00:00:00 2001 From: cal <78729586+calvwang9@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:01:44 +1000 Subject: [PATCH 7/9] drop nil, add test for dropping duplicate reports --- go/stream.go | 4 ++ go/stream_test.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/go/stream.go b/go/stream.go index 81fe025..8287511 100644 --- a/go/stream.go +++ b/go/stream.go @@ -361,6 +361,10 @@ func (s *stream) Close() (err error) { } func (s *stream) accept(ctx context.Context, m *message) (err error) { + if m.Report == nil { + return nil + } + id := m.Report.FeedID.String() ts := m.Report.ObservationsTimestamp.UnixMilli() diff --git a/go/stream_test.go b/go/stream_test.go index 071d389..060d09f 100644 --- a/go/stream_test.go +++ b/go/stream_test.go @@ -958,6 +958,157 @@ func TestClient_StreamOutOfOrder_DefaultDrop(t *testing.T) { } } +func TestClient_StreamDuplicateAlwaysDropped(t *testing.T) { + // Exact timestamp duplicates must be dropped regardless of WsAllowOutOfOrder. + reports := []*ReportResponse{ + {FeedID: feed1, ObservationsTimestamp: time.Unix(100, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(100, 0)}, // exact duplicate + } + + ms := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + return + } + + conn, err := websocket.Accept( + w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionContextTakeover}, + ) + if err != nil { + t.Fatalf("error accepting connection: %s", err) + } + defer func() { _ = conn.CloseNow() }() + + for _, rpt := range reports { + b, err := json.Marshal(&message{rpt}) + if err != nil { + t.Errorf("failed to serialize message: %s", err) + } + if err := conn.Write(context.Background(), websocket.MessageBinary, b); err != nil { + t.Errorf("failed to write message: %s", err) + } + } + + for conn.Ping(context.Background()) == nil { + time.Sleep(100 * time.Millisecond) + } + }) + defer ms.Close() + + streamsClient, err := ms.Client() + if err != nil { + t.Fatalf("error creating client %s", err) + } + + cc := streamsClient.(*client) + cc.config.WsAllowOutOfOrder = true // duplicate must still be dropped even with flag set + + sub, err := streamsClient.Stream(context.Background(), []feed.ID{feed1}) + if err != nil { + t.Fatalf("error subscribing %s", err) + } + defer sub.Close() + + rep, err := sub.Read(context.Background()) + if err != nil { + t.Fatalf("error reading report %s", err) + } + if !rep.ObservationsTimestamp.Equal(time.Unix(100, 0)) { + t.Errorf("unexpected report timestamp: %v", rep.ObservationsTimestamp) + } + + time.Sleep(50 * time.Millisecond) + + stats := sub.Stats() + if stats.Accepted != 1 { + t.Errorf("stats.Accepted = %d, want 1", stats.Accepted) + } + if stats.Deduplicated != 1 { + t.Errorf("stats.Deduplicated = %d, want 1", stats.Deduplicated) + } + if stats.OutOfOrder != 0 { + t.Errorf("stats.OutOfOrder = %d, want 0", stats.OutOfOrder) + } +} + +func TestClient_StreamMultipleOutOfOrder_Allowed(t *testing.T) { + // With WsAllowOutOfOrder=true, every report is delivered even when several + // arrive out of order after the watermark advances. + reports := []*ReportResponse{ + {FeedID: feed1, ObservationsTimestamp: time.Unix(100, 0)}, + {FeedID: feed1, ObservationsTimestamp: time.Unix(105, 0)}, // advances watermark to 105 + {FeedID: feed1, ObservationsTimestamp: time.Unix(103, 0)}, // OOO + {FeedID: feed1, ObservationsTimestamp: time.Unix(102, 0)}, // OOO + {FeedID: feed1, ObservationsTimestamp: time.Unix(104, 0)}, // OOO + } + + ms := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + return + } + + conn, err := websocket.Accept( + w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionContextTakeover}, + ) + if err != nil { + t.Fatalf("error accepting connection: %s", err) + } + defer func() { _ = conn.CloseNow() }() + + for _, rpt := range reports { + b, err := json.Marshal(&message{rpt}) + if err != nil { + t.Errorf("failed to serialize message: %s", err) + } + if err := conn.Write(context.Background(), websocket.MessageBinary, b); err != nil { + t.Errorf("failed to write message: %s", err) + } + } + + for conn.Ping(context.Background()) == nil { + time.Sleep(100 * time.Millisecond) + } + }) + defer ms.Close() + + streamsClient, err := ms.Client() + if err != nil { + t.Fatalf("error creating client %s", err) + } + + cc := streamsClient.(*client) + cc.config.WsAllowOutOfOrder = true + + sub, err := streamsClient.Stream(context.Background(), []feed.ID{feed1}) + if err != nil { + t.Fatalf("error subscribing %s", err) + } + defer sub.Close() + + var received []*ReportResponse + for i := 0; i < len(reports); i++ { + rep, err := sub.Read(context.Background()) + if err != nil { + t.Fatalf("error reading report %s", err) + } + received = append(received, rep) + } + + if !reportResponsesEqual(received, reports) { + t.Errorf("Read() = %v, want %v", received, reports) + } + + stats := sub.Stats() + if stats.Accepted != 5 { + t.Errorf("stats.Accepted = %d, want 5", stats.Accepted) + } + if stats.OutOfOrder != 3 { + t.Errorf("stats.OutOfOrder = %d, want 3", stats.OutOfOrder) + } + if stats.Deduplicated != 0 { + t.Errorf("stats.Deduplicated = %d, want 0", stats.Deduplicated) + } +} + // Tests that when in HA mode both origins are up after a recovery period even if one origin is down on initial connection func TestClient_StreamHA_OneOriginDownRecovery(t *testing.T) { connectAttempts := &atomic.Uint64{} From f64475f6d14d2a45b195124ac3ddf9ec741ff424 Mon Sep 17 00:00:00 2001 From: cal <78729586+calvwang9@users.noreply.github.com> Date: Wed, 15 Apr 2026 18:08:07 +1000 Subject: [PATCH 8/9] move mutex into dedup, reorder watermark update --- go/dedup.go | 13 ++++++++++--- go/stream.go | 5 +---- rust/crates/sdk/src/stream/dedup.rs | 12 ++++++------ typescript/src/stream/deduplication.ts | 9 +++++---- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/go/dedup.go b/go/dedup.go index fc9bda8..e2d8d2d 100644 --- a/go/dedup.go +++ b/go/dedup.go @@ -1,5 +1,7 @@ package streams +import "sync" + const seenBufferSize = 32 type Verdict int @@ -19,6 +21,7 @@ type feedState struct { } type FeedDeduplicator struct { + mu sync.Mutex feeds map[string]*feedState } @@ -27,6 +30,9 @@ func NewFeedDeduplicator() *FeedDeduplicator { } func (d *FeedDeduplicator) Check(feedID string, ts int64) Verdict { + d.mu.Lock() + defer d.mu.Unlock() + fs := d.feeds[feedID] if fs == nil { fs = &feedState{set: make(map[int64]struct{}, seenBufferSize)} @@ -48,12 +54,13 @@ func (d *FeedDeduplicator) Check(feedID string, ts int64) Verdict { fs.cursor = (fs.cursor + 1) % seenBufferSize isOutOfOrder := fs.watermark > 0 && ts < fs.watermark + if isOutOfOrder { + return OutOfOrder + } + if ts > fs.watermark { fs.watermark = ts } - if isOutOfOrder { - return OutOfOrder - } return Accept } diff --git a/go/stream.go b/go/stream.go index 8287511..a3c759f 100644 --- a/go/stream.go +++ b/go/stream.go @@ -83,8 +83,7 @@ type stream struct { closeError atomic.Value connStatusCallback func(isConneccted bool, host string, origin string) - feedsMu sync.Mutex - dedup *FeedDeduplicator + dedup *FeedDeduplicator stats struct { accepted atomic.Uint64 @@ -368,9 +367,7 @@ func (s *stream) accept(ctx context.Context, m *message) (err error) { id := m.Report.FeedID.String() ts := m.Report.ObservationsTimestamp.UnixMilli() - s.feedsMu.Lock() verdict := s.dedup.Check(id, ts) - s.feedsMu.Unlock() switch verdict { case Duplicate: diff --git a/rust/crates/sdk/src/stream/dedup.rs b/rust/crates/sdk/src/stream/dedup.rs index 2c16a71..967d9f0 100644 --- a/rust/crates/sdk/src/stream/dedup.rs +++ b/rust/crates/sdk/src/stream/dedup.rs @@ -61,15 +61,15 @@ impl FeedDeduplicator { fs.cursor = (fs.cursor + 1) % SEEN_BUFFER_SIZE; let is_out_of_order = fs.watermark > 0 && ts < fs.watermark; - if ts > fs.watermark { - fs.watermark = ts; + if is_out_of_order { + return Verdict::OutOfOrder } - if is_out_of_order { - Verdict::OutOfOrder - } else { - Verdict::Accept + if ts > fs.watermark { + fs.watermark = ts; } + + return Verdict::Accept; } } diff --git a/typescript/src/stream/deduplication.ts b/typescript/src/stream/deduplication.ts index 97838e8..e0194be 100644 --- a/typescript/src/stream/deduplication.ts +++ b/typescript/src/stream/deduplication.ts @@ -86,13 +86,14 @@ export class ReportDeduplicator { state.seen.add(ts); const isOutOfOrder = state.watermark > 0 && ts < state.watermark; - if (ts > state.watermark) { - state.watermark = ts; - } - if (isOutOfOrder) { return Verdict.OutOfOrder; } + + if (ts > state.watermark) { + state.watermark = ts; + } + return Verdict.Accept; } From 2ffd82afdb84a71259db0ce4f4cf6e200b7bcd84 Mon Sep 17 00:00:00 2001 From: cal <78729586+calvwang9@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:11:45 +1000 Subject: [PATCH 9/9] fix: avoid nil stream in HA retry goroutine after named return --- go/stream.go | 7 ++++--- go/stream_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/go/stream.go b/go/stream.go index a3c759f..7a4389a 100644 --- a/go/stream.go +++ b/go/stream.go @@ -132,13 +132,14 @@ func (c *client) newStream(ctx context.Context, httpClient *http.Client, feedIDs c.config.logInfo("client: failed to connect to origin %s: %s", origins[x], err) errs = append(errs, fmt.Errorf("origin %s: %w", origins[x], err)) // Retry connecting to the origin in the background + localS := s // stable *stream for retry goroutine (named return s would become nil) go func() { - conn, err := s.newWSconnWithRetry(origins[x]) + conn, err := localS.newWSconnWithRetry(origins[x]) if err != nil { return } - go s.monitorConn(conn) - s.conns = append(s.conns, conn) + go localS.monitorConn(conn) + localS.conns = append(localS.conns, conn) }() continue } diff --git a/go/stream_test.go b/go/stream_test.go index 060d09f..20c93bc 100644 --- a/go/stream_test.go +++ b/go/stream_test.go @@ -801,6 +801,41 @@ func TestClient_StreamHA_OneOriginDown(t *testing.T) { } +// TestClient_StreamHA_AllOriginsFailInitialConnect covers HA mode when every origin +// rejects the WebSocket: newStream spawns retry goroutines then returns an error. +// Those goroutines must not close over the named result *stream (which becomes nil +// on return), or the first newWSconnWithRetry iteration panics on s.closed.Load(). +func TestClient_StreamHA_AllOriginsFailInitialConnect(t *testing.T) { + ms := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + w.Header().Add(cllAvailOriginsHeader, "{001,002}") + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Path != apiV2WS { + return + } + w.WriteHeader(http.StatusForbidden) + }) + defer ms.Close() + + streamsClient, err := ms.Client() + if err != nil { + t.Fatalf("error creating client %s", err) + } + + cc := streamsClient.(*client) + cc.config.WsHA = true + + _, err = streamsClient.Stream(context.Background(), []feed.ID{feed1}) + if err == nil { + t.Fatal("expected Stream error when every origin rejects the websocket") + } + + // Allow retry goroutines to run; a nil *stream receiver would fault immediately. + time.Sleep(300 * time.Millisecond) +} + func TestClient_StreamOutOfOrder(t *testing.T) { reports := []*ReportResponse{ {FeedID: feed1, ObservationsTimestamp: time.Unix(100, 0)},