From b41de4cf49478b52e1ba02bbe09444eefc6ee1c5 Mon Sep 17 00:00:00 2001 From: Karim <98668332+khadni@users.noreply.github.com> Date: Mon, 29 Apr 2024 10:31:24 +0200 Subject: [PATCH 1/3] Update DS SD guides --- .../DataStreams/StreamsDirect/client.go | 170 ------------------ .../DataStreams/StreamsDirect/decoder.go | 129 ------------- .../samples/DataStreams/StreamsDirect/mod.go | 24 --- .../StreamsDirect/multipleFeeds.go | 77 -------- .../DataStreams/StreamsDirect/singleFeed.go | 73 -------- .../DataStreams/StreamsDirect/ws/clientWs.go | 168 ----------------- .../StreamsDirect/ws/decodeFullReportHex.go | 40 ----- .../DataStreams/StreamsDirect/ws/mod.go | 25 --- .../StreamsDirect/ws/wsConnectAndListen.go | 25 --- .../tutorials/streams-direct-api.mdx | 91 +++------- .../tutorials/streams-direct-ws.mdx | 102 ++++------- .../common/gettingStartedHardhat.mdx | 1 - 12 files changed, 51 insertions(+), 874 deletions(-) delete mode 100644 public/samples/DataStreams/StreamsDirect/client.go delete mode 100644 public/samples/DataStreams/StreamsDirect/decoder.go delete mode 100644 public/samples/DataStreams/StreamsDirect/mod.go delete mode 100644 public/samples/DataStreams/StreamsDirect/multipleFeeds.go delete mode 100644 public/samples/DataStreams/StreamsDirect/singleFeed.go delete mode 100644 public/samples/DataStreams/StreamsDirect/ws/clientWs.go delete mode 100644 public/samples/DataStreams/StreamsDirect/ws/decodeFullReportHex.go delete mode 100644 public/samples/DataStreams/StreamsDirect/ws/mod.go delete mode 100644 public/samples/DataStreams/StreamsDirect/ws/wsConnectAndListen.go diff --git a/public/samples/DataStreams/StreamsDirect/client.go b/public/samples/DataStreams/StreamsDirect/client.go deleted file mode 100644 index cd89530d03a..00000000000 --- a/public/samples/DataStreams/StreamsDirect/client.go +++ /dev/null @@ -1,170 +0,0 @@ -// client.go - -package client - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/ethereum/go-ethereum/common/hexutil" -) - -type SingleReport struct { - FeedID hexutil.Bytes `json:"feedID"` - ValidFromTimestamp uint32 `json:"validFromTimestamp"` - ObservationsTimestamp uint32 `json:"observationsTimestamp"` - FullReport hexutil.Bytes `json:"fullReport"` -} - -type SingleReportResponse struct { - Report SingleReport `json:"report"` -} - -type BulkReportResponse struct { - Reports []SingleReport `json:"reports"` -} - -const ( - path = "/api/v1/reports" - bulkPath = "/api/v1/reports/bulk" -) - -func GenerateHMAC(method string, path string, body []byte, clientId string, timestamp int64, userSecret string) string { - serverBodyHash := sha256.New() - serverBodyHash.Write(body) - serverBodyHashString := fmt.Sprintf("%s %s %s %s %d", - method, - path, - hex.EncodeToString(serverBodyHash.Sum(nil)), - clientId, - timestamp) - fmt.Println("Generating HMAC with the following: ", serverBodyHashString) - signedMessage := hmac.New(sha256.New, []byte(userSecret)) - signedMessage.Write([]byte(serverBodyHashString)) - userHmac := hex.EncodeToString(signedMessage.Sum(nil)) - return userHmac -} - -func GenerateAuthHeaders(method string, pathAndParams string, clientId string, userSecret string) http.Header { - header := http.Header{} - timestamp := time.Now().UTC().UnixMilli() - hmacString := GenerateHMAC(method, pathAndParams, []byte(""), clientId, timestamp, userSecret) - - header.Add("Authorization", clientId) - header.Add("X-Authorization-Timestamp", strconv.FormatInt(timestamp, 10)) - header.Add("X-Authorization-Signature-SHA256", hmacString) - return header -} - -func FetchSingleReportSingleFeed(feedId string) (SingleReport, error) { - baseUrl := os.Getenv("BASE_URL") // Example: api.testnet-dataengine.chain.link - clientId := os.Getenv("CLIENT_ID") // Example: "00000000-0000-0000-0000-000000000000" - userSecret := os.Getenv("CLIENT_SECRET") // Example: "your-secret" - - timestamp := time.Now().UTC().UnixMilli() - 500 - - params := url.Values{ - "feedID": {feedId}, - "timestamp": {fmt.Sprintf("%d", timestamp/1000)}, - } - - req := &http.Request{ - Method: http.MethodGet, - URL: &url.URL{ - Scheme: "https", - Host: baseUrl, - Path: path, - RawQuery: params.Encode(), - }, - } - req.Header = GenerateAuthHeaders(req.Method, req.URL.RequestURI(), clientId, userSecret) - fmt.Println("base: ", baseUrl) - fmt.Println("header: ", req.Header) - fmt.Println("params: ", params) - - rawRes, err := http.DefaultClient.Do(req) - if err != nil { - return SingleReport{}, err - } - defer rawRes.Body.Close() - - body, err := io.ReadAll(rawRes.Body) - if err != nil { - return SingleReport{}, err - } - - if rawRes.StatusCode != http.StatusOK { - // Error messages are typically descriptive - return SingleReport{}, fmt.Errorf("unexpected status code %d: %v", rawRes.StatusCode, string(body)) - } - - var res SingleReportResponse - err = json.Unmarshal(body, &res) - if err != nil { - return SingleReport{}, err - } - - return res.Report, nil -} - -func FetchSingleReportManyFeeds(feedIds []string) ([]SingleReport, error) { - baseUrl := os.Getenv("BASE_URL") //Example: api.testnet-dataengine.chain.link - clientId := os.Getenv("CLIENT_ID") // Example: "00000000-0000-0000-0000-000000000000" - userSecret := os.Getenv("CLIENT_SECRET") // Example: "your-secret" - - timestamp := time.Now().UTC().UnixMilli() - 500 - - params := url.Values{ - "feedIDs": {strings.Join(feedIds, ",")}, - "timestamp": {fmt.Sprintf("%d", timestamp/1000)}, - } - - req := &http.Request{ - Method: http.MethodGet, - URL: &url.URL{ - Scheme: "https", - Host: baseUrl, - Path: bulkPath, - RawQuery: params.Encode(), - }, - } - - req.Header = GenerateAuthHeaders(req.Method, req.URL.RequestURI(), clientId, userSecret) - fmt.Println("base: ", baseUrl) - fmt.Println("header: ", req.Header) - fmt.Println("params: ", params) - - rawRes, err := http.DefaultClient.Do(req) - if err != nil { - return []SingleReport{}, err - } - defer rawRes.Body.Close() - - body, err := io.ReadAll(rawRes.Body) - if err != nil { - return []SingleReport{}, err - } - - if rawRes.StatusCode != http.StatusOK { - // Error messages are typically descriptive - return []SingleReport{}, fmt.Errorf("unexpected status code %d: %v", rawRes.StatusCode, string(body)) - } - - var res BulkReportResponse - err = json.Unmarshal(body, &res) - if err != nil { - return []SingleReport{}, err - } - - return res.Reports, nil -} \ No newline at end of file diff --git a/public/samples/DataStreams/StreamsDirect/decoder.go b/public/samples/DataStreams/StreamsDirect/decoder.go deleted file mode 100644 index dcdbe6887ad..00000000000 --- a/public/samples/DataStreams/StreamsDirect/decoder.go +++ /dev/null @@ -1,129 +0,0 @@ -// decoder.go - -package internal - -import ( - "encoding/binary" - "fmt" - - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/pkg/errors" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - v3report "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/types" -) - -type ReportWithContext struct { - FeedId mercuryutils.FeedID - FeedVersion mercuryutils.FeedVersion - V3Report *v3report.Report - Round uint8 - Epoch uint32 - Digest []byte -} - -type FullReport struct { - ReportContext [3][32]byte - ReportBlob []byte - RawRs [][32]byte - RawSs [][32]byte - RawVs [32]byte -} - -func mustNewType(t string) abi.Type { - result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) - if err != nil { - panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) - } - return result -} - -var schema = abi.Arguments{ - {Name: "reportContext", Type: mustNewType("bytes32[3]")}, - {Name: "reportBlob", Type: mustNewType("bytes")}, - {Name: "rawRs", Type: mustNewType("bytes32[]")}, - {Name: "rawSs", Type: mustNewType("bytes32[]")}, - {Name: "rawVs", Type: mustNewType("bytes32")}, -} - -/* - DecodeFullReport reads the "fullReport" from the API response into a struct containing the report context, report data, - and raw signatures. This functions requires no prep to use, because the schema for the "fullReport" blob is - common among all report versions (basic, premium, etc), -*/ - -func DecodeFullReport(fullReport []byte) (*FullReport, error) { - values, err := schema.Unpack(fullReport) - if err != nil { - return nil, fmt.Errorf("failed to decode FullReport: %w", err) - } - decoded := new(FullReport) - if err = schema.Copy(decoded, values); err != nil { - return nil, fmt.Errorf("failed to copy FullReport values to struct: %w", err) - } - - return decoded, nil -} - -/* -DecodeReportData takes the report blob (FullReport.ReportBlob), extracts the feeds id, calculates the version from the feed id, -and finally decodes the report blob using the lib that correlates with the version. The resulting interface can be cast into -the correct report type as needed. -*/ -func DecodeReportData(reportBlob []byte) (mercuryutils.FeedID, interface{}, error) { - feedIdAbi := abi.Arguments{ - {Name: "feedId", Type: mustNewType("bytes32")}, - } - reportElements := map[string]interface{}{} - if err := feedIdAbi.UnpackIntoMap(reportElements, reportBlob); err != nil { - return mercuryutils.FeedID{}, nil, err - } - feedIdInterface, ok := reportElements["feedId"] - if !ok { - return mercuryutils.FeedID{}, nil, errors.Errorf("unpacked ReportBlob has no 'feedId'") - } - feedIdBytes, ok := feedIdInterface.([32]byte) - if !ok { - return mercuryutils.FeedID{}, nil, errors.Errorf("cannot cast ReportBlob feedId to [32]byte, type is %T", feedIdBytes) - } - feedID := mercuryutils.FeedID(feedIdBytes) - - switch feedID.Version() { - case mercuryutils.REPORT_V3: - res, err := v3report.Decode(reportBlob) - return feedID, res, err - default: - return mercuryutils.FeedID{}, nil, errors.Errorf("unknown report version %d", feedID.Version()) - } -} - -/* -DecodeFullReportAndReportData takes the full report payload, decodes the fullReport blob, and then decodes the report data. -*/ -func DecodeFullReportAndReportData(payload []byte) (*ReportWithContext, error) { - fullReport, err := DecodeFullReport(payload) - if err != nil { - return nil, err - } - - feedID, report, err := DecodeReportData(fullReport.ReportBlob) - if err != nil { - return nil, err - } - - result := &ReportWithContext{ - FeedId: feedID, - FeedVersion: feedID.Version(), - Digest: fullReport.ReportContext[0][:], - Round: fullReport.ReportContext[1][31], - Epoch: binary.BigEndian.Uint32(fullReport.ReportContext[1][32-5 : 32-1]), - } - - switch feedID.Version() { - case mercuryutils.REPORT_V3: - result.V3Report = report.(*v3report.Report) - default: - return nil, errors.Errorf("unknown report version %d", feedID.Version()) - } - - return result, nil -} \ No newline at end of file diff --git a/public/samples/DataStreams/StreamsDirect/mod.go b/public/samples/DataStreams/StreamsDirect/mod.go deleted file mode 100644 index 89c5247a492..00000000000 --- a/public/samples/DataStreams/StreamsDirect/mod.go +++ /dev/null @@ -1,24 +0,0 @@ -module data-streams-direct - -go 1.21 - -require ( - github.com/ethereum/go-ethereum v1.12.2 // Ethereum blockchain interaction library - github.com/pkg/errors v0.9.1 // Library for handling errors - github.com/smartcontractkit/chainlink/v2 v2.2.1-0.20230823171354-1ead9ee6f6bb // Chainlink core components library -) - -replace ( - // Resolves version mismatch between cosmosSDK and hdevalence/ed25519consensus - filippo.io/edwards25519 => filippo.io/edwards25519 v1.0.0-rc.1 - - // Adds ARM support by updating CosmWasm to v1.2.4 - github.com/CosmWasm/wasmvm => github.com/CosmWasm/wasmvm v1.2.4 - - //// Fix go mod tidy issue for ambiguous imports from go-ethereum - //// See https://github.com/ugorji/go/issues/279 - github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.22.1 - - // Aligns protobuf version with cosmos SDK requirements - github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 -) \ No newline at end of file diff --git a/public/samples/DataStreams/StreamsDirect/multipleFeeds.go b/public/samples/DataStreams/StreamsDirect/multipleFeeds.go deleted file mode 100644 index 19c03dc332a..00000000000 --- a/public/samples/DataStreams/StreamsDirect/multipleFeeds.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "data-streams-direct/client" - "data-streams-direct/internal" - "fmt" - "log" - "os" -) - -func main() { - // Ensure the correct number of arguments are provided - if len(os.Args) < 2 { - log.Fatalf("Usage: %s ...", os.Args[0]) - } - - // Retrieve the feed IDs from command line arguments - feedIds := os.Args[1:] - - // Fetch reports for all provided feed IDs - reports, err := client.FetchSingleReportManyFeeds(feedIds) - if err != nil { - log.Fatalf("Failed to fetch reports: %v", err) - } - - // Slice to store decoded reports - var decodedReports []*internal.ReportWithContext - - // Process each report fetched - for index, report := range reports { - // Decode the full report data - decodedReport, err := internal.DecodeFullReportAndReportData(report.FullReport) - if err != nil { - log.Fatalf("Failed to decode report: %v", err) - } - decodedReports = append(decodedReports, decodedReport) - - // Print the full report in hex format as the payload - printPayload(report.FullReport, feedIds[index]) - } - - // Print details of all decoded reports - printReportDetails(decodedReports) -} - -// Helper function to print the full report (payload) in hex format -func printPayload(payload []byte, feedId string) { - fmt.Println("") - fmt.Printf("Payload for onchain verification for Feed ID %s\n", feedId) - fmt.Println("=============================================================") - fmt.Printf("Payload (hexadecimal): 0x%x\n", payload) - fmt.Println("-------------------------------------------------------------") - fmt.Println() -} - -// Helper function to print details of the decoded reports -func printReportDetails(reports []*internal.ReportWithContext) { - fmt.Println("") - fmt.Println("Report Details") - fmt.Println("==============") - for _, report := range reports { - fmt.Printf("Feed ID: %s\n", report.FeedId) - if report.V3Report != nil { - fmt.Println("Decoded V3 Report Details:") - fmt.Println("-------------------------") - fmt.Printf("Valid From Timestamp: %d\n", report.V3Report.ValidFromTimestamp) - fmt.Printf("Observations Timestamp: %d\n", report.V3Report.ObservationsTimestamp) - fmt.Printf("Native Fee: %s\n", report.V3Report.NativeFee.String()) - fmt.Printf("Link Fee: %s\n", report.V3Report.LinkFee.String()) - fmt.Printf("Expires At: %d\n", report.V3Report.ExpiresAt) - fmt.Printf("Benchmark Price: %s\n", report.V3Report.BenchmarkPrice.String()) - fmt.Printf("Bid: %s\n", report.V3Report.Bid.String()) - fmt.Printf("Ask: %s\n", report.V3Report.Ask.String()) - fmt.Println("------------------------------------------------") - } - } -} diff --git a/public/samples/DataStreams/StreamsDirect/singleFeed.go b/public/samples/DataStreams/StreamsDirect/singleFeed.go deleted file mode 100644 index 2c12dd5549e..00000000000 --- a/public/samples/DataStreams/StreamsDirect/singleFeed.go +++ /dev/null @@ -1,73 +0,0 @@ -// main.go for a single feed - -package main - -import ( - "data-streams-direct/client" - "data-streams-direct/internal" - "fmt" - "log" - "os" -) - -func main() { - // Check if a feed ID has been provided as an argument - if len(os.Args) < 2 { - log.Fatalf("Usage: %s ", os.Args[0]) - } - - // Retrieve the feedId from the CL arguments - feedId := os.Args[1] - - // Fetch the report for the specified feedId - report, err := client.FetchSingleReportSingleFeed(feedId) - if err != nil { - log.Fatalf("Failed to fetch report: %v", err) - } - - // Decode the full report data - decodedReport, err := internal.DecodeFullReportAndReportData(report.FullReport) - if err != nil { - log.Fatalf("Failed to decode report: %v", err) - } - - // Print details of the decoded report - printReportDetails(decodedReport) - - // Print the full report in hex format as the payload - printPayload(report.FullReport) -} - -// Helper function to print the full report (payload) in hex format -func printPayload(payload []byte) { - fmt.Println("") - fmt.Println("Payload for onchain verification") - fmt.Println("=========================================") - fmt.Printf("Payload (hexadecimal): 0x%x\n", payload) // Adding '0x' prefix for hexadecimal representation - fmt.Println("------------------------------------------------") - fmt.Println() -} - - -// Helper function to print details of the decoded report -func printReportDetails(report *internal.ReportWithContext) { - fmt.Println("") - fmt.Println("Report Details") - fmt.Println("==============") - fmt.Printf("Feed ID: %s\n", report.FeedId) - fmt.Println() - - if report.V3Report != nil { - fmt.Println("Decoded V3 Report Details:") - fmt.Println("-------------------------") - fmt.Printf("Valid From Timestamp: %d\n", report.V3Report.ValidFromTimestamp) - fmt.Printf("Observations Timestamp: %d\n", report.V3Report.ObservationsTimestamp) - fmt.Printf("Native Fee: %s\n", report.V3Report.NativeFee.String()) - fmt.Printf("Link Fee: %s\n", report.V3Report.LinkFee.String()) - fmt.Printf("Expires At: %d\n", report.V3Report.ExpiresAt) - fmt.Printf("Benchmark Price: %s\n", report.V3Report.BenchmarkPrice.String()) - fmt.Printf("Bid: %s\n", report.V3Report.Bid.String()) - fmt.Printf("Ask: %s\n", report.V3Report.Ask.String()) - fmt.Println() - } -} diff --git a/public/samples/DataStreams/StreamsDirect/ws/clientWs.go b/public/samples/DataStreams/StreamsDirect/ws/clientWs.go deleted file mode 100644 index 8d17155c1ed..00000000000 --- a/public/samples/DataStreams/StreamsDirect/ws/clientWs.go +++ /dev/null @@ -1,168 +0,0 @@ -// clientWs.go - -package client - -import ( - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "log" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/gorilla/websocket" -) - -// Constants for ping interval, pong timeout, write timeout, and retry backoff -const pingInterval = 5 * time.Second -const pongTimeout = 10 * time.Second -const writeTimeout = 5 * time.Second - -// NewReportWSMessage is the struct that we'll send to a subscribed client's MessageChan to be forwarded to the client's websocket connection -type NewReportWSMessage struct { - Report struct { - FeedId hexutil.Bytes `json:"feedID"` - FullReport hexutil.Bytes `json:"fullReport"` - } `json:"report"` -} - -const ( - wsPath = "/api/v1/ws" -) - -func GenerateHMAC(method string, path string, body []byte, clientId string, timestamp int64, userSecret string) string { - serverBodyHash := sha256.New() - serverBodyHash.Write(body) - serverBodyHashString := fmt.Sprintf("%s %s %s %s %d", - method, - path, - hex.EncodeToString(serverBodyHash.Sum(nil)), - clientId, - timestamp) - fmt.Println("Generating HMAC with the following: ", serverBodyHashString) - signedMessage := hmac.New(sha256.New, []byte(userSecret)) - signedMessage.Write([]byte(serverBodyHashString)) - userHmac := hex.EncodeToString(signedMessage.Sum(nil)) - return userHmac -} - -func GenerateAuthHeaders(method string, pathAndParams string, clientId string, userSecret string) http.Header { - header := http.Header{} - timestamp := time.Now().UTC().UnixMilli() - hmacString := GenerateHMAC(method, pathAndParams, []byte(""), clientId, timestamp, userSecret) - - header.Add("Authorization", clientId) - header.Add("X-Authorization-Timestamp", strconv.FormatInt(timestamp, 10)) - header.Add("X-Authorization-Signature-SHA256", hmacString) - return header -} - -// connectAndListen connects to the WebSocket server and starts listening for messages. -// It also handles ping/pong communication to keep the connection alive. -func ConnectAndListen(ctx context.Context, feedIds []string) error { - conn, err := openWebsocketConnection(ctx, feedIds) - if err != nil { - return err - } - defer conn.Close() - - // Start the ping/pong handling - go pingLoop(ctx, conn) - - // Set the initial read deadline - err = conn.SetReadDeadline(time.Now().Add(pongTimeout)) - if err != nil { - return err - } - - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - _, msg, err := conn.ReadMessage() - if err != nil { - return err - } - - var decoded NewReportWSMessage - err = json.Unmarshal(msg, &decoded) - if err != nil { - return fmt.Errorf("failed to unmarshal message: %w", err) - } - - // From here, you have decoded.FeedId and decoded.FullReport - // See examples for decoding the full report in other code snippets - fmt.Println("Received the following message: ", decoded) - - // There may be some latency between when you receive a message and when the report is retrievable - // So pause for a minute before taking action on the message - time.Sleep(500 * time.Millisecond) - } - } -} - -// openWebsocketConnection opens a WebSocket connection to the server. -func openWebsocketConnection(ctx context.Context, feedIds []string) (*websocket.Conn, error) { - baseUrl := os.Getenv("BASE_URL") // Example: https://ws.testnet-dataengine.chain.link - clientId := os.Getenv("CLIENT_ID") // Example: "00000000-0000-0000-0000-000000000000" - userSecret := os.Getenv("CLIENT_SECRET") // Example: "your-secret" - - if len(feedIds) == 0 { - return nil, fmt.Errorf("no feed ID(s) provided") - } - - params := url.Values{ - "feedIDs": {strings.Join(feedIds, ",")}, - } - - reqURL := &url.URL{ - Scheme: "wss", // Note the scheme here - Host: baseUrl, - Path: wsPath, - RawQuery: params.Encode(), - } - - headers := GenerateAuthHeaders("GET", reqURL.RequestURI(), clientId, userSecret) - conn, _, err := websocket.DefaultDialer.DialContext(ctx, reqURL.String(), headers) - if err != nil { - return nil, err - } - - // Add the Pong handler - conn.SetPongHandler(func(string) error { - log.Println("Websocket:", "Received pong...") - err := conn.SetReadDeadline(time.Now().Add(pongTimeout)) - return err - }) - - return conn, nil -} - -// PingPongLoop is a function that handles sending ping messages to the websocket and handles pong messages received -func pingLoop(ctx context.Context, conn *websocket.Conn) { - ticker := time.NewTicker(pingInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - log.Println("Websocket:", "Sending ping...") - err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeTimeout)) - if err != nil { - log.Printf("Failed to send ping: %v", err) - return - } - } - } -} \ No newline at end of file diff --git a/public/samples/DataStreams/StreamsDirect/ws/decodeFullReportHex.go b/public/samples/DataStreams/StreamsDirect/ws/decodeFullReportHex.go deleted file mode 100644 index 3bfb7e61637..00000000000 --- a/public/samples/DataStreams/StreamsDirect/ws/decodeFullReportHex.go +++ /dev/null @@ -1,40 +0,0 @@ -// decodeFullReportHex.go - -package main - -import ( - "data-streams-direct-ws/internal" - "encoding/hex" - "fmt" - "log" -) - -func main() { - // Sample FullReport payload extracted from the WebSocket message as a hex string - fullReportHex := "00067f14c763070bec1de1118aceeed1546878ab24e3213de21127249adabcbd0000000000000000000000000000000000000000000000000000000011f0c90b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000001e00000000000000000000000000000000000000000000000000000000000000240010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000027bbaff688c906a3e20a34fe951715d1018d262a5b66e38eda027a674cd1b0000000000000000000000000000000000000000000000000000000065cdcd950000000000000000000000000000000000000000000000000000000065cdcd95000000000000000000000000000000000000000000000000000020b5e9c686e80000000000000000000000000000000000000000000000000011b8f926846fb80000000000000000000000000000000000000000000000000000000065cf1f15000000000000000000000000000000000000000000000096ba314c8f5ec2800000000000000000000000000000000000000000000000000000000000000000029c85f7bb779ce5316821a6efd3bdc843e32f1438bad24a6b269871ade0c166d2142574b78d7a75d3cb855e51caf9cb36e1d598b43c881989251bd2c450624a2600000000000000000000000000000000000000000000000000000000000000021c1e5c393a57a24f2c698e23c59e6a954a9ef006b63c7f58eeabdd0bbeff21e5353ddef53e9beb93f628cd23e2cc28750533f444559622640bcf7dcc935d66af" - - // Convert the hex string to a byte slice - fullReportPayload, err := hex.DecodeString(fullReportHex) - if err != nil { - log.Fatalf("Failed to decode hex string: %v", err) - } - - // Decode the full report - decodedReport, err := internal.DecodeFullReportAndReportData(fullReportPayload) - if err != nil { - log.Fatalf("Failed to decode report: %v", err) - } - - fmt.Printf("Decoded Report: %+v\n", decodedReport) - - if decodedReport.FeedVersion == 3 && decodedReport.V3Report != nil { - fmt.Printf("Valid From Timestamp: %d\n", decodedReport.V3Report.ValidFromTimestamp) - fmt.Printf("Observations Timestamp: %d\n", decodedReport.V3Report.ObservationsTimestamp) - fmt.Printf("Native Fee: %s\n", decodedReport.V3Report.NativeFee.String()) - fmt.Printf("Link Fee: %s\n", decodedReport.V3Report.LinkFee.String()) - fmt.Printf("Expires At: %d\n", decodedReport.V3Report.ExpiresAt) - fmt.Printf("Benchmark Price: %s\n", decodedReport.V3Report.BenchmarkPrice.String()) - fmt.Printf("Bid: %s\n", decodedReport.V3Report.Bid.String()) - fmt.Printf("Ask: %s\n", decodedReport.V3Report.Ask.String()) - } -} \ No newline at end of file diff --git a/public/samples/DataStreams/StreamsDirect/ws/mod.go b/public/samples/DataStreams/StreamsDirect/ws/mod.go deleted file mode 100644 index 8a4e20baa3c..00000000000 --- a/public/samples/DataStreams/StreamsDirect/ws/mod.go +++ /dev/null @@ -1,25 +0,0 @@ -module data-streams-direct-ws - -go 1.21 - -require ( - github.com/ethereum/go-ethereum v1.12.2 // Ethereum blockchain interaction library - github.com/gorilla/websocket v1.5.0 // Websocket library - github.com/pkg/errors v0.9.1 // Library for handling errors - github.com/smartcontractkit/chainlink/v2 v2.2.1-0.20230823171354-1ead9ee6f6bb // Chainlink core components library -) - -replace ( - // Resolves version mismatch between cosmosSDK and hdevalence/ed25519consensus - filippo.io/edwards25519 => filippo.io/edwards25519 v1.0.0-rc.1 - - // Adds ARM support by updating CosmWasm to v1.2.4 - github.com/CosmWasm/wasmvm => github.com/CosmWasm/wasmvm v1.2.4 - - //// Fix go mod tidy issue for ambiguous imports from go-ethereum - //// See https://github.com/ugorji/go/issues/279 - github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.22.1 - - // Aligns protobuf version with cosmos SDK requirements - github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 -) \ No newline at end of file diff --git a/public/samples/DataStreams/StreamsDirect/ws/wsConnectAndListen.go b/public/samples/DataStreams/StreamsDirect/ws/wsConnectAndListen.go deleted file mode 100644 index 7df83a4fe08..00000000000 --- a/public/samples/DataStreams/StreamsDirect/ws/wsConnectAndListen.go +++ /dev/null @@ -1,25 +0,0 @@ -// main.go - -package main - -import ( - "context" - "data-streams-direct-ws/client" - "log" - "os" -) - -func main() { - if len(os.Args) < 2 { - log.Fatalf("Usage: %s ...", os.Args[0]) - } - - feedIds := os.Args[1:] - - ctx := context.Background() - - // Pass feed IDs to the ConnectAndListen function - if err := client.ConnectAndListen(ctx, feedIds); err != nil { - log.Fatalf("Error connecting and listening: %v", err) - } -} diff --git a/src/content/data-streams/tutorials/streams-direct-api.mdx b/src/content/data-streams/tutorials/streams-direct-api.mdx index 6f06c435a65..c409e67560c 100644 --- a/src/content/data-streams/tutorials/streams-direct-api.mdx +++ b/src/content/data-streams/tutorials/streams-direct-api.mdx @@ -10,8 +10,7 @@ whatsnext: } --- -import { CodeSample, CopyText } from "@components" -import { Tabs } from "@components/Tabs" +import { CopyText } from "@components" import DataStreams from "@features/data-streams/common/DataStreams.astro" @@ -20,8 +19,9 @@ In this tutorial, you'll learn how to use Chainlink Data Streams with the _[Stre -## Before you begin +## Requirements +- **Git**: Make sure you have Git installed. You can check your current version by running in your terminal and download the latest version from the official [Git website](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) if necessary. - **Go Version**: Make sure you have Go version 1.21 or higher. You can check your current version by running `go version` in your terminal and download the latest version from the official [Go website](https://go.dev/) if necessary. - **API Credentials**: Access to the Streams Direct implementation requires API credentials. If you haven't already, [contact us](https://chainlinkcommunity.typeform.com/datastreams?#ref_id=docs) to talk to an expert about integrating Chainlink Data Streams with your applications. @@ -31,65 +31,32 @@ You'll start with the set up of your Go project. Next, you'll fetch and decode r ### Set up your Go project -1. Create and navigate to your project directory: +1. Clone the [repository](https://github.com/smartcontractkit/smart-contract-examples) that contains the project setup for this guide and change directories to the `api` example: - ```bash - mkdir data-streams-direct && cd data-streams-direct - ``` - -1. Initialize a Go module: - - ```bash - go mod init data-streams-direct - ``` - -1. Open the `go.mod` file at the root of your project directory and include the necessary module and package information: - - - -1. Create `client` and `internal` subfolders within your project directory: - - ```bash - mkdir client internal - ``` - -1. Create a `client.go` file in the `client` subfolder and a `decoder.go` file in the `internal` subfolder: - - ```bash - touch client/client.go - touch internal/decoder.go - ``` - -1. Insert the provided code into `client.go` and `decoder.go` to enable your application to fetch and decode reports: + ```bash + git clone https://github.com/smartcontractkit/smart-contract-examples.git + cd smart-contract-examples/data-streams/streams-direct/api + ``` - - client/client.go - internal/decoder.go - - - - - - - +1. Execute the command below to download dependencies and generate the `go.sum` file: -1. Execute the command below to download dependencies and generate the `go.sum` file: - - ```bash - go mod tidy - ``` + ```bash + go mod tidy + ``` - Your project directory should now have the following structure: + Your project directory should now have the following structure: - ```plaintext - data-streams-direct/ + ```plaintext + api/ ├── client/ │ └── client.go ├── go.mod ├── go.sum ├── internal/ │ └── decoder.go - ``` + ├── main-multiple-feeds.go + └── main-single-feed.go + ``` ### Set environment variables @@ -106,22 +73,12 @@ Set the required environment variables in your terminal session to authenticate ### Fetch and decode a report with a single feed -1. Create `main.go` at the root of your project directory: - - ```bash - touch main.go - ``` - -1. Open `main.go` and insert the following code to fetch and decode a single feed: - - - -1. For this example, you will read from the ETH/USD Data Streams feed on Arbitrum Sepolia. This feed ID is . See the [Data Streams Feed IDs](/data-streams/stream-ids) page for a complete list of available assets. +For this example, you will read from the ETH/USD Data Streams feed on Arbitrum Sepolia. This feed ID is `0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782`. See the [Data Streams Feed IDs](/data-streams/stream-ids) page for a complete list of available assets. Execute your application: ```bash - go run main.go 0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782 + go run main-single-feed.go 0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782 ``` Expect output similar to the following in your terminal: @@ -178,12 +135,6 @@ production environment, you should verify the data onchain to ensure its integri ### Fetch and decode a report with multiple feeds -1. Open your `main.go` file at the root of your project directory. - -1. Replace the `main.go` file content with the following code to fetch and decode multiple feeds: - - - 1. Ensure the required environment variables are still defined in your terminal session. ```bash @@ -197,7 +148,7 @@ production environment, you should verify the data onchain to ensure its integri 1. For this example, you will read from the ETH/USD and LINK/USD Data Streams feeds on Arbitrum Sepolia. Run your application: ```bash - go run main.go 0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782 0x00036fe43f87884450b4c7e093cd5ed99cac6640d8c2000e6afc02c8838d0265 + go run main-multiple-feeds.go 0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782 0x00036fe43f87884450b4c7e093cd5ed99cac6640d8c2000e6afc02c8838d0265 ``` Expect to see the output below in your terminal: diff --git a/src/content/data-streams/tutorials/streams-direct-ws.mdx b/src/content/data-streams/tutorials/streams-direct-ws.mdx index f9dce5b2fe3..3b8dac7dc8a 100644 --- a/src/content/data-streams/tutorials/streams-direct-ws.mdx +++ b/src/content/data-streams/tutorials/streams-direct-ws.mdx @@ -9,8 +9,7 @@ whatsnext: } --- -import { CodeSample, CopyText } from "@components" -import { Tabs } from "@components/Tabs" +import { CopyText } from "@components" import DataStreams from "@features/data-streams/common/DataStreams.astro" @@ -19,8 +18,9 @@ In this tutorial, you'll learn how to use Chainlink Data Streams with the _[Stre -## Before you begin +## Requirements +- **Git**: Make sure you have Git installed. You can check your current version by running in your terminal and download the latest version from the official [Git website](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) if necessary. - **Go Version**: Make sure you have Go version 1.21 or higher. You can check your current version by running `go version` in your terminal and download the latest version from the official [Go website](https://go.dev/) if necessary. - **API Credentials**: Access to the Streams Direct implementation requires API credentials. If you haven't already, [contact us](https://chainlinkcommunity.typeform.com/datastreams?#ref_id=docs) to talk to an expert about integrating Chainlink Data Streams with your applications. @@ -28,65 +28,32 @@ In this tutorial, you'll learn how to use Chainlink Data Streams with the _[Stre ### Set up your Go project -1. Create and navigate to your project directory: +1. Clone the [repository](https://github.com/smartcontractkit/smart-contract-examples) that contains the project setup for this guide and change directories to the `websocket` example: - ```bash - mkdir data-streams-direct-ws && cd data-streams-direct-ws - ``` - -1. Initialize a Go module: - - ```bash - go mod init data-streams-direct-ws - ``` - -1. Open the `go.mod` file at the root of your project directory and include the necessary module and package information: - - - -1. Create `client` and `internal` subfolders within your project directory: - - ```bash - mkdir client internal - ``` - -1. Create a `clientWs.go` file in the `client` subfolder and a `decoder.go` file in the `internal` subfolder: - - ```bash - touch client/clientWs.go - touch internal/decoder.go - ``` - -1. Insert the provided code into `clientWs.go` and `decoder.go` to allow your application to establish a WebSocket connection, listen for new reports, and decode them: + ```bash + git clone https://github.com/smartcontractkit/smart-contract-examples.git + cd smart-contract-examples/data-streams/streams-direct/websocket + ``` - - client/clientWs.go - internal/decoder.go - - - - - - - +1. Execute the command below to download dependencies and generate the `go.sum` file: -1. Execute the command below to download dependencies and generate the `go.sum` file: + ```bash + go mod tidy + ``` - ```bash - go mod tidy - ``` - - Your project directory should now have the following structure: + Your project directory should now have the following structure: - ```plaintext - data-streams-direct-ws/ + ```plaintext + websocket/ ├── client/ │ └── clientWs.go - ├── go.mod - ├── go.sum ├── internal/ │ └── decoder.go - ``` + ├── decodeFullReportHex.go + ├── go.mod + ├── go.sum + └── main.go + ``` ### Set environment variables @@ -103,17 +70,7 @@ Set the required environment variables in your terminal session to authenticate ### Establish a WebSocket connection and listen for real-time reports -1. Create `main.go` at the root of your project directory: - - ```bash - touch main.go - ``` - -1. Open `main.go` and insert the following code to establish a WebSocket connection and listen for real-time data reports from the Data Streams Aggregation Network: - - - -1. For this example, you will read from the ETH/USD Data Streams feed on Arbitrum Sepolia. This feed ID is . See the [Data Streams Feed IDs](/data-streams/stream-ids) page for a complete list of available assets. +For this example, you will read from the ETH/USD Data Streams feed on Arbitrum Sepolia. This feed ID is . See the [Data Streams Feed IDs](/data-streams/stream-ids) page for a complete list of available assets. Launch the WebSocket listener by running the following command in your terminal: @@ -153,18 +110,19 @@ go run main.go 0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba78 ### Decode the reports -1. Create `decodeFullReportHex.go` at the root of your project directory: +1. Open `decodeFullReportHex.go` file located at the root of your project directory. - ```bash - touch decodeFullReportHex.go - ``` +1. Replace the `fullReportHex` variable value within the `main` function with the hexadecimal value you saved earlier from your WebSocket message output, without the `0x` prefix. E.g.: -1. - Open `decodeFullReportHex.go` and insert the sample code provided below. It uses the `DecodeFullReportAndReportData` function from the `internal` package to decode the `FullReport` part of the message. - - Replace the `fullReportHex` variable value with the hexadecimal value you saved earlier from your WebSocket message output, without the `0x` prefix. + ```go + [...] - {" "} + func main() { + // Sample FullReport payload extracted from the WebSocket message as a hex string + fullReportHex := "0006f9b553e393ced311551efd30d1decedb63d76ad41737462e2cdbbdff1578000000000000000000000000000000000000000000000000000000001d49ab07000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000220000000000000000000000000000000000000000000000000000000000000028000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000120000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba7820000000000000000000000000000000000000000000000000000000066138c5a0000000000000000000000000000000000000000000000000000000066138c5a00000000000000000000000000000000000000000000000000001a8cf593f46c000000000000000000000000000000000000000000000000001419360d1b077c000000000000000000000000000000000000000000000000000000006614ddda0000000000000000000000000000000000000000000000b9b286b0a60b5f3ca00000000000000000000000000000000000000000000000b9ae8a4a15117180000000000000000000000000000000000000000000000000b9b4eabe86f969652000000000000000000000000000000000000000000000000000000000000000023ede508913fc4066c754edba91d60e12c71911d616231da453e27eba07f34538dc3f32ae21d6ab44cff5fbba64b240f38252ee0a483c6450518257dc748926e900000000000000000000000000000000000000000000000000000000000000021650e1dc132ec7d7ef19672d3ae8c4e0bd1a069439d017664bb665d29a3b415163c6aa6172b2fb6e1658dee2e09292b741fc81cd17cce224e6e790fe3749df2f" - + [...] + ``` 1. Run the decoding script by executing the following command in your terminal: diff --git a/src/features/data-streams/common/gettingStartedHardhat.mdx b/src/features/data-streams/common/gettingStartedHardhat.mdx index ff6cf3718a5..84089df04f4 100644 --- a/src/features/data-streams/common/gettingStartedHardhat.mdx +++ b/src/features/data-streams/common/gettingStartedHardhat.mdx @@ -32,7 +32,6 @@ This guide uses the [Hardhat](https://hardhat.org/) development environment to d ```bash git clone https://github.com/smartcontractkit/smart-contract-examples.git cd smart-contract-examples/data-streams/getting-started/hardhat - cd smart-contract-examples/data-streams/getting-started/hardhat ``` 1. Install the dependencies: From ed26682c3b1282346288bcaec51fc5a5e8ebdd8d Mon Sep 17 00:00:00 2001 From: Karim <98668332+khadni@users.noreply.github.com> Date: Tue, 30 Apr 2024 13:49:52 +0200 Subject: [PATCH 2/3] Update DS SD guides --- src/content/data-streams/tutorials/streams-direct-api.mdx | 4 ++-- src/content/data-streams/tutorials/streams-direct-ws.mdx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/content/data-streams/tutorials/streams-direct-api.mdx b/src/content/data-streams/tutorials/streams-direct-api.mdx index c409e67560c..5810fe64d18 100644 --- a/src/content/data-streams/tutorials/streams-direct-api.mdx +++ b/src/content/data-streams/tutorials/streams-direct-api.mdx @@ -211,13 +211,13 @@ production environment, you should verify the data onchain to ensure its integri ### Fetching a report -The application fetches a report from the Data Streams Aggregation Network using either the `FetchSingleReportSingleFeed` function for a single feed or the `FetchSingleReportMultipleFeeds` function for multiple feeds, found in the `client` package. This step involves sending an API request and receiving a response with the report data in either a `SingleReport` struct (single feed) or a `BulkReportResponse` struct (multiple feeds). +The application fetches a report from the Data Streams Aggregation Network using either the [`FetchSingleReportSingleFeed`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/api/client/client.go#L69) function for a single feed or the [`FetchSingleReportManyFeeds`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/api/client/client.go#L120) function for multiple feeds, found in the `client` package. This step involves sending an API request and receiving a response with the report data in either a [`SingleReport`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/api/client/client.go#L22) struct (single feed) or a [`BulkReportResponse`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/api/client/client.go#L33) struct (multiple feeds). The response includes a `FullReport` blob containing the encoded report's context and observations. ### Decoding a report -When the application successfully fetches the report, it decodes the report with the `DecodeFullReportAndReportData` function from the `internal` package. The process includes: +When the application successfully fetches the report, it decodes the report with the [`DecodeFullReportAndReportData`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/api/internal/decoder.go#L102) function from the `internal` package. The process includes: 1. **`FullReport` blob decoding**: It first decodes the `FullReport` blob into a `FullReport` struct, extracting the report context (including the feed ID(s) and the report version) and the report data. In this example, the feed uses the `V3` report version. diff --git a/src/content/data-streams/tutorials/streams-direct-ws.mdx b/src/content/data-streams/tutorials/streams-direct-ws.mdx index 3b8dac7dc8a..0f3f473f7ac 100644 --- a/src/content/data-streams/tutorials/streams-direct-ws.mdx +++ b/src/content/data-streams/tutorials/streams-direct-ws.mdx @@ -173,14 +173,14 @@ production environment, you should verify the data onchain to ensure its integri ### Establishing a WebSocket connection and listening for message reports -The `ConnectAndListen` function in the `client` package initializes the connection and enables the application to listen for messages from the Data Streams Aggregation Network via WebSocket. +The [`ConnectAndListen`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/websocket/client/clientWs.go#L70) function in the `client` package initializes the connection and enables the application to listen for messages from the Data Streams Aggregation Network via WebSocket. - Authentication and security: Uses HMAC headers, created with `CLIENT_ID` and `CLIENT_SECRET`, for secure, authenticated communication. - Continuous listening: After authentication, continuously listens for real-time report messages, which include the `feedId` and the encoded `FullReport`. ### Decoding a report -The `DecodeFullReportAndReportData` function in the `internal` package unpacks the `FullReport` payload, decodes the `FullReport`'s `ReportBlob`, and decodes the report data into a structured format: +The [`DecodeFullReportAndReportData`](https://github.com/smartcontractkit/smart-contract-examples/blob/c7a3acfecd30902b8fa7929d276a2b421a55fdec/data-streams/streams-direct/websocket/internal/decoder.go#L102) function in the `internal` package unpacks the `FullReport` payload, decodes the `FullReport`'s `ReportBlob`, and decodes the report data into a structured format: 1. It unpacks the `FullReport` payload that includes the report context, the report blob and raw signatures: From 9bfd440c987b8dbd88800a7f75ae5029fa4fddc0 Mon Sep 17 00:00:00 2001 From: Karim H <98668332+khadni@users.noreply.github.com> Date: Wed, 1 May 2024 14:08:10 +0200 Subject: [PATCH 3/3] Update src/content/data-streams/tutorials/streams-direct-ws.mdx Co-authored-by: Dwight Lyle --- src/content/data-streams/tutorials/streams-direct-ws.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/data-streams/tutorials/streams-direct-ws.mdx b/src/content/data-streams/tutorials/streams-direct-ws.mdx index 0f3f473f7ac..300abdd1089 100644 --- a/src/content/data-streams/tutorials/streams-direct-ws.mdx +++ b/src/content/data-streams/tutorials/streams-direct-ws.mdx @@ -112,7 +112,7 @@ go run main.go 0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba78 1. Open `decodeFullReportHex.go` file located at the root of your project directory. -1. Replace the `fullReportHex` variable value within the `main` function with the hexadecimal value you saved earlier from your WebSocket message output, without the `0x` prefix. E.g.: +1. Replace the `fullReportHex` variable value within the `main` function with the hexadecimal value you saved earlier from your WebSocket message output, without the `0x` prefix: ```go [...]