Merge pull request #6 from weaveVM/v2

V2

Author: charmful0x

Cargo.lock

@@ -1,19 +1,17 @@
 [package]
 name = "wvm-blobscan"
-version = "1.0.0"
+version = "0.2.0"
 edition = "2021"
 
 [dependencies]
-alloy-chains = "0.1"
+alloy-chains = "0.1.0"
 alloy-primitives = { version = "0.8", default-features = false, features = [
     "std",
     "serde",
 ] }
-alloy-rpc-types-eth = "0.4"
-alloy-eips = "0.4"
-alloy-serde = "0.4"
-
-foundry-blob-explorers = {git = "https://github.com/charmful0x/block-explorers.git"}
+alloy-rpc-types-eth = "0.13.0"
+alloy-eips = "0.13.0"
+alloy-serde = "0.13.0"
 tokio = { version = "1.41.1", features = ["full"]}
 ethers = {version = "2.0.14", features = ["ws"]}
 eyre = "0.6.12"
@@ -22,8 +20,8 @@ serde = "1.0.215"
 serde_json = "1.0.133"
 planetscale-driver = "0.5.1"
 dotenv = "0.15.0"
-anyhow = "1.0.93"
 shuttle-runtime = "0.50.0"
 shuttle-axum = "0.50.0"
 axum = "0.7.9"
-reqwest = "0.12.12"
+reqwest = {version = "0.12.12", features=["json"]}
+anyhow = "1.0.98"

Cargo.toml

@@ -1,18 +1,12 @@
-<p align="center">
-  <a href="https://wvm.dev">
-    <img src="https://github.com/raw/weaveVM/.github/main/profile/bg.png">
-  </a>
-</p>
-
 ## About
-`wvm-blobscan` is a data bridge built to archive Ethereum's blob data on [WeaveVM](https://wvm.dev). It grabs blocks containing eip-4844 transactions from [Blobscan](https://blobscan.com) and archive them permanently on WeaveVM..
+`wvm-blobscan` is a data bridge built to archive Ethereum's blob data on [Load Network](https://load.network). It grabs blocks containing eip-4844 transactions from [Blobscan](https://blobscan.com) and archive them permanently on Load Network.
 
 ### Prerequisites & Dependencies
 
 While the core functionality of this ETL codebase can run without web2 component dependencies, this node implementation uses [planetscale](https://planetscale.com) for cloud indexing and [shuttle.dev](https://shuttle.dev) for backend hosting. Check [.env.example](./env.example) to set up your environment variables.
 
 ```js
-blobscan_pk="" // WeaveVM Blobscan archiver PK
+blobscan_pk="" // Load Blobscan archiver PK
 
 DATABASE_HOST="" // planetscale
 DATABASE_USERNAME="" // planetscale
@@ -47,7 +41,7 @@ graph TD
 ### Retrieve blob data and the associated archive txid by VersionedHash
 
 ```bash
-curl -X GET https://blobscan.wvm.network/v1/blob/$BLOB_VERSIONED_HASH
+curl -X GET https://blobscan.load.rs/v1/blob/$BLOB_VERSIONED_HASH
 ```
 
 Returns the res in the format as in below:
@@ -64,7 +58,7 @@ pub struct PsGetBlockByVersionedHash {
 ### Retrieve Archiver stats
 
 ```bash
-curl -X GET https://blobscan.wvm.network/v1/stats
+curl -X GET https://blobscan.load.rs/v1/stats
 ```
 
 ## License

README.md

@@ -3,7 +3,7 @@ use {
     std::sync::Arc,
     tokio::sync::RwLock,
     utils::{
-        blobscan::{get_block_by_id, insert_block}, // backfill_blobscan_blobs
+        blobscan::{get_blobs_of_block, insert_block},
         constants::FIRST_ETH_L1_EIP4844_BLOCK,
         eth::Ethereum,
         planetscale::get_latest_block_id,
@@ -39,11 +39,13 @@ async fn main(
             println!("latest archived block id: {}", latest_archived_block);
             let mut block_number = reader_block_number.read().await;
             if *block_number > FIRST_ETH_L1_EIP4844_BLOCK && latest_archived_block < *block_number {
-                let block = get_block_by_id(latest_archived_block + 1).await;
-                match block {
-                    Ok(block) => {
-                        println!("block response: {:?}", block);
-                        let res = insert_block(block).await;
+                let target_block_id = latest_archived_block + 1;
+                let blobs = get_blobs_of_block(target_block_id).await;
+                println!("GOT BLOBS OF BLOCK #{:?}", target_block_id);
+                match blobs {
+                    Ok(blobs) => {
+                        println!("INSERTING: {:?} BLOBS", blobs.len());
+                        let res = insert_block(target_block_id, blobs).await;
                         match res {
                             Ok(res) => latest_archived_block += 1,
                             _ => eprintln!("error updating planetscale"),
@@ -61,7 +63,7 @@ async fn main(
     let eth_block_updater = tokio::spawn(async move {
         loop {
             let mut block_number = writer_block_number.write().await;
-            *block_number += Ethereum::get_latest_eth_block().await.unwrap();
+            *block_number = Ethereum::get_latest_eth_block().await.unwrap();
             println!("Updated Ethereum Block Number: {}", *block_number);
             tokio::time::sleep(tokio::time::Duration::from_secs(12)).await;
         }

src/main.rs

@@ -1,48 +1,78 @@
 use {
-    crate::utils::env_var::get_env_var,
     crate::utils::{
+        env_var::get_env_var,
         planetscale::{ps_archive_block, ps_get_all_versioned_hashes_paginated},
         types::BlobInfo,
         wvm::send_wvm_calldata,
     },
     eyre::{eyre, Error, Result},
-    foundry_blob_explorers::{BlockResponse, Client},
-    reqwest, serde_json,
+    reqwest,
+    serde_json::{self, Value},
     std::io::{Read, Write},
 };
 
-pub async fn get_block_by_id(block_id: u32) -> Result<BlockResponse, Error> {
-    let block_id = block_id.to_string();
-    let client = Client::mainnet();
-
-    let block = client.block(block_id.parse().unwrap()).await;
+pub async fn get_blobs_versioned_hashes_of_block(
+    block_id: u32,
+) -> Result<Vec<String>, eyre::Error> {
+    let url = format!(
+        "https://api.blobscan.com/blocks/{}?type=canonical",
+        block_id
+    );
+    let req: Value = reqwest::Client::new()
+        .get(url)
+        .send()
+        .await
+        .unwrap()
+        .json()
+        .await?;
+    let versioned_hashes: Vec<String> = req
+        .pointer("/transactions")
+        .and_then(|txs| txs.as_array())
+        .map(|txs| {
+            txs.iter()
+                .filter_map(|tx| tx.pointer("/blobs"))
+                .filter_map(|blobs| blobs.as_array())
+                .flat_map(|blobs| {
+                    blobs
+                        .iter()
+                        .filter_map(|blob| blob.pointer("/versionedHash"))
+                        .filter_map(|hash| hash.as_str())
+                        .map(String::from)
+                        .collect::<Vec<String>>()
+                })
+                .collect()
+        })
+        .unwrap_or_default();
+    Ok(versioned_hashes)
+}
 
-    match block {
-        Ok(block) => Ok(block),
-        Err(e) => {
-            eprintln!("Error getting block: {:?}", e);
-            return Err(eyre!("Error getting block: {:?}", e));
-        }
-    }
+async fn get_blob_data(versioned_hash: &str) -> Result<String, eyre::Error> {
+    let url = format!("https://api.blobscan.com/blobs/{}/data", versioned_hash);
+    let res = reqwest::Client::new()
+        .get(url)
+        .send()
+        .await?
+        .text()
+        .await
+        .unwrap_or_default();
+    Ok(res)
 }
 
-pub fn get_blobs_of_block(block: BlockResponse) -> Result<Vec<BlobInfo>> {
+pub async fn get_blobs_of_block(block_id: u32) -> Result<Vec<BlobInfo>> {
+    let versioned_hashes = get_blobs_versioned_hashes_of_block(block_id)
+        .await
+        .unwrap_or_default();
     let mut res: Vec<BlobInfo> = Vec::new();
-    let txs = block.transactions;
-    for _tx in txs {
-        let blobs = _tx.blobs;
-
-        println!("BLOCKS COUNT IN BLOCK #{} is {}", block.number, blobs.len());
-
-        for _blob in blobs {
-            let to_blob_info = BlobInfo::from(
-                block.number,
-                _blob.versioned_hash.to_string(),
-                _blob.data.to_string(),
-            );
-            // println!("{:?}", to_blob_info);
-            res.push(to_blob_info);
-        }
+    for hash in versioned_hashes {
+        let blob_data = get_blob_data(&hash).await.unwrap();
+
+        let blob = BlobInfo {
+            ethereum_block_number: block_id as u64,
+            versioned_hash: hash,
+            data: blob_data,
+        };
+
+        res.push(blob);
     }
 
     Ok(res)
@@ -54,21 +84,13 @@ pub fn serialize_blobscan_block(block: &BlobInfo) -> Result<Vec<u8>> {
     Ok(compressed_data)
 }
 
-pub async fn insert_block(block: BlockResponse) -> Result<(), Error> {
-    let blobs = get_blobs_of_block(block.clone())?;
-
+pub async fn insert_block(block_id: u32, blobs: Vec<BlobInfo>) -> Result<(), Error> {
     for blob in blobs {
         let wvm_data_input = serialize_blobscan_block(&blob)?;
-        let raw_block_data = serde_json::to_string(&blob)?;
         let wvm_txid = send_wvm_calldata(wvm_data_input).await.unwrap();
-        let res = ps_archive_block(
-            &(block.clone().number as u32),
-            &wvm_txid,
-            &blob.versioned_hash,
-            &blob.data,
-        )
-        .await
-        .unwrap();
+        let _res = ps_archive_block(&block_id, &wvm_txid, &blob.versioned_hash, &blob.data)
+            .await
+            .unwrap();
         let _send_to_blobscan = send_blob_to_blobscan(&blob.versioned_hash).await.unwrap();
     }
 

src/utils/blobscan.rs

@@ -1,5 +1,5 @@
 pub const FIRST_ETH_L1_EIP4844_BLOCK: u32 = 19_426_589;
 pub const ETH_RPC_URL: &str = "https://eth.llamarpc.com";
-pub const WVM_RPC_URL: &str = "https://testnet-rpc.wvm.dev";
+pub const WVM_RPC_URL: &str = "https://alphanet.load.network";
 pub const WVM_CHAIN_ID: u32 = 9496;
 pub const WVM_ARCHIVER_ADDRESS: &str = "0x2BC885e26A3947646FAbb96C68cE82f5937038a7";

src/utils/constants.rs

@@ -27,6 +27,7 @@ pub async fn ps_archive_block(
 ) -> Result<(), Error> {
     // format to the table VAR(66) limitation
     let wvm_calldata_txid = wvm_calldata_txid.trim_matches('"');
+    let blob_data = blob_data.trim_matches('"');
     let conn = ps_init().await;
 
     let res = query(

src/utils/planetscale.rs

@@ -39,3 +39,13 @@ impl BlobInfo {
         }
     }
 }
+
+impl std::fmt::Display for BlobInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Block: {}, Hash: {}, Data: {}",
+            self.ethereum_block_number, self.versioned_hash, self.data
+        )
+    }
+}