Bitcoin archive

crates/archive/src/archive.rs

@@ -8,6 +8,7 @@ use crate::server::run_server;
 use crate::writer::Writer;
 use anyhow::{ensure, Context};
 use prometheus_client::registry::Registry;
+use sqd_data::bitcoin::tables::BitcoinChunkBuilder;
 use sqd_data::evm::tables::EvmChunkBuilder;
 use sqd_data::hyperliquid_fills::tables::HyperliquidFillsChunkBuilder;
 use sqd_data::hyperliquid_replica_cmds::tables::HyperliquidReplicaCmdsChunkBuilder;
@@ -65,6 +66,7 @@ pub async fn run(args: Cli) -> anyhow::Result<()> {
     }
 
     let proc_task = match args.network_kind {
+        NetworkKind::Bitcoin => proc!(BitcoinChunkBuilder::default()),
         NetworkKind::Solana => proc!(SolanaChunkBuilder::default()),
         NetworkKind::HyperliquidFills => proc!(HyperliquidFillsChunkBuilder::default()),
         NetworkKind::HyperliquidReplicaCmds => proc!(HyperliquidReplicaCmdsChunkBuilder::default()),

crates/archive/src/cli.rs

@@ -5,6 +5,7 @@ use url::Url;
 
 #[derive(ValueEnum, Clone, Debug)]
 pub enum NetworkKind {
+    Bitcoin,
     Solana,
     HyperliquidFills,
     HyperliquidReplicaCmds,

crates/data/src/bitcoin/mod.rs

@@ -0,0 +1,2 @@
+pub mod model;
+pub mod tables;

crates/data/src/bitcoin/model.rs

@@ -0,0 +1,130 @@
+use crate::types::HexBytes;
+use serde::Deserialize;
+use sqd_primitives::BlockNumber;
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BlockHeader {
+    pub number: BlockNumber,
+    pub hash: HexBytes,
+    pub parent_hash: HexBytes,
+    pub timestamp: u32,
+    pub median_time: u32,
+    pub version: u32,
+    pub merkle_root: HexBytes,
+    pub nonce: u32,
+    pub target: HexBytes,
+    pub bits: HexBytes,
+    pub difficulty: f64,
+    pub chain_work: HexBytes,
+    pub stripped_size: u64,
+    pub size: u64,
+    pub weight: u64,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ScriptSig {
+    pub hex: HexBytes,
+    pub asm: Option<String>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ScriptPubKey {
+    pub hex: HexBytes,
+    pub asm: Option<String>,
+    pub desc: Option<String>,
+    #[serde(rename = "type")]
+    pub type_: Option<String>,
+    pub address: Option<String>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Prevout {
+    pub generated: bool,
+    pub height: BlockNumber,
+    pub value: f64,
+    pub script_pub_key: ScriptPubKey,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TransactionInputTx {
+    pub txid: HexBytes,
+    pub vout: u32,
+    pub script_sig: ScriptSig,
+    pub sequence: u32,
+    pub tx_in_witness: Option<Vec<HexBytes>>,
+    pub prevout: Option<Prevout>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TransactionInputCoinbase {
+    pub coinbase: HexBytes,
+    pub sequence: u32,
+    pub tx_in_witness: Option<Vec<HexBytes>>,
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+pub enum TransactionInput {
+    #[serde(rename = "tx")]
+    Tx(TransactionInputTx),
+    #[serde(rename = "coinbase")]
+    Coinbase(TransactionInputCoinbase),
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TransactionOutput {
+    pub value: f64,
+    pub n: u32,
+    pub script_pub_key: ScriptPubKey,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Transaction {
+    pub hex: HexBytes,
+    pub txid: HexBytes,
+    pub hash: HexBytes,
+    pub size: u64,
+    pub vsize: u64,
+    pub weight: u64,
+    pub version: u32,
+    pub locktime: u32,
+    pub vin: Vec<TransactionInput>,
+    pub vout: Vec<TransactionOutput>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Block {
+    pub header: BlockHeader,
+    pub transactions: Vec<Transaction>,
+}
+
+impl sqd_primitives::Block for Block {
+    fn number(&self) -> BlockNumber {
+        self.header.number
+    }
+
+    fn hash(&self) -> &str {
+        &self.header.hash
+    }
+
+    fn parent_number(&self) -> BlockNumber {
+        self.header.number.saturating_sub(1)
+    }
+
+    fn parent_hash(&self) -> &str {
+        &self.header.parent_hash
+    }
+
+    fn timestamp(&self) -> Option<i64> {
+        Some(self.header.timestamp as i64 * 1000)
+    }
+}

crates/data/src/bitcoin/tables/block.rs

@@ -0,0 +1,55 @@
+use crate::bitcoin::model::BlockHeader;
+use crate::bitcoin::tables::common::*;
+use sqd_array::builder::{
+    Float64Builder, TimestampSecondBuilder, UInt32Builder, UInt64Builder,
+};
+use sqd_data_core::table_builder;
+
+
+table_builder! {
+    BlockBuilder {
+        number: UInt64Builder,
+        hash: HexBytesBuilder,
+        parent_hash: HexBytesBuilder,
+        timestamp: TimestampSecondBuilder,
+        median_time: TimestampSecondBuilder,
+        version: UInt32Builder,
+        merkle_root: HexBytesBuilder,
+        nonce: UInt32Builder,
+        target: HexBytesBuilder,
+        bits: HexBytesBuilder,
+        difficulty: Float64Builder,
+        chain_work: HexBytesBuilder,
+        stripped_size: UInt64Builder,
+        size: UInt64Builder,
+        weight: UInt64Builder,
+    }
+
+    description(d) {
+        d.downcast.block_number = vec!["number"];
+        d.sort_key = vec!["number"];
+        d.options.add_stats("number");
+        d.options.row_group_size = 5_000;
+    }
+}
+
+
+impl BlockBuilder {
+    pub fn push(&mut self, row: &BlockHeader) {
+        self.number.append(row.number);
+        self.hash.append(&row.hash);
+        self.parent_hash.append(&row.parent_hash);
+        self.timestamp.append(row.timestamp as i64);
+        self.median_time.append(row.median_time as i64);
+        self.version.append(row.version);
+        self.merkle_root.append(&row.merkle_root);
+        self.nonce.append(row.nonce);
+        self.target.append(&row.target);
+        self.bits.append(&row.bits);
+        self.difficulty.append(row.difficulty);
+        self.chain_work.append(&row.chain_work);
+        self.stripped_size.append(row.stripped_size);
+        self.size.append(row.size);
+        self.weight.append(row.weight);
+    }
+}

crates/data/src/bitcoin/tables/common.rs

@@ -0,0 +1,4 @@
+use sqd_array::builder::{ListBuilder, StringBuilder};
+
+pub type HexBytesBuilder = StringBuilder;
+pub type WitnessListBuilder = ListBuilder<HexBytesBuilder>;

crates/data/src/bitcoin/tables/input.rs

@@ -0,0 +1,134 @@
+use crate::bitcoin::model::{Block, TransactionInput};
+use crate::bitcoin::tables::common::*;
+use sqd_array::builder::{
+    BooleanBuilder, Float64Builder, StringBuilder, UInt32Builder, UInt64Builder,
+};
+use sqd_data_core::table_builder;
+
+
+table_builder! {
+    InputBuilder {
+        block_number: UInt64Builder,
+        transaction_index: UInt32Builder,
+        input_index: UInt32Builder,
+        r#type: StringBuilder,
+        txid: HexBytesBuilder,
+        vout: UInt32Builder,
+        script_sig_hex: HexBytesBuilder,
+        script_sig_asm: StringBuilder,
+        sequence: UInt32Builder,
+        coinbase: HexBytesBuilder,
+        tx_in_witness: WitnessListBuilder,
+
+        // Denormalized prevout data
+        prevout_generated: BooleanBuilder,
+        prevout_height: UInt64Builder,
+        prevout_value: Float64Builder,
+        prevout_script_pub_key_hex: HexBytesBuilder,
+        prevout_script_pub_key_asm: StringBuilder,
+        prevout_script_pub_key_desc: StringBuilder,
+        prevout_script_pub_key_type: StringBuilder,
+        prevout_script_pub_key_address: StringBuilder,
+    }
+
+    description(d) {
+        d.downcast.block_number = vec!["block_number", "prevout_height"];
+        d.downcast.item_index = vec!["transaction_index", "input_index"];
+        d.sort_key = vec!["prevout_script_pub_key_address", "block_number", "transaction_index", "input_index"];
+        d.options.add_stats("block_number");
+        d.options.add_stats("transaction_index");
+        d.options.add_stats("type");
+        d.options.add_stats("prevout_script_pub_key_address");
+        d.options.add_stats("prevout_script_pub_key_type");
+        d.options.add_stats("prevout_generated");
+        d.options.use_dictionary("type");
+        d.options.use_dictionary("prevout_script_pub_key_type");
+        d.options.use_dictionary("prevout_script_pub_key_address");
+        d.options.row_group_size = 10_000;
+    }
+}
+
+
+impl InputBuilder {
+    pub fn push(
+        &mut self,
+        block: &Block,
+        transaction_index: u32,
+        input_index: u32,
+        row: &TransactionInput,
+    ) {
+        self.block_number.append(block.header.number);
+        self.transaction_index.append(transaction_index);
+        self.input_index.append(input_index);
+
+        match row {
+            TransactionInput::Tx(tx_input) => {
+                self.r#type.append("tx");
+                self.txid.append(&tx_input.txid);
+                self.vout.append(tx_input.vout);
+                self.script_sig_hex.append(&tx_input.script_sig.hex);
+                self.script_sig_asm
+                    .append_option(tx_input.script_sig.asm.as_deref());
+                self.sequence.append(tx_input.sequence);
+                self.coinbase.append_option(None::<&str>);
+
+                for witness in tx_input.tx_in_witness.iter().flatten() {
+                    self.tx_in_witness.values().append(witness);
+                }
+                self.tx_in_witness.append();
+
+                // Denormalized prevout data
+                if let Some(prevout) = &tx_input.prevout {
+                    self.prevout_generated.append(prevout.generated);
+                    self.prevout_height.append(prevout.height);
+                    self.prevout_value.append(prevout.value);
+                    self.prevout_script_pub_key_hex
+                        .append(&prevout.script_pub_key.hex);
+                    self.prevout_script_pub_key_asm
+                        .append_option(prevout.script_pub_key.asm.as_deref());
+                    self.prevout_script_pub_key_desc
+                        .append_option(prevout.script_pub_key.desc.as_deref());
+                    self.prevout_script_pub_key_type
+                        .append_option(prevout.script_pub_key.type_.as_deref());
+                    self.prevout_script_pub_key_address
+                        .append_option(prevout.script_pub_key.address.as_deref());
+                } else {
+                    self.prevout_generated.append_option(None::<bool>);
+                    self.prevout_height.append_option(None::<u64>);
+                    self.prevout_value.append_option(None::<f64>);
+                    self.prevout_script_pub_key_hex.append_option(None::<&str>);
+                    self.prevout_script_pub_key_asm.append_option(None::<&str>);
+                    self.prevout_script_pub_key_desc.append_option(None::<&str>);
+                    self.prevout_script_pub_key_type.append_option(None::<&str>);
+                    self.prevout_script_pub_key_address
+                        .append_option(None::<&str>);
+                }
+            }
+            TransactionInput::Coinbase(cb_input) => {
+                self.r#type.append("coinbase");
+                self.txid.append_option(None::<&str>);
+                self.vout.append_option(None::<u32>);
+                self.script_sig_hex.append_option(None::<&str>);
+                self.script_sig_asm.append_option(None::<&str>);
+                self.sequence.append(cb_input.sequence);
+                self.coinbase.append(&cb_input.coinbase);
+
+                for witness in cb_input.tx_in_witness.iter().flatten() {
+                    self.tx_in_witness.values().append(witness);
+                }
+                self.tx_in_witness.append();
+
+                // Coinbase inputs have no prevout
+                self.prevout_generated.append_option(None::<bool>);
+                self.prevout_height.append_option(None::<u64>);
+                self.prevout_value.append_option(None::<f64>);
+                self.prevout_script_pub_key_hex.append_option(None::<&str>);
+                self.prevout_script_pub_key_asm.append_option(None::<&str>);
+                self.prevout_script_pub_key_desc.append_option(None::<&str>);
+                self.prevout_script_pub_key_type.append_option(None::<&str>);
+                self.prevout_script_pub_key_address
+                    .append_option(None::<&str>);
+            }
+        }
+    }
+}