diff --git a/build.rs b/build.rs index 64cb08d..d9bda18 100644 --- a/build.rs +++ b/build.rs @@ -4,16 +4,21 @@ //! This ensures the binaries are always consistent with the circuit crate version //! and eliminates the need to manually run `quantus developer build-circuits`. //! -//! Outputs are written to `OUT_DIR` (required by cargo) and then copied to -//! `generated-bins/` in the project root for runtime access — but only during -//! normal builds, **not** during `cargo publish` verification where modifying the -//! source directory is forbidden. +//! Outputs are written to `OUT_DIR` (required by cargo) and, during local source +//! builds only, linked/copied to `generated-bins/` in the project root. When the +//! crate is consumed via `cargo install` or `cargo publish` verification, the +//! manifest lives under `~/.cargo/registry/src/` or `target/package/` +//! respectively — locations the installed binary cannot reach — so the project +//! copy is skipped. Installed binaries regenerate the files on first run via +//! `crate::bins::ensure_bins_dir()`. //! //! Set `SKIP_CIRCUIT_BUILD=1` to skip circuit generation (useful for CI jobs //! that don't need the circuits, like clippy/doc checks). use std::{env, path::Path, time::Instant}; +include!("src/bins_consts.rs"); + /// Compute Poseidon2 hash of bytes and return hex string fn poseidon_hex(data: &[u8]) -> String { let hash = qp_poseidon_core::hash_bytes(data); @@ -48,9 +53,8 @@ fn main() { let build_output_dir = Path::new(&out_dir).join("generated-bins"); let num_leaf_proofs: usize = env::var("QP_NUM_LEAF_PROOFS") - .unwrap_or_else(|_| "16".to_string()) - .parse() - .expect("QP_NUM_LEAF_PROOFS must be a valid usize"); + .map(|v| v.parse().expect("QP_NUM_LEAF_PROOFS must be a valid usize")) + .unwrap_or(DEFAULT_NUM_LEAF_PROOFS); // Don't emit any rerun-if-changed directives - this forces the build script // to run on every build. Circuit generation is fast enough in release mode. @@ -73,6 +77,10 @@ fn main() { ) .expect("Failed to generate circuit binaries"); + let pkg_version = env::var("CARGO_PKG_VERSION").expect("CARGO_PKG_VERSION not set"); + std::fs::write(build_output_dir.join(VERSION_MARKER), &pkg_version) + .expect("Failed to write version marker"); + let elapsed = start.elapsed(); println!( "cargo:warning=[quantus-cli] ZK circuit binaries generated in {:.2}s", @@ -88,10 +96,15 @@ fn main() { print_bin_hash(&build_output_dir, "aggregated_verifier.bin"); print_bin_hash(&build_output_dir, "aggregated_prover.bin"); - // Copy bins to project root for runtime access, but NOT during `cargo publish` - // verification (manifest_dir is inside target/package/ in that case). + // Copy bins to project root for runtime access, but only during local source + // builds — never during `cargo publish` verification (manifest_dir is inside + // `target/package/`) nor during `cargo install` (manifest_dir is inside + // `.cargo/registry/src/`). In those cases the installed binary can't see the + // project dir; runtime lazy-generation takes over instead. let project_bins = Path::new(&manifest_dir).join("generated-bins"); - if !manifest_dir.contains("target/package/") { + let is_source_build = + !manifest_dir.contains("target/package/") && !manifest_dir.contains(".cargo/registry/src"); + if is_source_build { // Prefer a symlink to avoid copying large prover binaries on every build. // If symlink creation fails (e.g. on filesystems without symlink support), // fall back to copying and surface errors. diff --git a/src/bins.rs b/src/bins.rs new file mode 100644 index 0000000..a526449 --- /dev/null +++ b/src/bins.rs @@ -0,0 +1,119 @@ +//! Circuit binaries path resolution and lazy generation. +//! +//! The CLI needs access to several large ZK-circuit files (`prover.bin`, +//! `verifier.bin`, `aggregated_*.bin`, etc.). During `cargo build`/`cargo install` +//! these are produced by `build.rs` into `$OUT_DIR/generated-bins/`, but +//! `cargo install` does not copy build-script outputs alongside the installed +//! executable. To make installed binaries self-sufficient, this module resolves +//! a persistent storage location and regenerates the binaries there on demand. +//! +//! Resolution order: +//! 1. `QUANTUS_BINS_DIR` env var (explicit override). +//! 2. `./generated-bins/` in the current directory (local dev). +//! 3. `~/.quantus/generated-bins/` (default for installed binaries). + +use crate::{ + error::{QuantusError, Result}, + log_print, log_success, +}; +use std::path::{Path, PathBuf}; + +include!("bins_consts.rs"); + +/// Environment variable used to override the bins directory. +pub const BINS_DIR_ENV: &str = "QUANTUS_BINS_DIR"; + +/// Files that must be present for all wormhole operations to succeed. +const REQUIRED_FILES: &[&str] = &[ + "prover.bin", + "verifier.bin", + "common.bin", + "aggregated_prover.bin", + "aggregated_verifier.bin", + "aggregated_common.bin", + "dummy_proof.bin", + "config.json", +]; + +/// Resolve the path where circuit binaries should live. +/// +/// This never generates anything; see [`ensure_bins_dir`] for the full +/// resolve-and-generate flow. +pub fn resolve_bins_dir() -> PathBuf { + if let Ok(dir) = std::env::var(BINS_DIR_ENV) { + return PathBuf::from(dir); + } + + let cwd_dir = PathBuf::from("generated-bins"); + if cwd_dir.join("config.json").exists() { + return cwd_dir; + } + + user_bins_dir() +} + +/// Location used for auto-generated binaries on installed systems. +fn user_bins_dir() -> PathBuf { + dirs::home_dir() + .expect("Could not determine home directory for ~/.quantus/generated-bins") + .join(".quantus") + .join("generated-bins") +} + +/// Resolve the bins directory and generate any missing circuit binaries. +/// +/// Safe to call multiple times; regeneration only happens when the target is +/// empty, partially populated, or was produced by a different CLI version. +pub fn ensure_bins_dir() -> Result { + let dir = resolve_bins_dir(); + + if is_ready(&dir) { + return Ok(dir); + } + + let num_leaf_proofs = env_num_leaf_proofs(); + generate(&dir, num_leaf_proofs)?; + Ok(dir) +} + +fn is_ready(dir: &Path) -> bool { + if !REQUIRED_FILES.iter().all(|f| dir.join(f).exists()) { + return false; + } + match std::fs::read_to_string(dir.join(VERSION_MARKER)) { + Ok(v) => v.trim() == env!("CARGO_PKG_VERSION"), + Err(_) => false, + } +} + +fn env_num_leaf_proofs() -> usize { + std::env::var("QP_NUM_LEAF_PROOFS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_NUM_LEAF_PROOFS) +} + +fn generate(dir: &Path, num_leaf_proofs: usize) -> Result<()> { + std::fs::create_dir_all(dir).map_err(|e| { + QuantusError::Generic(format!("Failed to create bins directory {}: {}", dir.display(), e)) + })?; + + log_print!(""); + log_print!("🛠️ Generating ZK circuit binaries (first-time setup, ~30s)..."); + log_print!(" Target: {}", dir.display()); + log_print!(" num_leaf_proofs: {}", num_leaf_proofs); + + let start = std::time::Instant::now(); + qp_wormhole_circuit_builder::generate_all_circuit_binaries(dir, true, num_leaf_proofs, None) + .map_err(|e| { + QuantusError::Generic(format!("Failed to generate circuit binaries: {}", e)) + })?; + + std::fs::write(dir.join(VERSION_MARKER), env!("CARGO_PKG_VERSION")) + .map_err(|e| QuantusError::Generic(format!("Failed to write version marker: {}", e)))?; + + let elapsed = start.elapsed(); + log_success!("Circuit binaries ready in {:.1}s", elapsed.as_secs_f64()); + log_print!(""); + Ok(()) +} diff --git a/src/bins_consts.rs b/src/bins_consts.rs new file mode 100644 index 0000000..068a8f4 --- /dev/null +++ b/src/bins_consts.rs @@ -0,0 +1,8 @@ +/// Filename of the marker recording which CLI version produced the bins. +/// When the CLI is upgraded this mismatches and the runtime regenerates. +/// Shared by `build.rs` and `crate::bins` via `include!`. +const VERSION_MARKER: &str = ".quantus-cli-version"; + +/// Number of leaf proofs aggregated into a single batch (default for both +/// build-time generation and runtime lazy generation). +const DEFAULT_NUM_LEAF_PROOFS: usize = 16; diff --git a/src/cli/wormhole.rs b/src/cli/wormhole.rs index 2313b84..8e882c1 100644 --- a/src/cli/wormhole.rs +++ b/src/cli/wormhole.rs @@ -30,7 +30,6 @@ use qp_zk_circuits_common::{ }; use rand::RngCore; use sp_core::crypto::{AccountId32, Ss58Codec}; -use std::path::Path; use subxt::{ blocks::Block, ext::{ @@ -1122,13 +1121,10 @@ async fn aggregate_proofs( ) -> crate::error::Result<()> { use qp_wormhole_aggregator::aggregator::{AggregationBackend, CircuitType, Layer0Aggregator}; - use std::path::Path; - log_print!("Aggregating {} proofs...", proof_files.len()); - // Load config first to validate and calculate padding needs - let bins_dir = Path::new("generated-bins"); - let agg_config = CircuitBinsConfig::load(bins_dir).map_err(|e| { + let bins_dir = crate::bins::ensure_bins_dir()?; + let agg_config = CircuitBinsConfig::load(&bins_dir).map_err(|e| { crate::error::QuantusError::Generic(format!( "Failed to load circuit bins config from {:?}: {}", bins_dir, e @@ -1148,7 +1144,7 @@ async fn aggregate_proofs( log_print!(" Loading aggregator and generating {} dummy proofs...", num_padding_proofs); - let mut aggregator = Layer0Aggregator::new(bins_dir).map_err(|e| { + let mut aggregator = Layer0Aggregator::new(&bins_dir).map_err(|e| { crate::error::QuantusError::Generic(format!( "Failed to load aggregator from pre-built bins: {}", e @@ -2016,9 +2012,8 @@ async fn run_multiround( log_print!("=================================================="); log_print!(""); - // Load aggregation config from generated-bins/config.json - let bins_dir = Path::new("generated-bins"); - let agg_config = CircuitBinsConfig::load(bins_dir).map_err(|e| { + let bins_dir = crate::bins::ensure_bins_dir()?; + let agg_config = CircuitBinsConfig::load(&bins_dir).map_err(|e| { crate::error::QuantusError::Generic(format!("Failed to load aggregation config: {}", e)) })?; @@ -2333,8 +2328,7 @@ async fn generate_proof( asset_id: NATIVE_ASSET_ID, }; - // Generate proof using wormhole_lib - let bins_dir = Path::new("generated-bins"); + let bins_dir = crate::bins::ensure_bins_dir()?; let result = wormhole_lib::generate_proof( &input, &bins_dir.join("prover.bin"), @@ -2435,11 +2429,9 @@ async fn verify_aggregated_and_get_events( let proof_bytes = read_hex_proof_file_to_bytes(proof_file)?; - // Verify locally before submitting on-chain log_verbose!("Verifying aggregated proof locally before on-chain submission..."); - let bins_dir = Path::new("generated-bins"); + let bins_dir = crate::bins::ensure_bins_dir()?; - // Log circuit binary hashes for debugging let common_bytes = std::fs::read(bins_dir.join("aggregated_common.bin")).map_err(|e| { crate::error::QuantusError::Generic(format!("Failed to read aggregated_common.bin: {}", e)) })?; @@ -2601,7 +2593,7 @@ async fn parse_proof_file( log_print!("Proof size: {} bytes", proof_bytes.len()); - let bins_dir = Path::new("generated-bins"); + let bins_dir = crate::bins::ensure_bins_dir()?; if aggregated { // Load aggregated verifier @@ -2818,9 +2810,8 @@ async fn run_dissolve( crate::error::QuantusError::Generic(format!("Failed to create output directory: {}", e)) })?; - // Load aggregation config - let bins_dir = std::path::Path::new("generated-bins"); - let agg_config = CircuitBinsConfig::load(bins_dir).map_err(|e| { + let bins_dir = crate::bins::ensure_bins_dir()?; + let agg_config = CircuitBinsConfig::load(&bins_dir).map_err(|e| { crate::error::QuantusError::Generic(format!( "Failed to load aggregation circuit config: {}", e @@ -3149,8 +3140,8 @@ async fn run_collect_rewards( match step { "derive" => log_print!("{}", format!("Step 1: {}...", details).bright_yellow()), "query" => log_print!("{}", format!("Step 2: {}...", details).bright_yellow()), - "nullifiers" => log_print!("{}", format!("Step 3: {}...", details).bright_yellow()), - "connect" => log_print!("{}", format!("Step 4: {}...", details).bright_yellow()), + "connect" => log_print!("{}", format!("Step 3: {}...", details).bright_yellow()), + "nullifiers" => log_print!("{}", format!("Step 4: {}...", details).bright_yellow()), "proofs" => log_print!("{}", format!("Step 5: {}...", details).bright_yellow()), "submit" => log_print!("{}", format!("Step 6: {}...", details).bright_yellow()), _ => log_print!(" {}: {}", step, details), @@ -3178,7 +3169,7 @@ async fn run_collect_rewards( destination_address: destination_address.clone(), subsquid_url, node_url: node_url.to_string(), - bins_dir: "generated-bins".to_string(), + bins_dir: crate::bins::ensure_bins_dir()?.to_string_lossy().into_owned(), amount: amount_planck, dry_run, at_block, @@ -3222,8 +3213,8 @@ async fn run_collect_rewards( fn aggregate_proofs_to_file(proof_files: &[String], output_file: &str) -> crate::error::Result<()> { use qp_wormhole_aggregator::aggregator::Layer0Aggregator; - let bins_dir = std::path::Path::new("generated-bins"); - let mut aggregator = Layer0Aggregator::new(bins_dir).map_err(|e| { + let bins_dir = crate::bins::ensure_bins_dir()?; + let mut aggregator = Layer0Aggregator::new(&bins_dir).map_err(|e| { crate::error::QuantusError::Generic(format!("Failed to create aggregator: {}", e)) })?; diff --git a/src/lib.rs b/src/lib.rs index 4bc9e9c..a07f455 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ //! It can be used as a dependency in other Rust projects that need to interact with //! the Quantus blockchain. +pub mod bins; pub mod chain; pub mod cli; pub mod collect_rewards_lib; diff --git a/src/main.rs b/src/main.rs index 872c886..4b90cb1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ use clap::Parser; use colored::Colorize; +mod bins; mod chain; mod cli; mod collect_rewards_lib;