From 06886a22867ca34fa901beffeceab34ab29cd87f Mon Sep 17 00:00:00 2001 From: Bartal Laearsson Date: Mon, 17 Aug 2026 18:02:47 +0100 Subject: [PATCH] starting phase 5; clap cli --- Cargo.lock | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + config.json | 7 +++ src/cli.rs | 32 +++++++++++++ src/ingest.rs | 4 +- src/main.rs | 77 ++++++++++++++++++++++++++++++-- src/types.rs | 15 +++++++ 7 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 config.json create mode 100644 src/cli.rs diff --git a/Cargo.lock b/Cargo.lock index 3543712..7053aeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -375,6 +425,52 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "colored" version = "3.1.1" @@ -756,6 +852,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "clap", "duckdb", "mime_guess", "mockito", @@ -1091,6 +1188,12 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -1396,6 +1499,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl" version = "0.10.81" @@ -1946,6 +2055,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.27.2" @@ -2395,6 +2510,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 16fbc4a..38439a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ rust-embed = "8.12" mime_guess = "2" tokio-util = { version = "0.7", features = ["io"] } tempfile = "3" +clap = { version = "4", features = ["derive"] } [dev-dependencies] mockito = "1" diff --git a/config.json b/config.json new file mode 100644 index 0000000..403fe0d --- /dev/null +++ b/config.json @@ -0,0 +1,7 @@ +{ + "duckdb_path": "hagfish.db", + "bind_address": "127.0.0.1:8090", + "data_source_url": "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px", + "log_file_path": "hagfish.log", + "allowed_origins": [] +} diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..6c8f7bd --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,32 @@ +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +/// Faroese fisheries data pipeline and dashboard +#[derive(Parser)] +#[command(name = "hagfish", version, about)] +pub struct Cli { + /// Path to configuration file + #[arg(short, long, value_name = "FILE", default_value = "config.json")] + pub config: PathBuf, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand)] +pub enum Command { + /// Fetch data from Hagstova API and store in DuckDB + Ingest { + /// Force full backfill instead of incremental + #[arg(long)] + full: bool, + }, + /// Start the HTTP server + Serve, + /// Export landings data to Parquet + Export { + /// Output file path + #[arg(short, long, value_name = "FILE")] + out: PathBuf, + }, +} diff --git a/src/ingest.rs b/src/ingest.rs index 1bb238e..82b53cd 100644 --- a/src/ingest.rs +++ b/src/ingest.rs @@ -22,7 +22,7 @@ fn is_sentinel(v: f64) -> bool { .any(|s| v.total_cmp(s) == std::cmp::Ordering::Equal) } -pub async fn fetch_metadata(client: &Client, url: &str) -> Result { +pub async fn fetch_metadata(client: &Client, url: &str) -> Result<(LookupMap, MetadataResponse)> { info!("Fetching metadata from {}", url); let resp = client @@ -63,7 +63,7 @@ pub async fn fetch_metadata(client: &Client, url: &str) -> Result { lookup_map.insert(var.code.clone(), lookup); } - Ok(lookup_map) + Ok((lookup_map, meta)) } pub fn extract_available_months(meta: &MetadataResponse) -> Vec { diff --git a/src/main.rs b/src/main.rs index 96f4102..67e644a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,14 @@ mod api; +mod cli; mod db; mod ingest; mod types; -use axum::serve; +use std::path::Path; use std::sync::Arc; + +use axum::serve; +use clap::Parser; use tokio::net::TcpListener; use tracing_subscriber::EnvFilter; @@ -36,15 +40,24 @@ async fn main() -> anyhow::Result<()> { .with_env_filter(EnvFilter::from_default_env()) .init(); - let config = Arc::new(types::Config::default()); + let cli = cli::Cli::parse(); + let config = types::Config::load(&cli.config)?; + match cli.command { + cli::Command::Serve => run_serve(&config).await, + cli::Command::Ingest { full } => run_ingest(&config, full).await, + cli::Command::Export { out } => run_export(&config, &out), + } +} + +async fn run_serve(config: &types::Config) -> anyhow::Result<()> { cleanup_stale_parquet_files(); let conn = db::init(&config.duckdb_path)?; let state = api::AppState { conn: Arc::new(tokio::sync::Mutex::new(conn)), - config: config.clone(), + config: Arc::new(config.clone()), }; let app = api::build_router(state); @@ -56,3 +69,61 @@ async fn main() -> anyhow::Result<()> { Ok(()) } + +async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> { + let conn = db::init(&config.duckdb_path)?; + let client = reqwest::Client::new(); + + let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url).await?; + + db::update_lookups(&conn, &lookup_maps)?; + + let all_months = ingest::extract_available_months(&metadata); + + let months_to_fetch: Vec = if full { + all_months + } else { + let last = db::get_last_month(&conn)?; + match last { + Some(last_month) => all_months + .into_iter() + .filter(|m| m.as_str() > last_month.as_str()) + .collect(), + None => all_months, + } + }; + + if months_to_fetch.is_empty() { + tracing::info!("No new months to ingest"); + return Ok(()); + } + + tracing::info!("Ingesting {} month(s)", months_to_fetch.len()); + + // TODO: 5.2 — batch into chunks of <=12 months when count exceeds 12 + let query = ingest::build_query(&months_to_fetch); + let data = ingest::fetch_data(&client, &config.data_source_url, &query).await?; + + let mut landings = Vec::with_capacity(data.dataset.value.len()); + for i in 0..data.dataset.value.len() { + let row = ingest::parse_row(i, &data, &lookup_maps)?; + landings.push(ingest::data_row_to_landing(&row)); + } + + let count = db::upsert_landings(&conn, &landings)?; + tracing::info!("Ingested {} rows across {} month(s)", count, months_to_fetch.len()); + + Ok(()) +} + +fn run_export(config: &types::Config, out: &Path) -> anyhow::Result<()> { + let conn = db::init(&config.duckdb_path)?; + let path_str = out + .to_str() + .ok_or_else(|| anyhow::anyhow!("Output path contains invalid UTF-8"))?; + + db::export_parquet(&conn, path_str)?; + tracing::info!("Exported landings to {}", path_str); + + Ok(()) +} diff --git a/src/types.rs b/src/types.rs index 71cb12f..57044b0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::path::Path; #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Config { @@ -28,6 +29,20 @@ impl Default for Config { } } +impl Config { + pub fn load(path: &Path) -> anyhow::Result { + if path.exists() { + let contents = std::fs::read_to_string(path)?; + let config: Config = serde_json::from_str(&contents)?; + tracing::info!("Loaded config from {}", path.display()); + Ok(config) + } else { + tracing::warn!("Config file {} not found, using defaults", path.display()); + Ok(Self::default()) + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct MetadataResponse { #[serde(default)]