starting phase 5; clap cli

Cargo.lock

@@ -40,6 +40,56 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anstream"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
+[[package]]
+name = "anstyle-parse"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
+dependencies = [
+ "anstyle",
+ "once_cell_polyfill",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.104"
@@ -375,6 +425,52 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "clap"
+version = "4.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.3",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+
 [[package]]
 name = "colored"
 version = "3.1.1"
@@ -756,6 +852,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "axum",
+ "clap",
  "duckdb",
  "mime_guess",
  "mockito",
@@ -1091,6 +1188,12 @@ version = "2.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -1396,6 +1499,12 @@ version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+
 [[package]]
 name = "openssl"
 version = "0.10.81"
@@ -1946,6 +2055,12 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "strum"
 version = "0.27.2"
@@ -2395,6 +2510,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "valuable"
 version = "0.1.1"

Cargo.toml

@@ -21,6 +21,7 @@ rust-embed = "8.12"
 mime_guess = "2"
 tokio-util = { version = "0.7", features = ["io"] }
 tempfile = "3"
+clap = { version = "4", features = ["derive"] }
 
 [dev-dependencies]
 mockito = "1"

config.json

@@ -0,0 +1,7 @@
+{
+  "duckdb_path": "hagfish.db",
+  "bind_address": "127.0.0.1:8090",
+  "data_source_url": "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px",
+  "log_file_path": "hagfish.log",
+  "allowed_origins": []
+}

src/cli.rs

@@ -0,0 +1,32 @@
+use clap::{Parser, Subcommand};
+use std::path::PathBuf;
+
+/// Faroese fisheries data pipeline and dashboard
+#[derive(Parser)]
+#[command(name = "hagfish", version, about)]
+pub struct Cli {
+    /// Path to configuration file
+    #[arg(short, long, value_name = "FILE", default_value = "config.json")]
+    pub config: PathBuf,
+
+    #[command(subcommand)]
+    pub command: Command,
+}
+
+#[derive(Subcommand)]
+pub enum Command {
+    /// Fetch data from Hagstova API and store in DuckDB
+    Ingest {
+        /// Force full backfill instead of incremental
+        #[arg(long)]
+        full: bool,
+    },
+    /// Start the HTTP server
+    Serve,
+    /// Export landings data to Parquet
+    Export {
+        /// Output file path
+        #[arg(short, long, value_name = "FILE")]
+        out: PathBuf,
+    },
+}

src/ingest.rs

@@ -22,7 +22,7 @@ fn is_sentinel(v: f64) -> bool {
         .any(|s| v.total_cmp(s) == std::cmp::Ordering::Equal)
 }
 
-pub async fn fetch_metadata(client: &Client, url: &str) -> Result<LookupMap> {
+pub async fn fetch_metadata(client: &Client, url: &str) -> Result<(LookupMap, MetadataResponse)> {
     info!("Fetching metadata from {}", url);
 
     let resp = client
@@ -63,7 +63,7 @@ pub async fn fetch_metadata(client: &Client, url: &str) -> Result<LookupMap> {
         lookup_map.insert(var.code.clone(), lookup);
     }
 
-    Ok(lookup_map)
+    Ok((lookup_map, meta))
 }
 
 pub fn extract_available_months(meta: &MetadataResponse) -> Vec<String> {

src/main.rs

@@ -1,10 +1,14 @@
 mod api;
+mod cli;
 mod db;
 mod ingest;
 mod types;
 
-use axum::serve;
+use std::path::Path;
 use std::sync::Arc;
+
+use axum::serve;
+use clap::Parser;
 use tokio::net::TcpListener;
 use tracing_subscriber::EnvFilter;
 
@@ -36,15 +40,24 @@ async fn main() -> anyhow::Result<()> {
         .with_env_filter(EnvFilter::from_default_env())
         .init();
 
-    let config = Arc::new(types::Config::default());
+    let cli = cli::Cli::parse();
+    let config = types::Config::load(&cli.config)?;
 
+    match cli.command {
+        cli::Command::Serve => run_serve(&config).await,
+        cli::Command::Ingest { full } => run_ingest(&config, full).await,
+        cli::Command::Export { out } => run_export(&config, &out),
+    }
+}
+
+async fn run_serve(config: &types::Config) -> anyhow::Result<()> {
     cleanup_stale_parquet_files();
 
     let conn = db::init(&config.duckdb_path)?;
 
     let state = api::AppState {
         conn: Arc::new(tokio::sync::Mutex::new(conn)),
-        config: config.clone(),
+        config: Arc::new(config.clone()),
     };
 
     let app = api::build_router(state);
@@ -56,3 +69,61 @@ async fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
+    let conn = db::init(&config.duckdb_path)?;
+    let client = reqwest::Client::new();
+
+    let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url).await?;
+
+    db::update_lookups(&conn, &lookup_maps)?;
+
+    let all_months = ingest::extract_available_months(&metadata);
+
+    let months_to_fetch: Vec<String> = if full {
+        all_months
+    } else {
+        let last = db::get_last_month(&conn)?;
+        match last {
+            Some(last_month) => all_months
+                .into_iter()
+                .filter(|m| m.as_str() > last_month.as_str())
+                .collect(),
+            None => all_months,
+        }
+    };
+
+    if months_to_fetch.is_empty() {
+        tracing::info!("No new months to ingest");
+        return Ok(());
+    }
+
+    tracing::info!("Ingesting {} month(s)", months_to_fetch.len());
+
+    // TODO: 5.2 — batch into chunks of <=12 months when count exceeds 12
+    let query = ingest::build_query(&months_to_fetch);
+    let data = ingest::fetch_data(&client, &config.data_source_url, &query).await?;
+
+    let mut landings = Vec::with_capacity(data.dataset.value.len());
+    for i in 0..data.dataset.value.len() {
+        let row = ingest::parse_row(i, &data, &lookup_maps)?;
+        landings.push(ingest::data_row_to_landing(&row));
+    }
+
+    let count = db::upsert_landings(&conn, &landings)?;
+    tracing::info!("Ingested {} rows across {} month(s)", count, months_to_fetch.len());
+
+    Ok(())
+}
+
+fn run_export(config: &types::Config, out: &Path) -> anyhow::Result<()> {
+    let conn = db::init(&config.duckdb_path)?;
+    let path_str = out
+        .to_str()
+        .ok_or_else(|| anyhow::anyhow!("Output path contains invalid UTF-8"))?;
+
+    db::export_parquet(&conn, path_str)?;
+    tracing::info!("Exported landings to {}", path_str);
+
+    Ok(())
+}

src/types.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::path::Path;
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct Config {
@@ -28,6 +29,20 @@ impl Default for Config {
     }
 }
 
+impl Config {
+    pub fn load(path: &Path) -> anyhow::Result<Self> {
+        if path.exists() {
+            let contents = std::fs::read_to_string(path)?;
+            let config: Config = serde_json::from_str(&contents)?;
+            tracing::info!("Loaded config from {}", path.display());
+            Ok(config)
+        } else {
+            tracing::warn!("Config file {} not found, using defaults", path.display());
+            Ok(Self::default())
+        }
+    }
+}
+
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct MetadataResponse {
     #[serde(default)]