add structured logging

src/main.rs

@@ -13,16 +13,45 @@ use clap::Parser;
 use tokio::net::TcpListener;
 use tracing_subscriber::EnvFilter;
 
-fn init_logging() {
+struct LoggerGuard {
+    _guard: Option<tracing_appender::non_blocking::WorkerGuard>,
+}
+
+fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
     let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
 
-    if std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json") {
-        tracing_subscriber::fmt()
-            .with_env_filter(filter)
-            .json()
-            .init();
-    } else {
-        tracing_subscriber::fmt().with_env_filter(filter).init();
+    let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json");
+
+    match log_file_path {
+        Some(path) => {
+            let file = std::fs::File::create(path).expect("Failed to create log file");
+            let (non_blocking, guard) = tracing_appender::non_blocking(file);
+
+            let subscriber = tracing_subscriber::fmt()
+                .with_env_filter(filter.clone())
+                .with_writer(non_blocking);
+
+            if is_json {
+                subscriber.json().init();
+            } else {
+                subscriber.init();
+            }
+
+            LoggerGuard {
+                _guard: Some(guard),
+            }
+        }
+        None => {
+            let subscriber = tracing_subscriber::fmt().with_env_filter(filter.clone());
+
+            if is_json {
+                subscriber.json().init();
+            } else {
+                subscriber.init();
+            }
+
+            LoggerGuard { _guard: None }
+        }
     }
 }
 
@@ -51,10 +80,23 @@ fn cleanup_stale_parquet_files() {
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    init_logging();
-
     let cli = cli::Cli::parse();
-    let config = types::Config::load(&cli.config)?;
+
+    let config = match &cli.config {
+        Some(path) => types::Config::load(path)?,
+        None => {
+            let default_path = std::path::Path::new("config.json");
+            if default_path.exists() {
+                types::Config::load(default_path)?
+            } else {
+                tracing::warn!("No config file found, using defaults");
+                types::Config::default()
+            }
+        }
+    };
+
+    // Initialize logging AFTER config is loaded so we have log_file_path
+    let _logger_guard = init_logging(config.log_file_path.as_deref());
 
     match cli.command {
         cli::Command::Serve => run_serve(&config).await,
@@ -142,7 +184,6 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
     );
 
     let mut total_rows = 0;
-    let mut total_skipped = 0;
     let mut errors = 0;
 
     for (i, batch) in batches.iter().enumerate() {
@@ -156,7 +197,7 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
             "fetching batch"
         );
 
-        let query = ingest::build_query(batch);
+        let query = ingest::build_query(batch)?;
 
         let data = match ingest::fetch_data(&client, &config.data_source_url, &query).await {
             Ok(d) => d,
@@ -178,11 +219,7 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
         for j in 0..data.dataset.value.len() {
             match ingest::parse_row(j, &data.dataset, &lookup_maps) {
                 Ok(row) => {
-                    if row.value.is_some() {
-                        landings.push(ingest::data_row_to_landing(&row));
-                    } else {
-                        total_skipped += 1;
-                    }
+                    landings.push(ingest::data_row_to_landing(&row));
                 }
                 Err(e) => {
                     tracing::warn!(
@@ -218,11 +255,15 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
             batch = i + 1,
             total_batches = batches.len(),
             rows_inserted = count,
-            null_rows_skipped = total_skipped,
             parse_errors = parse_errors,
             elapsed_ms = batch_elapsed.as_millis() as u64,
             "batch complete"
         );
+
+        // Backoff between batches to avoid rate limiting
+        if i < batches.len() - 1 {
+            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+        }
     }
 
     let elapsed = run_start.elapsed();
@@ -232,7 +273,6 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
         months_requested = months_to_fetch.len(),
         batches = batches.len(),
         total_rows = total_rows,
-        null_rows_skipped = total_skipped,
         errors = errors,
         elapsed_ms = elapsed.as_millis() as u64,
         "ingestion run complete"