add data validation

src/main.rs

@@ -1,4 +1,3 @@
-// src/main.rs
 mod api;
 mod cli;
 mod db;
@@ -150,8 +149,24 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
 
     db::update_lookups(&conn, &lookup_maps)?;
 
+    let species_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_SPECIES);
+    let gear_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_GEAR);
+    let zone_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_ZONE);
+
+    tracing::info!(
+        species_count = species_codes.len(),
+        gear_count = gear_codes.len(),
+        zone_count = zone_codes.len(),
+        "Extracted non-TOTAL dimension values for leaf-only query"
+    );
+
     let all_months = ingest::extract_available_months(&metadata);
 
+    if full {
+        conn.execute("DELETE FROM landings", [])?;
+        tracing::info!("Cleared landings table for full re-ingest");
+    }
+
     let months_to_fetch: Vec<String> = if full {
         all_months
     } else {
@@ -203,7 +218,7 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
             "fetching batch"
         );
 
-        let query = ingest::build_query(batch)?;
+        let query = ingest::build_query(batch, &species_codes, &gear_codes, &zone_codes)?;
 
         let data = match ingest::fetch_data(&client, &config.data_source_url, &query).await {
             Ok(d) => d,
@@ -225,6 +240,12 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
         for j in 0..data.dataset.value.len() {
             match ingest::parse_row(j, &data.dataset, &lookup_maps) {
                 Ok(row) => {
+                    if row.species_code == "TOTAL"
+                        || row.gear_code == "TOTAL"
+                        || row.zone_code == "TOTAL"
+                    {
+                        continue;
+                    }
                     landings.push(ingest::data_row_to_landing(row));
                 }
                 Err(e) => {
@@ -331,8 +352,8 @@ mod tests {
             month: "2024M01".to_string(),
             species_code: "COD".to_string(),
             species_label: "Toskur".to_string(),
-            gear_code: "TOTAL".to_string(),
-            zone_code: "TOTAL".to_string(),
+            gear_code: "TR1".to_string(),
+            zone_code: "FO".to_string(),
             processing_code: "TOTAL".to_string(),
             preservation_code: "TOTAL".to_string(),
             shipsize_code: "TOTAL".to_string(),