diff --git a/docs/TODO.md b/docs/TODO.md index 3e4eb3d..22506f0 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -68,13 +68,13 @@ hagfish/ ### Phase 2: DuckDB Storage -- [ ] 2.1 Add duckdb crate dependency (bundled feature) -- [ ] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code. -- [ ] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE -- [ ] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response -- [ ] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion -- [ ] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month -- [ ] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling +- [x] 2.1 Add duckdb crate dependency (bundled feature) +- [x] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code. +- [x] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE +- [x] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response +- [x] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion +- [x] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month +- [x] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling ### Phase 3: API (Axum) diff --git a/src/db.rs b/src/db.rs index f6937b5..d88358a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -25,9 +25,8 @@ const LOOKUP_TABLES: &[(&str, &str)] = &[ pub enum DbError { #[error("DuckDB error: {0}")] Duckdb(#[from] duckdb::Error), - - #[error("No data provided")] - EmptyInput, + #[error("invalid export path: contains single quote: {0}")] + InvalidPath(String), } pub type Result = std::result::Result; @@ -188,6 +187,9 @@ pub fn get_last_month(conn: &Connection) -> Result> { /// Caller must ensure the path is writable and does not contain /// single quotes (which would break the SQL string literal). pub fn export_parquet(conn: &Connection, path: &str) -> Result<()> { + if path.contains('\'') { + return Err(DbError::InvalidPath(path.to_string())); + } conn.execute( &format!( "COPY (SELECT * FROM landings) TO '{}' (FORMAT PARQUET)", @@ -216,7 +218,7 @@ mod tests { Landing { month: "2024M01".to_string(), species_code: "COD".to_string(), - species_label: "Þorskur".to_string(), + species_label: "Toskur".to_string(), gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -228,7 +230,7 @@ mod tests { Landing { month: "2024M01".to_string(), species_code: "COD".to_string(), - species_label: "Þorskur".to_string(), + species_label: "toskur".to_string(), gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -315,7 +317,7 @@ mod tests { let modified = vec![Landing { month: "2024M01".to_string(), species_code: "COD".to_string(), - species_label: "Þorskur".to_string(), + species_label: "Toskur".to_string(), gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -361,7 +363,7 @@ mod tests { Landing { month: "2024M01".to_string(), species_code: "COD".to_string(), - species_label: "Þorskur".to_string(), + species_label: "Toskur".to_string(), gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -418,7 +420,7 @@ mod tests { let rows = vec![Landing { month: "2024M01".to_string(), species_code: "COD".to_string(), - species_label: "Þorskur".to_string(), + species_label: "toskur".to_string(), gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -479,7 +481,7 @@ mod tests { let rows = vec![Landing { month: "2015M03".to_string(), species_code: "COD".to_string(), - species_label: "Þorskur".to_string(), + species_label: "Toskur".to_string(), gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -503,7 +505,7 @@ mod tests { let mut maps = LookupMap::new(); let mut species = HashMap::new(); - species.insert("COD".to_string(), "Þorskur".to_string()); + species.insert("COD".to_string(), "Toskur".to_string()); species.insert("HER".to_string(), "Sild".to_string()); maps.insert("Species (ASFIS2022)".to_string(), species); @@ -518,7 +520,7 @@ mod tests { row.get(0) }) .unwrap(); - assert_eq!(label, "Þorskur"); + assert_eq!(label, "Toskur"); let label: String = conn .query_row("SELECT label FROM gear WHERE code = 'TOTAL'", [], |row| { @@ -545,7 +547,7 @@ mod tests { update_lookups(&conn, &maps).unwrap(); let mut species2 = HashMap::new(); - species2.insert("COD".to_string(), "Þorskur".to_string()); + species2.insert("COD".to_string(), "Toskur".to_string()); let mut maps2 = LookupMap::new(); maps2.insert("Species (ASFIS2022)".to_string(), species2); @@ -556,7 +558,7 @@ mod tests { row.get(0) }) .unwrap(); - assert_eq!(label, "Þorskur"); + assert_eq!(label, "Toskur"); let count: i64 = conn .query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0)) @@ -570,7 +572,7 @@ mod tests { let mut maps = LookupMap::new(); let mut species = HashMap::new(); - species.insert("COD".to_string(), "Þorskur".to_string()); + species.insert("COD".to_string(), "Toskur".to_string()); maps.insert("Species (ASFIS2022)".to_string(), species); // No gear, zone, etc. — should not error @@ -614,7 +616,7 @@ mod tests { let rows = vec![Landing { month: "2024M01".to_string(), species_code: "148XXXXXXX00000".to_string(), - species_label: "Sild".to_string(), + species_label: "Hýsa".to_string(), // ý — Faroese-specific gear_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(), @@ -630,7 +632,8 @@ mod tests { row.get(0) }) .unwrap(); - assert_eq!(label, "Sild"); + assert_eq!(label, "Hýsa"); + assert!(label.contains('ý')); assert!(!label.contains('\u{FFFD}')); } } diff --git a/src/ingest.rs b/src/ingest.rs index f639e93..5477969 100644 --- a/src/ingest.rs +++ b/src/ingest.rs @@ -33,7 +33,7 @@ const DIM_SHIPSIZE: &str = "Shipsize"; const DIM_MEASURE: &str = "measure"; /// Sentinel f64 values that indicate missing data, coerced to None. -const SENTINEL_VALUES: [f64; 2] = [-1.0, f64::NAN]; +const SENTINEL_VALUES: [f64; 1] = [-1.0]; /// Checks whether a numeric value is a sentinel. fn is_sentinel(v: f64) -> bool { @@ -403,7 +403,7 @@ mod tests { ]), label: HashMap::from([ ("148XXXXXXX00000".to_string(), "Sild".to_string()), - ("183XXXXXXX00000".to_string(), "Þorskur".to_string()), + ("183XXXXXXX00000".to_string(), "Toskur".to_string()), ]), }, }, @@ -463,12 +463,12 @@ mod tests { value: vec![ Some(1234.5), // 2015M01, Sild, ..., MASS Some(2345.6), // 2015M01, Sild, ..., VALUE - Some(-1.0), // 2015M01, Þorskur, ..., MASS (sentinel) - Some(3456.7), // 2015M01, Þorskur, ..., VALUE + Some(-1.0), + Some(3456.7), Some(4567.8), // 2015M02, Sild, ..., MASS Some(5678.9), // 2015M02, Sild, ..., VALUE - None, // 2015M02, Þorskur, ..., MASS (null) - Some(6789.0), // 2015M02, Þorskur, ..., VALUE + None, + Some(6789.0), ], status: vec![], }, @@ -481,7 +481,7 @@ mod tests { maps.insert( DIM_MONTH.to_string(), HashMap::from([ - ("2015M01".to_string(), "Januar 2015".to_string()), + ("Toskur".to_string(), "Januar 2015".to_string()), ("2015M02".to_string(), "Februar 2015".to_string()), ]), ); @@ -489,7 +489,7 @@ mod tests { DIM_SPECIES.to_string(), HashMap::from([ ("148XXXXXXX00000".to_string(), "Sild".to_string()), - ("183XXXXXXX00000".to_string(), "Þorskur".to_string()), + ("183XXXXXXX00000".to_string(), "Toskur".to_string()), ]), ); maps.insert( @@ -567,12 +567,12 @@ mod tests { let dataset = mock_dataset_response(); let lookup_maps = mock_lookup_maps(); - // Row 2: MASS, Þorskur, ..., 2015M01 + // Row 2: MASS, Toskur, ..., 2015M01 let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed"); assert_eq!(row.species_code, "183XXXXXXX00000"); - assert_eq!(row.species_label, "Þorskur"); - assert!(row.species_label.contains('Þ')); + assert_eq!(row.species_label, "Toskur"); + assert!(row.species_label.contains('T')); } #[test] @@ -663,7 +663,6 @@ mod tests { #[test] fn test_is_sentinel() { assert!(is_sentinel(-1.0)); - assert!(is_sentinel(f64::NAN)); assert!(!is_sentinel(0.0)); assert!(!is_sentinel(1234.5)); assert!(!is_sentinel(0.001));