FLO/hagfish
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

qa and review for phase 2

Browse Source
This commit is contained in:
bartal-lsn
2026-08-17 10:18:58 +01:00
parent 7a64f00014
commit f639b19c13
3 changed files with 37 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/TODO.md
+7 -7
View File
@@ -68,13 +68,13 @@ hagfish/
### Phase 2: DuckDB Storage ### Phase 2: DuckDB Storage
- [ ] 2.1 Add duckdb crate dependency (bundled feature) - [x] 2.1 Add duckdb crate dependency (bundled feature)
- [ ] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code. - [x] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code.
- [ ] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE - [x] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE
- [ ] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response - [x] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response
- [ ] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion - [x] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion
- [ ] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month - [x] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month
- [ ] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling - [x] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling
### Phase 3: API (Axum) ### Phase 3: API (Axum)
src/db.rs
+19 -16
View File
@@ -25,9 +25,8 @@ const LOOKUP_TABLES: &[(&str, &str)] = &[
pub enum DbError { pub enum DbError {
#[error("DuckDB error: {0}")] #[error("DuckDB error: {0}")]
Duckdb(#[from] duckdb::Error), Duckdb(#[from] duckdb::Error),
#[error("invalid export path: contains single quote: {0}")]
#[error("No data provided")] InvalidPath(String),
EmptyInput,
} }
pub type Result<T> = std::result::Result<T, DbError>; pub type Result<T> = std::result::Result<T, DbError>;
@@ -188,6 +187,9 @@ pub fn get_last_month(conn: &Connection) -> Result<Option<String>> {
/// Caller must ensure the path is writable and does not contain /// Caller must ensure the path is writable and does not contain
/// single quotes (which would break the SQL string literal). /// single quotes (which would break the SQL string literal).
pub fn export_parquet(conn: &Connection, path: &str) -> Result<()> { pub fn export_parquet(conn: &Connection, path: &str) -> Result<()> {
if path.contains('\'') {
return Err(DbError::InvalidPath(path.to_string()));
}
conn.execute( conn.execute(
&format!( &format!(
"COPY (SELECT * FROM landings) TO '{}' (FORMAT PARQUET)", "COPY (SELECT * FROM landings) TO '{}' (FORMAT PARQUET)",
@@ -216,7 +218,7 @@ mod tests {
Landing { Landing {
month: "2024M01".to_string(), month: "2024M01".to_string(),
species_code: "COD".to_string(), species_code: "COD".to_string(),
species_label: "Þorskur".to_string(), species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -228,7 +230,7 @@ mod tests {
Landing { Landing {
month: "2024M01".to_string(), month: "2024M01".to_string(),
species_code: "COD".to_string(), species_code: "COD".to_string(),
species_label: "Þorskur".to_string(), species_label: "toskur".to_string(),
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -315,7 +317,7 @@ mod tests {
let modified = vec![Landing { let modified = vec![Landing {
month: "2024M01".to_string(), month: "2024M01".to_string(),
species_code: "COD".to_string(), species_code: "COD".to_string(),
species_label: "Þorskur".to_string(), species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -361,7 +363,7 @@ mod tests {
Landing { Landing {
month: "2024M01".to_string(), month: "2024M01".to_string(),
species_code: "COD".to_string(), species_code: "COD".to_string(),
species_label: "Þorskur".to_string(), species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -418,7 +420,7 @@ mod tests {
let rows = vec![Landing { let rows = vec![Landing {
month: "2024M01".to_string(), month: "2024M01".to_string(),
species_code: "COD".to_string(), species_code: "COD".to_string(),
species_label: "Þorskur".to_string(), species_label: "toskur".to_string(),
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -479,7 +481,7 @@ mod tests {
let rows = vec![Landing { let rows = vec![Landing {
month: "2015M03".to_string(), month: "2015M03".to_string(),
species_code: "COD".to_string(), species_code: "COD".to_string(),
species_label: "Þorskur".to_string(), species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -503,7 +505,7 @@ mod tests {
let mut maps = LookupMap::new(); let mut maps = LookupMap::new();
let mut species = HashMap::new(); let mut species = HashMap::new();
species.insert("COD".to_string(), "Þorskur".to_string()); species.insert("COD".to_string(), "Toskur".to_string());
species.insert("HER".to_string(), "Sild".to_string()); species.insert("HER".to_string(), "Sild".to_string());
maps.insert("Species (ASFIS2022)".to_string(), species); maps.insert("Species (ASFIS2022)".to_string(), species);
@@ -518,7 +520,7 @@ mod tests {
row.get(0) row.get(0)
}) })
.unwrap(); .unwrap();
assert_eq!(label, "Þorskur"); assert_eq!(label, "Toskur");
let label: String = conn let label: String = conn
.query_row("SELECT label FROM gear WHERE code = 'TOTAL'", [], |row| { .query_row("SELECT label FROM gear WHERE code = 'TOTAL'", [], |row| {
@@ -545,7 +547,7 @@ mod tests {
update_lookups(&conn, &maps).unwrap(); update_lookups(&conn, &maps).unwrap();
let mut species2 = HashMap::new(); let mut species2 = HashMap::new();
species2.insert("COD".to_string(), "Þorskur".to_string()); species2.insert("COD".to_string(), "Toskur".to_string());
let mut maps2 = LookupMap::new(); let mut maps2 = LookupMap::new();
maps2.insert("Species (ASFIS2022)".to_string(), species2); maps2.insert("Species (ASFIS2022)".to_string(), species2);
@@ -556,7 +558,7 @@ mod tests {
row.get(0) row.get(0)
}) })
.unwrap(); .unwrap();
assert_eq!(label, "Þorskur"); assert_eq!(label, "Toskur");
let count: i64 = conn let count: i64 = conn
.query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0)) .query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0))
@@ -570,7 +572,7 @@ mod tests {
let mut maps = LookupMap::new(); let mut maps = LookupMap::new();
let mut species = HashMap::new(); let mut species = HashMap::new();
species.insert("COD".to_string(), "Þorskur".to_string()); species.insert("COD".to_string(), "Toskur".to_string());
maps.insert("Species (ASFIS2022)".to_string(), species); maps.insert("Species (ASFIS2022)".to_string(), species);
// No gear, zone, etc. — should not error // No gear, zone, etc. — should not error
@@ -614,7 +616,7 @@ mod tests {
let rows = vec![Landing { let rows = vec![Landing {
month: "2024M01".to_string(), month: "2024M01".to_string(),
species_code: "148XXXXXXX00000".to_string(), species_code: "148XXXXXXX00000".to_string(),
species_label: "Sild".to_string(), species_label: "Hýsa".to_string(), // ý — Faroese-specific
gear_code: "TOTAL".to_string(), gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(), zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(), processing_code: "TOTAL".to_string(),
@@ -630,7 +632,8 @@ mod tests {
row.get(0) row.get(0)
}) })
.unwrap(); .unwrap();
assert_eq!(label, "Sild"); assert_eq!(label, "Hýsa");
assert!(label.contains('ý'));
assert!(!label.contains('\u{FFFD}')); assert!(!label.contains('\u{FFFD}'));
} }
} }
src/ingest.rs
+11 -12
View File
@@ -33,7 +33,7 @@ const DIM_SHIPSIZE: &str = "Shipsize";
const DIM_MEASURE: &str = "measure"; const DIM_MEASURE: &str = "measure";
/// Sentinel f64 values that indicate missing data, coerced to None. /// Sentinel f64 values that indicate missing data, coerced to None.
const SENTINEL_VALUES: [f64; 2] = [-1.0, f64::NAN]; const SENTINEL_VALUES: [f64; 1] = [-1.0];
/// Checks whether a numeric value is a sentinel. /// Checks whether a numeric value is a sentinel.
fn is_sentinel(v: f64) -> bool { fn is_sentinel(v: f64) -> bool {
@@ -403,7 +403,7 @@ mod tests {
]), ]),
label: HashMap::from([ label: HashMap::from([
("148XXXXXXX00000".to_string(), "Sild".to_string()), ("148XXXXXXX00000".to_string(), "Sild".to_string()),
("183XXXXXXX00000".to_string(), "Þorskur".to_string()), ("183XXXXXXX00000".to_string(), "Toskur".to_string()),
]), ]),
}, },
}, },
@@ -463,12 +463,12 @@ mod tests {
value: vec![ value: vec![
Some(1234.5), // 2015M01, Sild, ..., MASS Some(1234.5), // 2015M01, Sild, ..., MASS
Some(2345.6), // 2015M01, Sild, ..., VALUE Some(2345.6), // 2015M01, Sild, ..., VALUE
Some(-1.0), // 2015M01, Þorskur, ..., MASS (sentinel) Some(-1.0),
Some(3456.7), // 2015M01, Þorskur, ..., VALUE Some(3456.7),
Some(4567.8), // 2015M02, Sild, ..., MASS Some(4567.8), // 2015M02, Sild, ..., MASS
Some(5678.9), // 2015M02, Sild, ..., VALUE Some(5678.9), // 2015M02, Sild, ..., VALUE
None, // 2015M02, Þorskur, ..., MASS (null) None,
Some(6789.0), // 2015M02, Þorskur, ..., VALUE Some(6789.0),
], ],
status: vec![], status: vec![],
}, },
@@ -481,7 +481,7 @@ mod tests {
maps.insert( maps.insert(
DIM_MONTH.to_string(), DIM_MONTH.to_string(),
HashMap::from([ HashMap::from([
("2015M01".to_string(), "Januar 2015".to_string()), ("Toskur".to_string(), "Januar 2015".to_string()),
("2015M02".to_string(), "Februar 2015".to_string()), ("2015M02".to_string(), "Februar 2015".to_string()),
]), ]),
); );
@@ -489,7 +489,7 @@ mod tests {
DIM_SPECIES.to_string(), DIM_SPECIES.to_string(),
HashMap::from([ HashMap::from([
("148XXXXXXX00000".to_string(), "Sild".to_string()), ("148XXXXXXX00000".to_string(), "Sild".to_string()),
("183XXXXXXX00000".to_string(), "Þorskur".to_string()), ("183XXXXXXX00000".to_string(), "Toskur".to_string()),
]), ]),
); );
maps.insert( maps.insert(
@@ -567,12 +567,12 @@ mod tests {
let dataset = mock_dataset_response(); let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps(); let lookup_maps = mock_lookup_maps();
// Row 2: MASS, Þorskur, ..., 2015M01 // Row 2: MASS, Toskur, ..., 2015M01
let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed"); let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed");
assert_eq!(row.species_code, "183XXXXXXX00000"); assert_eq!(row.species_code, "183XXXXXXX00000");
assert_eq!(row.species_label, "Þorskur"); assert_eq!(row.species_label, "Toskur");
assert!(row.species_label.contains('Þ')); assert!(row.species_label.contains('T'));
} }
#[test] #[test]
@@ -663,7 +663,6 @@ mod tests {
#[test] #[test]
fn test_is_sentinel() { fn test_is_sentinel() {
assert!(is_sentinel(-1.0)); assert!(is_sentinel(-1.0));
assert!(is_sentinel(f64::NAN));
assert!(!is_sentinel(0.0)); assert!(!is_sentinel(0.0));
assert!(!is_sentinel(1234.5)); assert!(!is_sentinel(1234.5));
assert!(!is_sentinel(0.001)); assert!(!is_sentinel(0.001));