FLO/hagfish
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add data validation

Browse Source
This commit is contained in:
bartal-lsn
2026-08-25 23:33:54 +01:00
parent 3220d1d9a4
commit f9cfb99916
4 changed files with 170 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingest.rs
+83 -51
View File
@@ -4,9 +4,9 @@ use std::collections::HashMap;
use tracing::{debug, info};
const DIM_MONTH: &str = "month";
const DIM_SPECIES: &str = "Species (ASFIS2022)";
const DIM_GEAR: &str = "Fishing Gear (ISSCFG2016)";
const DIM_ZONE: &str = "Economic Zone (GEONOM2023)";
pub const DIM_SPECIES: &str = "Species (ASFIS2022)";
pub const DIM_GEAR: &str = "Fishing Gear (ISSCFG2016)";
pub const DIM_ZONE: &str = "Economic Zone (GEONOM2023)";
const DIM_PROCESSING: &str = "Processing (EUMOFAPresentation)";
const DIM_PRESERVATION: &str = "Preservation (EUMOFAPreservation)";
const DIM_SHIPSIZE: &str = "Shipsize";
@@ -70,6 +70,20 @@ pub fn extract_available_months(meta: &MetadataResponse) -> Vec<String> {
.unwrap_or_default()
}
pub fn extract_non_total_values(meta: &MetadataResponse, dim_code: &str) -> Vec<String> {
meta.variables
.iter()
.find(|v| v.code == dim_code)
.map(|v| {
v.values
.iter()
.filter(|val| **val != "TOTAL")
.cloned()
.collect()
})
.unwrap_or_default()
}
pub fn chunk_months(months: &[String], batch_size: usize) -> Vec<Vec<String>> {
if months.is_empty() {
return vec![];
@@ -81,8 +95,13 @@ pub fn chunk_months(months: &[String], batch_size: usize) -> Vec<Vec<String>> {
.collect()
}
pub fn build_query(all_months: &[String]) -> Result<Query> {
if all_months.is_empty() {
pub fn build_query(
months: &[String],
species_codes: &[String],
gear_codes: &[String],
zone_codes: &[String],
) -> Result<Query> {
if months.is_empty() {
return Err(IngestError::InvalidValueCode(
"build_query requires at least one month".to_string(),
));
@@ -94,28 +113,28 @@ pub fn build_query(all_months: &[String]) -> Result<Query> {
code: DIM_MONTH.to_string(),
selection: Selection {
filter: "item".to_string(),
values: all_months.to_vec(),
values: months.to_vec(),
},
},
QueryItem {
code: DIM_SPECIES.to_string(),
selection: Selection {
filter: "all".to_string(),
values: vec!["*".to_string()],
filter: "item".to_string(),
values: species_codes.to_vec(),
},
},
QueryItem {
code: DIM_GEAR.to_string(),
selection: Selection {
filter: "all".to_string(),
values: vec!["*".to_string()],
filter: "item".to_string(),
values: gear_codes.to_vec(),
},
},
QueryItem {
code: DIM_ZONE.to_string(),
selection: Selection {
filter: "all".to_string(),
values: vec!["*".to_string()],
filter: "item".to_string(),
values: zone_codes.to_vec(),
},
},
QueryItem {
@@ -463,53 +482,66 @@ mod tests {
maps
}
#[test]
fn test_build_query_uses_correct_dimension_codes() {
let months = vec!["2015M01".to_string(), "2015M02".to_string()];
let query = build_query(&months).unwrap();
#[test]
fn test_build_query_uses_correct_dimension_codes() {
let months = vec!["2015M01".to_string(), "2015M02".to_string()];
let species = vec!["COD".to_string(), "HER".to_string()];
let gear = vec!["TR1".to_string(), "GN".to_string()];
let zones = vec!["FO".to_string(), "INTL".to_string()];
assert_eq!(query.response.format, "json-stat2");
assert_eq!(query.query.len(), 8);
let query = build_query(&months, &species, &gear, &zones).unwrap();
assert_eq!(query.query[0].code, DIM_MONTH);
assert_eq!(query.query[0].selection.filter, "item");
assert_eq!(query.query[0].selection.values.len(), 2);
assert_eq!(query.response.format, "json-stat2");
assert_eq!(query.query.len(), 8);
assert_eq!(query.query[1].code, DIM_SPECIES);
assert_eq!(query.query[1].selection.filter, "all");
assert_eq!(query.query[0].code, DIM_MONTH);
assert_eq!(query.query[0].selection.filter, "item");
assert_eq!(query.query[0].selection.values.len(), 2);
assert_eq!(query.query[4].code, DIM_PROCESSING);
assert_eq!(query.query[4].selection.filter, "item");
assert_eq!(query.query[4].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[1].code, DIM_SPECIES);
assert_eq!(query.query[1].selection.filter, "item");
assert_eq!(query.query[1].selection.values, vec!["COD", "HER"]);
assert_eq!(query.query[5].code, DIM_PRESERVATION);
assert_eq!(query.query[5].selection.filter, "item");
assert_eq!(query.query[5].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[2].code, DIM_GEAR);
assert_eq!(query.query[2].selection.filter, "item");
assert_eq!(query.query[2].selection.values, vec!["TR1", "GN"]);
assert_eq!(query.query[6].code, DIM_SHIPSIZE);
assert_eq!(query.query[6].selection.filter, "item");
assert_eq!(query.query[6].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[3].code, DIM_ZONE);
assert_eq!(query.query[3].selection.filter, "item");
assert_eq!(query.query[3].selection.values, vec!["FO", "INTL"]);
assert_eq!(query.query[7].code, DIM_MEASURE);
assert!(
query.query[7]
.selection
.values
.contains(&"MASS".to_string())
);
assert!(
query.query[7]
.selection
.values
.contains(&"VALUE".to_string())
);
}
assert_eq!(query.query[4].code, DIM_PROCESSING);
assert_eq!(query.query[4].selection.filter, "item");
assert_eq!(query.query[4].selection.values, vec!["TOTAL"]);
#[test]
fn test_build_query_returns_error_on_empty() {
let result = build_query(&[]);
assert!(result.is_err());
}
assert_eq!(query.query[5].code, DIM_PRESERVATION);
assert_eq!(query.query[5].selection.filter, "item");
assert_eq!(query.query[5].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[6].code, DIM_SHIPSIZE);
assert_eq!(query.query[6].selection.filter, "item");
assert_eq!(query.query[6].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[7].code, DIM_MEASURE);
assert!(
query.query[7]
.selection
.values
.contains(&"MASS".to_string())
);
assert!(
query.query[7]
.selection
.values
.contains(&"VALUE".to_string())
);
}
#[test]
fn test_build_query_returns_error_on_empty() {
let result = build_query(&[], &[], &[], &[]);
assert!(result.is_err());
}
#[test]
fn test_decode_key_indices_basic() {