FLO/hagfish
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase 5 complete, sending to QA and code review

Browse Source
This commit is contained in:
bartal-lsn
2026-08-17 21:20:35 +01:00
parent 06886a2286
commit 4ef3ca706a
6 changed files with 309 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingest.rs
+137 -45
View File
@@ -3,8 +3,6 @@ use reqwest::Client;
use std::collections::HashMap;
use tracing::{debug, info};
const PX_WEB_LANGUAGE: &str = "fo";
const DIM_MONTH: &str = "month";
const DIM_SPECIES: &str = "Species (ASFIS2022)";
const DIM_GEAR: &str = "Fishing Gear (ISSCFG2016)";
@@ -74,6 +72,17 @@ pub fn extract_available_months(meta: &MetadataResponse) -> Vec<String> {
.unwrap_or_default()
}
pub fn chunk_months(months: &[String], batch_size: usize) -> Vec<Vec<String>> {
if months.is_empty() {
return vec![];
}
months
.chunks(batch_size)
.map(|chunk| chunk.to_vec())
.collect()
}
pub fn build_query(all_months: &[String]) -> Query {
assert!(
!all_months.is_empty(),
@@ -113,22 +122,22 @@ pub fn build_query(all_months: &[String]) -> Query {
QueryItem {
code: DIM_PROCESSING.to_string(),
selection: Selection {
filter: "all".to_string(),
values: vec!["*".to_string()],
filter: "item".to_string(),
values: vec!["TOTAL".to_string()],
},
},
QueryItem {
code: DIM_PRESERVATION.to_string(),
selection: Selection {
filter: "all".to_string(),
values: vec!["*".to_string()],
filter: "item".to_string(),
values: vec!["TOTAL".to_string()],
},
},
QueryItem {
code: DIM_SHIPSIZE.to_string(),
selection: Selection {
filter: "all".to_string(),
values: vec!["*".to_string()],
filter: "item".to_string(),
values: vec!["TOTAL".to_string()],
},
},
QueryItem {
@@ -161,7 +170,20 @@ pub async fn fetch_data(client: &Client, url: &str, query: &Query) -> Result<Dat
return Err(IngestError::ApiStatus { status, body });
}
let data: DataResponse = resp.json().await?;
let body = resp.text().await?;
let preview = body.chars().take(500).collect::<String>();
let data: DataResponse = match serde_json::from_str(&body) {
Ok(d) => d,
Err(e) => {
tracing::error!(
error = %e,
body_preview = %preview,
"failed to deserialize JSON-stat2 response"
);
return Err(IngestError::JsonError(e));
}
};
if data.dataset.value.is_empty() {
return Err(IngestError::EmptyDataset);
@@ -185,13 +207,8 @@ fn decode_key_indices(flat_index: usize, key_sizes: &[usize]) -> Vec<usize> {
indices
}
pub fn parse_row(
row_index: usize,
dataset: &DataResponse,
lookup_maps: &LookupMap,
) -> Result<DataRow> {
let dim_info = &dataset.dataset.dimension;
let dim_order = &dim_info.id;
pub fn parse_row(row_index: usize, dataset: &Dataset, lookup_maps: &LookupMap) -> Result<DataRow> {
let dim_order = &dataset.id;
if dim_order.len() != 8 {
return Err(IngestError::MissingDimension(format!(
@@ -201,18 +218,18 @@ pub fn parse_row(
)));
}
if row_index >= dataset.dataset.value.len() {
if row_index >= dataset.value.len() {
return Err(IngestError::InvalidValueCode(format!(
"Row index {} out of bounds (max {})",
row_index,
dataset.dataset.value.len()
dataset.value.len()
)));
}
let category_lists: Vec<Vec<(String, String)>> = dim_order
.iter()
.map(|dim_code| {
let dim = dim_info.dimensions.get(dim_code).ok_or_else(|| {
let dim = dataset.dimension.get(dim_code).ok_or_else(|| {
IngestError::MissingDimension(format!(
"Dimension '{}' not found in response",
dim_code
@@ -275,7 +292,7 @@ pub fn parse_row(
let (shipsize_code, shipsize_label) = get(DIM_SHIPSIZE);
let (measure_code, measure_label) = get(DIM_MEASURE);
let raw_value = dataset.dataset.value[row_index];
let raw_value = dataset.value[row_index];
let value = match raw_value {
Some(v) if is_sentinel(v) => None,
other => other,
@@ -394,20 +411,18 @@ mod tests {
DataResponse {
dataset: Dataset {
dimension: DimInfo {
id: vec![
DIM_MONTH.to_string(),
DIM_SPECIES.to_string(),
DIM_GEAR.to_string(),
DIM_ZONE.to_string(),
DIM_PROCESSING.to_string(),
DIM_PRESERVATION.to_string(),
DIM_SHIPSIZE.to_string(),
DIM_MEASURE.to_string(),
],
size: vec![2, 2, 1, 1, 1, 1, 1, 2],
dimensions,
},
id: vec![
DIM_MONTH.to_string(),
DIM_SPECIES.to_string(),
DIM_GEAR.to_string(),
DIM_ZONE.to_string(),
DIM_PROCESSING.to_string(),
DIM_PRESERVATION.to_string(),
DIM_SHIPSIZE.to_string(),
DIM_MEASURE.to_string(),
],
size: vec![2, 2, 1, 1, 1, 1, 1, 2],
dimension: dimensions,
value: vec![
Some(1234.5),
Some(2345.6),
@@ -418,7 +433,8 @@ mod tests {
None,
Some(6789.0),
],
status: vec![],
extra_fields: HashMap::new(),
status_array: vec![],
},
}
}
@@ -464,6 +480,18 @@ mod tests {
assert_eq!(query.query[1].code, DIM_SPECIES);
assert_eq!(query.query[1].selection.filter, "all");
assert_eq!(query.query[4].code, DIM_PROCESSING);
assert_eq!(query.query[4].selection.filter, "item");
assert_eq!(query.query[4].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[5].code, DIM_PRESERVATION);
assert_eq!(query.query[5].selection.filter, "item");
assert_eq!(query.query[5].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[6].code, DIM_SHIPSIZE);
assert_eq!(query.query[6].selection.filter, "item");
assert_eq!(query.query[6].selection.values, vec!["TOTAL"]);
assert_eq!(query.query[7].code, DIM_MEASURE);
assert!(
query.query[7]
@@ -499,7 +527,7 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
let row = parse_row(0, &dataset, &lookup_maps).expect("parse failed");
let row = parse_row(0, &dataset.dataset, &lookup_maps).expect("parse failed");
assert_eq!(row.month, "2015M01");
assert_eq!(row.species_code, "148XXXXXXX00000");
@@ -514,7 +542,7 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed");
let row = parse_row(2, &dataset.dataset, &lookup_maps).expect("parse failed");
assert_eq!(row.species_code, "183XXXXXXX00000");
assert_eq!(row.species_label, "Toskur");
@@ -526,7 +554,7 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed");
let row = parse_row(2, &dataset.dataset, &lookup_maps).expect("parse failed");
assert!(
row.value.is_none(),
@@ -540,7 +568,7 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
let row = parse_row(6, &dataset, &lookup_maps).expect("parse failed");
let row = parse_row(6, &dataset.dataset, &lookup_maps).expect("parse failed");
assert!(row.value.is_none());
}
@@ -550,7 +578,7 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
let result = parse_row(100, &dataset, &lookup_maps);
let result = parse_row(100, &dataset.dataset, &lookup_maps);
assert!(result.is_err());
match result {
@@ -562,11 +590,11 @@ mod tests {
#[test]
fn test_parse_row_wrong_dimension_count() {
let mut dataset = mock_dataset_response();
dataset.dataset.dimension.id.pop();
dataset.dataset.dimension.size.pop();
dataset.dataset.id.pop();
dataset.dataset.size.pop();
let lookup_maps = mock_lookup_maps();
let result = parse_row(0, &dataset, &lookup_maps);
let result = parse_row(0, &dataset.dataset, &lookup_maps);
assert!(result.is_err());
}
@@ -575,7 +603,7 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
let row = parse_row(0, &dataset, &lookup_maps).expect("parse failed");
let row = parse_row(0, &dataset.dataset, &lookup_maps).expect("parse failed");
let landing = data_row_to_landing(&row);
assert_eq!(landing.month, row.month);
@@ -591,7 +619,7 @@ mod tests {
let lookup_maps = mock_lookup_maps();
for i in 0..8 {
let row = parse_row(i, &dataset, &lookup_maps);
let row = parse_row(i, &dataset.dataset, &lookup_maps);
assert!(row.is_ok(), "Row {} failed", i);
if let Ok(r) = row {
@@ -672,4 +700,68 @@ mod tests {
assert_eq!(map.get("VALUE"), Some(&"Virði".to_string()));
assert_eq!(map.len(), 2);
}
#[test]
fn test_chunk_months_basic() {
let months = vec![
"2024M01".to_string(),
"2024M02".to_string(),
"2024M03".to_string(),
"2024M04".to_string(),
"2024M05".to_string(),
];
let batches = chunk_months(&months, 3);
assert_eq!(batches.len(), 2);
assert_eq!(batches[0], vec!["2024M01", "2024M02", "2024M03"]);
assert_eq!(batches[1], vec!["2024M04", "2024M05"]);
}
#[test]
fn test_chunk_months_exact_division() {
let months = vec![
"2024M01".to_string(),
"2024M02".to_string(),
"2024M03".to_string(),
"2024M04".to_string(),
"2024M05".to_string(),
"2024M06".to_string(),
];
let batches = chunk_months(&months, 3);
assert_eq!(batches.len(), 2);
assert_eq!(batches[0].len(), 3);
assert_eq!(batches[1].len(), 3);
}
#[test]
fn test_chunk_months_empty() {
let months: Vec<String> = vec![];
let batches = chunk_months(&months, 12);
assert!(batches.is_empty());
}
#[test]
fn test_chunk_months_single_item() {
let months = vec!["2024M01".to_string()];
let batches = chunk_months(&months, 12);
assert_eq!(batches.len(), 1);
assert_eq!(batches[0], vec!["2024M01"]);
}
#[test]
fn test_chunk_months_large_batch() {
let months: Vec<String> = (1..=37)
.map(|m| format!("2024M{:02}", ((m - 1) % 12) + 1))
.collect();
let batches = chunk_months(&months, 12);
assert_eq!(batches.len(), 4);
assert_eq!(batches[0].len(), 12);
assert_eq!(batches[1].len(), 12);
assert_eq!(batches[2].len(), 12);
assert_eq!(batches[3].len(), 1);
}
}