FLO/hagfish
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase 3 complete

Browse Source
This commit is contained in:
bartal-lsn
2026-08-17 14:08:20 +01:00
parent f639b19c13
commit d52bcf2d64
11 changed files with 1260 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingest.rs
+4 -60
View File
@@ -1,28 +1,10 @@
//! Data ingestion from PX-Web API.
//!
//! Handles metadata fetching, query construction, data retrieval, and
//! parsing of JSON-stat2 responses into structured rows.
//!
//! ## API contract (confirmed 2026-08-16)
//!
//! - GET endpoint: `https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px`
//! - POST endpoint: same URL
//! - Dimension codes are NOT Faroese short names — they use classification
//! identifiers like "Species (ASFIS2022)", "Fishing Gear (ISSCFG2016)".
//! - `role` is null for all variables on this endpoint.
//! - Metadata uses parallel `values`/`valueTexts` string arrays.
//! - Sentinel values: -1.0 indicates missing data.
use crate::types::*;
use reqwest::Client;
use std::collections::HashMap;
use tracing::{debug, info};
/// Language code for PX-Web queries.
const PX_WEB_LANGUAGE: &str = "fo";
/// Dimension codes — confirmed against live API on 2026-08-16.
/// These are the `code` field values from the metadata response.
const DIM_MONTH: &str = "month";
const DIM_SPECIES: &str = "Species (ASFIS2022)";
const DIM_GEAR: &str = "Fishing Gear (ISSCFG2016)";
@@ -32,19 +14,14 @@ const DIM_PRESERVATION: &str = "Preservation (EUMOFAPreservation)";
const DIM_SHIPSIZE: &str = "Shipsize";
const DIM_MEASURE: &str = "measure";
/// Sentinel f64 values that indicate missing data, coerced to None.
const SENTINEL_VALUES: [f64; 1] = [-1.0];
/// Checks whether a numeric value is a sentinel.
fn is_sentinel(v: f64) -> bool {
SENTINEL_VALUES
.iter()
.any(|s| v.total_cmp(s) == std::cmp::Ordering::Equal)
}
/// Fetches metadata from PX-Web API endpoint.
///
/// Returns a `LookupMap` mapping dimension code → (value code → label).
pub async fn fetch_metadata(client: &Client, url: &str) -> Result<LookupMap> {
info!("Fetching metadata from {}", url);
@@ -89,9 +66,6 @@ pub async fn fetch_metadata(client: &Client, url: &str) -> Result<LookupMap> {
Ok(lookup_map)
}
/// Extracts all available months from metadata.
///
/// Uses the "month" dimension code since `role` is null on this endpoint.
pub fn extract_available_months(meta: &MetadataResponse) -> Vec<String> {
meta.variables
.iter()
@@ -100,13 +74,6 @@ pub fn extract_available_months(meta: &MetadataResponse) -> Vec<String> {
.unwrap_or_default()
}
/// Constructs a query body for fetching all landing data.
///
/// Sets categorical dimensions to wildcard ("all" filter), time to explicit
/// months, and measure to MASS + VALUE.
///
/// # Panics
/// Panics if `all_months` is empty.
pub fn build_query(all_months: &[String]) -> Query {
assert!(
!all_months.is_empty(),
@@ -176,7 +143,6 @@ pub fn build_query(all_months: &[String]) -> Query {
}
}
/// Fetches data from PX-Web API using the provided query.
pub async fn fetch_data(client: &Client, url: &str, query: &Query) -> Result<DataResponse> {
let month_count = query
.query
@@ -206,9 +172,6 @@ pub async fn fetch_data(client: &Client, url: &str, query: &Query) -> Result<Dat
Ok(data)
}
/// Decodes a flat row-major index into per-dimension indices.
///
/// JSON-stat2 uses row-major order: the last dimension varies fastest.
fn decode_key_indices(flat_index: usize, key_sizes: &[usize]) -> Vec<usize> {
let mut indices = Vec::with_capacity(key_sizes.len());
let mut remaining = flat_index;
@@ -222,14 +185,6 @@ fn decode_key_indices(flat_index: usize, key_sizes: &[usize]) -> Vec<usize> {
indices
}
/// Parses a single row from a JSON-stat2 response into a `DataRow`.
///
/// Dimension order in the response `id` array determines positional mapping.
/// The expected order (from API metadata) is:
/// measure, species, gear, zone, processing, preservation, shipsize, month
///
/// However, JSON-stat2 `dimension.id` defines the actual order — we read it
/// dynamically and map by dimension code, not by position assumption.
pub fn parse_row(
row_index: usize,
dataset: &DataResponse,
@@ -254,8 +209,6 @@ pub fn parse_row(
)));
}
// Build ordered category lists: for each dimension, extract (code, label)
// pairs sorted by their JSON-stat2 index position.
let category_lists: Vec<Vec<(String, String)>> = dim_order
.iter()
.map(|dim_code| {
@@ -301,7 +254,6 @@ pub fn parse_row(
.map(|(idx, list)| list[idx].clone())
.collect();
// Build a lookup from dimension code → (code, label) for this row
let mut dim_map: HashMap<&str, (String, String)> = HashMap::with_capacity(8);
for (dim_code, values) in dim_order.iter().zip(dimension_values.iter()) {
dim_map.insert(dim_code.as_str(), values.clone());
@@ -323,7 +275,6 @@ pub fn parse_row(
let (shipsize_code, shipsize_label) = get(DIM_SHIPSIZE);
let (measure_code, measure_label) = get(DIM_MEASURE);
// Coerce sentinel values to None
let raw_value = dataset.dataset.value[row_index];
let value = match raw_value {
Some(v) if is_sentinel(v) => None,
@@ -355,7 +306,6 @@ pub fn parse_row(
})
}
/// Converts a `DataRow` to a `Landing` struct for database insertion.
pub fn data_row_to_landing(row: &DataRow) -> Landing {
Landing {
month: row.month.clone(),
@@ -442,8 +392,6 @@ mod tests {
},
);
// Dimension order from API: month, species, gear, zone, processing,
// preservation, shipsize, measure
DataResponse {
dataset: Dataset {
dimension: DimInfo {
@@ -461,12 +409,12 @@ mod tests {
dimensions,
},
value: vec![
Some(1234.5), // 2015M01, Sild, ..., MASS
Some(2345.6), // 2015M01, Sild, ..., VALUE
Some(1234.5),
Some(2345.6),
Some(-1.0),
Some(3456.7),
Some(4567.8), // 2015M02, Sild, ..., MASS
Some(5678.9), // 2015M02, Sild, ..., VALUE
Some(4567.8),
Some(5678.9),
None,
Some(6789.0),
],
@@ -475,7 +423,6 @@ mod tests {
}
}
/// Test fixture: mock lookup maps with Faroese labels.
fn mock_lookup_maps() -> LookupMap {
let mut maps = LookupMap::new();
maps.insert(
@@ -567,7 +514,6 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
// Row 2: MASS, Toskur, ..., 2015M01
let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed");
assert_eq!(row.species_code, "183XXXXXXX00000");
@@ -580,7 +526,6 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
// Row 2 has -1.0 sentinel
let row = parse_row(2, &dataset, &lookup_maps).expect("parse failed");
assert!(
@@ -595,7 +540,6 @@ mod tests {
let dataset = mock_dataset_response();
let lookup_maps = mock_lookup_maps();
// Row 6 has None
let row = parse_row(6, &dataset, &lookup_maps).expect("parse failed");
assert!(row.value.is_none());