phase 1 complete

2026-08-16 22:22:59 +01:00
parent 24d5a4e57a
commit 5ec60a71c3
6 changed files with 2707 additions and 341 deletions
@@ -0,0 +1,192 @@
+
+# API Documentation
+
+This document describes the PX-Web API used by hagfish to fetch Faroese fisheries statistics from the official Statbank.
+
+## EndpointBase URL: https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px
+- **GET**: Returns metadata (table structure, dimension codes, labels)
+- **POST**: Returns data in JSON-stat2 format
+
+## Metadata Request (GET)
+
+### Requestbash
+curl -s "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
+### Response Structurejson
+{
+"title": "AVR01010 Avreiðingar í nøgd og virði...",
+"variables": [
+{
+"code": "measure",
+"text": "mát",
+"role": null,
+"values": ["MASS", "VALUE"],
+"valueTexts": ["Nøgd", "Virði"]
+}
+]
+}
+### Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `title` | string | Table name in Faroese |
+| `variables[].code` | string | Dimension identifier (used in queries) |
+| `variables[].text` | string | Human-readable label in Faroese |
+| `variables[].role` | null/string | Classification (always `null` on this endpoint) |
+| `variables[].values` | string[] | Valid dimension codes |
+| `variables[].valueTexts` | string[] | Display labels (parallel with `values`) |
+
+### Verified Dimension Codes
+
+| Variable Code | Faroese Label | Values |
+|---------------|---------------|--------|
+| `measure` | mát | `MASS`, `VALUE` |
+| `Species (ASFIS2022)` | Fiskaslag (ASFIS2022) | `TOTAL`, `148XXXXXXX00000`, ... (72 total) |
+| `Fishing Gear (ISSCFG2016)` | Reiðskapur (ISSCFG2016) | `TOTAL`, ... (12 total) |
+| `Economic Zone (GEONOM2023)` | Búskapar øki (GEONOM2023) | `TOTAL`, ... (10 total) |
+| `Processing (EUMOFAPresentation)` | Virking (EUMOFAPresentation) | `TOTAL`, ... (6 total) |
+| `Preservation (EUMOFAPreservation)` | Viðgerð (EUMOFAPreservation) | `TOTAL`, ... (9 total) |
+| `Shipsize` | Skipastødd | `TOTAL`, ... (11 total) |
+| `month` | mánaður | `2015M01` through `2026M05` (137 months) |
+
+**Note**: `role` is always `null`. Time dimension identification must use `code == "month"`.
+
+---
+
+## Data Request (POST)
+
+### Request Body Formatjson
+{
+"query": [
+{
+"code": "dimension_code",
+"selection": {
+"filter": "item",
+"values": ["value1", "value2"]
+}
+}
+],
+"response": {
+"format": "json-stat2"
+}
+}
+### Filter Types
+
+| Filter | Values | Description |
+|--------|--------|-------------|
+| `item` | explicit codes | Select specific values |
+| `all` | `["*"]` | Wildcard — select all values |
+| `top` | `["TOP_N"]` | Top N values by magnitude |
+
+### Example Querybash
+curl -s -X POST 
+-H "Content-Type: application/json" 
+-d '{
+"query": [
+{"code": "month", "selection": {"filter": "item", "values": ["2024M01", "2024M02"]}},
+{"code": "Species (ASFIS2022)", "selection": {"filter": "all", "values": [""]}},
+{"code": "Fishing Gear (ISSCFG2016)", "selection": {"filter": "all", "values": [""]}},
+{"code": "Economic Zone (GEONOM2023)", "selection": {"filter": "all", "values": ["*"]}},
+{"code": "Processing (EUMOFAPresentation)", "selection": {"filter": "item", "values": ["TOTAL"]}},
+{"code": "Preservation (EUMOFAPreservation)", "selection": {"filter": "item", "values": ["TOTAL"]}},
+{"code": "Shipsize", "selection": {"filter": "item", "values": ["TOTAL"]}},
+{"code": "measure", "selection": {"filter": "item", "values": ["MASS", "VALUE"]}}
+],
+"response": {"format": "json-stat2"}
+}' 
+"https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
+### Cell Limit
+
+- Maximum cells per query: ~8,000,000
+- Recommended maximum: 1,000,000 for reliability
+- To fetch full dataset, paginate by month or use smaller dimension selections
+
+---
+
+## Response Format (JSON-stat2)
+
+### Structurejson
+{
+"class": "dataset",
+"label": "...",
+"id": ["measure", "Species (ASFIS2022)", ..., "month"],
+"size": [2, 72, 12, 10, 1, 1, 1, 2],
+"dimension": {
+"measure": {
+"label": "mát",
+"category": {
+"index": {"MASS": 0, "VALUE": 1},
+"label": {"MASS": "Nøgd", "VALUE": "Virði"}
+}
+}
+},
+"value": [73653738, 1234567, ...]
+}
+### Fields
+
+| Field | Description |
+|-------|-------------|
+| `id` | Array of dimension codes in order (defines cube layout) |
+| `size` | Cardinality per dimension (parallel with `id`) |
+| `dimension.<code>.category.index` | Maps value code → numeric position in dimension |
+| `dimension.<code>.category.label` | Maps value code → display text |
+| `value` | Flattened array of measurements (row-major order) |
+
+### Row-Major Index Decoding
+
+Values are stored in row-major order: the last dimension (`month`) varies fastest.
+
+Given `size = [2, 72, 12, 10, 1, 1, 1, 2]`:
+
+- Flat index `0` → `[0,0,0,0,0,0,0,0]` = measure=MASS, species=TOTAL, ..., month=2024M01
+- Flat index `1` → `[0,0,0,0,0,0,0,1]` = measure=MASS, species=TOTAL, ..., month=2024M02
+- Flat index `2` → `[0,0,0,0,0,0,1,0]` = measure=MASS, species=SPECIES_1, ..., month=2024M01
+
+Decoding algorithm (reverse modulo):rust
+fn decode(flat_index: usize, sizes: &[usize]) -> Vec<usize> {
+let mut indices = Vec::new();
+let mut remaining = flat_index;
+for &size in sizes.iter().rev() {
+indices.push(remaining % size);
+remaining /= size;
+}
+indices.reverse();
+indices
+}
+### Sentinel Values
+
+| Value | Meaning |
+|-------|---------|
+| `-1.0` | Missing/no data (treated as `NULL`) |
+| `null` | Missing/no data (already nullable in JSON) |
+
+---
+
+## Known Constraints
+
+1. **Cell limit**: ~8 million cells per query
+2. **Rate limits**: Unknown — assume reasonable backoff for large downloads
+3. **Language**: API responds in requested language (`fo` or `en`)
+4. **Time zone**: No timezone specified — treat timestamps as local
+5. **Updates**: Data updated irregularly — metadata timestamp in response header
+
+---
+
+## Integration Checklist
+
+- [x] Metadata endpoint reachable via GET
+- [x] POST queries return valid JSON-stat2
+- [x] Dimension codes match `variables[].code` from metadata
+- [x] Row-major index decoding verified
+- [x] Sentinel value coercion (`-1.0` → `None`) implemented
+- [x] Unicode (Faroese characters) handled correctly
+- [ ] Incremental ingestion logic tested
+- [ ] Parquet export tested
+- [ ] Error handling for network timeouts implemented
+
+---
+
+## References
+
+- Official PxWeb documentation: https://pxweb.github.io/docs/
+- JSON-stat2 specification: http://json-stat.org/format/
+- Hagstova Føroya: https://www.hagstova.fo/
@@ -59,12 +59,12 @@ hagfish/

 ### Phase 1: Types & Ingestion

- [ ] 1.1 Define types in types.rs: MetadataResponse, VariableMeta, DataResponse, DataRow, Query, Selection, QueryItem, Config — all with serde derives
- [ ] 1.2 Implement ingest.rs::fetch_metadata(url) — GET request, parse JSON, return HashMap<(variable_code, value_code), faroese_label>
- [ ] 1.3 Implement ingest.rs::build_query(months: &[String]) — construct POST body with all species/gear/zones set to "*", processing/preservation/shipsize set to TOTAL, measure set to both MASS and VALUE
- [ ] 1.4 Implement ingest.rs::fetch_data(url, query) — POST request, parse JSON-stat2 response, return Vec<DataRow>
- [ ] 1.5 Implement ingest.rs::parse_row(row, lookup_maps) — decode key[] positions into labeled Landing struct. Handle "-" → None.
- [ ] 1.6 Write unit tests: mock JSON-stat2 response, verify key-to-label mapping, verify "-" handling, verify Faroese Unicode characters in species names (ð, á, í, ý, ø, ó)
+- [x] 1.1 Define types in types.rs: MetadataResponse, VariableMeta, DataResponse, DataRow, Query, Selection, QueryItem, Config — all with serde derives
+- [x] 1.2 Implement ingest.rs::fetch_metadata(url) — GET request, parse JSON, return HashMap<(variable_code, value_code), faroese_label>
+- [x] 1.3 Implement ingest.rs::build_query(months: &[String]) — construct POST body with all species/gear/zones set to "*", processing/preservation/shipsize set to TOTAL, measure set to both MASS and VALUE
+- [x] 1.4 Implement ingest.rs::fetch_data(url, query) — POST request, parse JSON-stat2 response, return Vec<DataRow>
+- [x] 1.5 Implement ingest.rs::parse_row(row, lookup_maps) — decode key[] positions into labeled Landing struct. Handle "-" → None.
+- [x] 1.6 Write unit tests: mock JSON-stat2 response, verify key-to-label mapping, verify "-" handling, verify Faroese Unicode characters in species names (ð, á, í, ý, ø, ó)

 ### Phase 2: DuckDB Storage

@@ -1,3 +1,3 @@
 fn main() {
-    println!("Hello, world!");
+    println!("hagfish — not yet wired. Run tests with: cargo test");
 }
@@ -20,63 +20,132 @@ impl Default for Config {
        Self {
            duckdb_path: "hagfish.db".to_string(),
            bind_address: "127.0.0.1:8090".to_string(),
-            data_source_url: "https://statbank.hagstova.fo/data/api/table/fisknv_md".to_string(),
+            data_source_url: "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
+                .to_string(),
            log_file_path: Some("hagfish.log".to_string()),
        }
    }
 }

+// ---------------------------------------------------------------------------
+// Metadata types (GET response)
+// ---------------------------------------------------------------------------
+
 /// Metadata response from PX-Web API GET request.
-/// Contains variable definitions and value codes with labels.
+///
+/// Confirmed structure from live API:
+/// ```json
+/// {
+///   "title": "AVR01010 ...",
+///   "variables": [
+///     {
+///       "code": "measure",
+///       "text": "mát",
+///       "role": null,
+///       "values": ["MASS", "VALUE"],
+///       "valueTexts": ["Nøgd", "Virði"]
+///     }
+///   ]
+/// }
+/// ```
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct MetadataResponse {
+    #[serde(default)]
+    pub title: Option<String>,
    pub variables: Vec<VariableMeta>,
-    pub values: HashMap<String, Vec<ValueMeta>>,
 }

 /// Variable metadata describing a dimension in the PX-Web table.
+///
+/// PxWeb v1 uses parallel string arrays for codes and labels.
+/// The `role` field is null on this endpoint — do not rely on it.
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct VariableMeta {
-    pub id: String,
+    /// Variable identifier used in queries (e.g. "month", "measure",
+    /// "Species (ASFIS2022)")
+    pub code: String,
+    /// Human-readable label in the queried language
+    #[serde(default)]
    pub text: String,
-    pub role: String,
-    /// Position in the key array (0-indexed)
-    #[serde(rename = "keyPosition", skip_serializing_if = "Option::is_none")]
-    pub key_position: Option<usize>,
+    /// Role classification — null on this endpoint
+    #[serde(default)]
+    pub role: Option<String>,
+    /// Code values for this variable (parallel with valueTexts)
+    #[serde(default)]
+    pub values: Vec<String>,
+    /// Display labels for each value (parallel with values)
+    #[serde(default, rename = "valueTexts")]
+    pub value_texts: Vec<String>,
+    /// Catch-all for unknown fields
+    #[serde(flatten)]
+    pub extra: HashMap<String, serde_json::Value>,
 }

-/// Value metadata: code → display label mapping for a variable.
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct ValueMeta {
-    pub code: String,
-    pub text: String,
+impl VariableMeta {
+    /// Build a code→label lookup map from the parallel arrays.
+    pub fn lookup_map(&self) -> HashMap<String, String> {
+        self.values
+            .iter()
+            .zip(self.value_texts.iter())
+            .map(|(code, label)| (code.clone(), label.clone()))
+            .collect()
+    }
 }

+// ---------------------------------------------------------------------------
+// Query types (POST request body)
+// ---------------------------------------------------------------------------
+
 /// Query body sent to PX-Web API POST endpoint.
+///
+/// Confirmed format:
+/// ```json
+/// {
+///   "query": [{"code": "month", "selection": {"filter": "item", "values": [...]}}],
+///   "response": {"format": "json-stat2"}
+/// }
+/// ```
 #[derive(Debug, Clone, Serialize)]
 pub struct Query {
    pub query: Vec<QueryItem>,
-    pub language: String,
+    pub response: QueryResponse,
 }

 /// Single query item representing a dimension selection.
 #[derive(Debug, Clone, Serialize)]
 pub struct QueryItem {
-    pub id: String,
+    pub code: String,
+    pub selection: Selection,
+}
+
+/// Selection within a query item.
+#[derive(Debug, Clone, Serialize)]
+pub struct Selection {
+    /// "item" for explicit values, "all" for wildcard, "top" for top-N
+    pub filter: String,
+    /// Values to select. For "all" filter, use ["*"].
    pub values: Vec<String>,
 }

-/// Selection helper for building queries programmatically.
-#[derive(Debug, Clone)]
-pub struct Selection {
-    /// Dimension name (matches variable.id from metadata)
-    pub dimension: String,
-    /// Codes to include, or "*" for all
-    pub codes: Vec<String>,
+/// Response format specification.
+#[derive(Debug, Clone, Serialize)]
+pub struct QueryResponse {
+    pub format: String,
 }

+impl Default for QueryResponse {
+    fn default() -> Self {
+        Self {
+            format: "json-stat2".to_string(),
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Data response types (JSON-stat2)
+// ---------------------------------------------------------------------------
+
 /// Raw data response from PX-Web API POST request.
-/// JSON-stat2 format with metadata and data sections.
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct DataResponse {
    pub dataset: Dataset,
@@ -87,24 +156,52 @@ pub struct DataResponse {
 pub struct Dataset {
    pub dimension: DimInfo,
    pub value: Vec<Option<f64>>,
+    /// Status codes per value (optional)
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub status: Vec<String>,
 }

-/// Dimension metadata describing key layout.
+/// Dimension metadata in JSON-stat2 format.
+///
+/// Uses a flat `id` array for ordering and a `size` array for cardinality.
+/// Each dimension is keyed by its code in the `dimensions` map.
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct DimInfo {
-    #[serde(rename = "key")]
-    pub keys: Vec<String>,
+    /// Ordered dimension IDs (defines cube layout)
+    #[serde(default)]
+    pub id: Vec<String>,
+    /// Size of each dimension
+    #[serde(default)]
+    pub size: Vec<usize>,
+    /// One entry per dimension, keyed by dimension code
+    #[serde(flatten)]
+    pub dimensions: HashMap<String, Dimension>,
+}
+
+/// A single dimension in the JSON-stat2 response.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct Dimension {
+    /// Display label
+    #[serde(default)]
+    pub label: String,
+    /// Category info with index and label maps
    pub category: CategoryInfo,
 }

-/// Category info containing value lists for each dimension.
+/// Category info containing index and label maps.
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct CategoryInfo {
-    pub label: HashMap<String, Vec<String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub index: Option<HashMap<String, Vec<usize>>>,
+    /// Maps category code → numeric position in the dimension
+    pub index: HashMap<String, usize>,
+    /// Maps category code → display label
+    #[serde(default)]
+    pub label: HashMap<String, String>,
 }

+// ---------------------------------------------------------------------------
+// Internal data models
+// ---------------------------------------------------------------------------
+
 /// Decoded row of landing data with labeled dimensions.
 #[derive(Debug, Clone)]
 pub struct DataRow {
@@ -126,7 +223,7 @@ pub struct DataRow {
    pub value: Option<f64>,
 }

-/// Structured representation of a single landing record.
+/// Structured representation of a single landing record for DB insertion.
 #[derive(Debug, Clone)]
 pub struct Landing {
    pub month: String,
@@ -141,6 +238,10 @@ pub struct Landing {
    pub value: Option<f64>,
 }

+// ---------------------------------------------------------------------------
+// Errors
+// ---------------------------------------------------------------------------
+
 /// Error types for ingestion module.
 #[derive(Debug, thiserror::Error)]
 pub enum IngestError {
@@ -153,18 +254,21 @@ pub enum IngestError {
    #[error("Missing dimension in response: {0}")]
    MissingDimension(String),

-    #[error("Invalid value code: {0}")]
+    #[error("Invalid value: {0}")]
    InvalidValueCode(String),

    #[error("API returned empty data set")]
    EmptyDataset,

-    #[error("Unicode decode error: {0}")]
-    UnicodeError(String),
+    #[error("Dimension '{0}' not found in metadata")]
+    DimensionNotFound(String),
 }

-/// Lookup map for decoding key arrays into labeled values.
-/// Keyed by dimension name, contains code → label mappings.
+// ---------------------------------------------------------------------------
+// Type aliases
+// ---------------------------------------------------------------------------
+
+/// Lookup map: dimension_code → (value_code → display_label).
 pub type LookupMap = HashMap<String, HashMap<String, String>>;

 /// Result alias using custom error type.