phase 1 complete
This commit is contained in:
+140
-36
@@ -20,63 +20,132 @@ impl Default for Config {
|
||||
Self {
|
||||
duckdb_path: "hagfish.db".to_string(),
|
||||
bind_address: "127.0.0.1:8090".to_string(),
|
||||
data_source_url: "https://statbank.hagstova.fo/data/api/table/fisknv_md".to_string(),
|
||||
data_source_url: "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
|
||||
.to_string(),
|
||||
log_file_path: Some("hagfish.log".to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metadata types (GET response)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Metadata response from PX-Web API GET request.
|
||||
/// Contains variable definitions and value codes with labels.
|
||||
///
|
||||
/// Confirmed structure from live API:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "title": "AVR01010 ...",
|
||||
/// "variables": [
|
||||
/// {
|
||||
/// "code": "measure",
|
||||
/// "text": "mát",
|
||||
/// "role": null,
|
||||
/// "values": ["MASS", "VALUE"],
|
||||
/// "valueTexts": ["Nøgd", "Virði"]
|
||||
/// }
|
||||
/// ]
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct MetadataResponse {
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
pub variables: Vec<VariableMeta>,
|
||||
pub values: HashMap<String, Vec<ValueMeta>>,
|
||||
}
|
||||
|
||||
/// Variable metadata describing a dimension in the PX-Web table.
|
||||
///
|
||||
/// PxWeb v1 uses parallel string arrays for codes and labels.
|
||||
/// The `role` field is null on this endpoint — do not rely on it.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct VariableMeta {
|
||||
pub id: String,
|
||||
/// Variable identifier used in queries (e.g. "month", "measure",
|
||||
/// "Species (ASFIS2022)")
|
||||
pub code: String,
|
||||
/// Human-readable label in the queried language
|
||||
#[serde(default)]
|
||||
pub text: String,
|
||||
pub role: String,
|
||||
/// Position in the key array (0-indexed)
|
||||
#[serde(rename = "keyPosition", skip_serializing_if = "Option::is_none")]
|
||||
pub key_position: Option<usize>,
|
||||
/// Role classification — null on this endpoint
|
||||
#[serde(default)]
|
||||
pub role: Option<String>,
|
||||
/// Code values for this variable (parallel with valueTexts)
|
||||
#[serde(default)]
|
||||
pub values: Vec<String>,
|
||||
/// Display labels for each value (parallel with values)
|
||||
#[serde(default, rename = "valueTexts")]
|
||||
pub value_texts: Vec<String>,
|
||||
/// Catch-all for unknown fields
|
||||
#[serde(flatten)]
|
||||
pub extra: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Value metadata: code → display label mapping for a variable.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ValueMeta {
|
||||
pub code: String,
|
||||
pub text: String,
|
||||
impl VariableMeta {
|
||||
/// Build a code→label lookup map from the parallel arrays.
|
||||
pub fn lookup_map(&self) -> HashMap<String, String> {
|
||||
self.values
|
||||
.iter()
|
||||
.zip(self.value_texts.iter())
|
||||
.map(|(code, label)| (code.clone(), label.clone()))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Query types (POST request body)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Query body sent to PX-Web API POST endpoint.
|
||||
///
|
||||
/// Confirmed format:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "query": [{"code": "month", "selection": {"filter": "item", "values": [...]}}],
|
||||
/// "response": {"format": "json-stat2"}
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Query {
|
||||
pub query: Vec<QueryItem>,
|
||||
pub language: String,
|
||||
pub response: QueryResponse,
|
||||
}
|
||||
|
||||
/// Single query item representing a dimension selection.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct QueryItem {
|
||||
pub id: String,
|
||||
pub code: String,
|
||||
pub selection: Selection,
|
||||
}
|
||||
|
||||
/// Selection within a query item.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Selection {
|
||||
/// "item" for explicit values, "all" for wildcard, "top" for top-N
|
||||
pub filter: String,
|
||||
/// Values to select. For "all" filter, use ["*"].
|
||||
pub values: Vec<String>,
|
||||
}
|
||||
|
||||
/// Selection helper for building queries programmatically.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Selection {
|
||||
/// Dimension name (matches variable.id from metadata)
|
||||
pub dimension: String,
|
||||
/// Codes to include, or "*" for all
|
||||
pub codes: Vec<String>,
|
||||
/// Response format specification.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct QueryResponse {
|
||||
pub format: String,
|
||||
}
|
||||
|
||||
impl Default for QueryResponse {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
format: "json-stat2".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Data response types (JSON-stat2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Raw data response from PX-Web API POST request.
|
||||
/// JSON-stat2 format with metadata and data sections.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DataResponse {
|
||||
pub dataset: Dataset,
|
||||
@@ -87,24 +156,52 @@ pub struct DataResponse {
|
||||
pub struct Dataset {
|
||||
pub dimension: DimInfo,
|
||||
pub value: Vec<Option<f64>>,
|
||||
/// Status codes per value (optional)
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub status: Vec<String>,
|
||||
}
|
||||
|
||||
/// Dimension metadata describing key layout.
|
||||
/// Dimension metadata in JSON-stat2 format.
|
||||
///
|
||||
/// Uses a flat `id` array for ordering and a `size` array for cardinality.
|
||||
/// Each dimension is keyed by its code in the `dimensions` map.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DimInfo {
|
||||
#[serde(rename = "key")]
|
||||
pub keys: Vec<String>,
|
||||
/// Ordered dimension IDs (defines cube layout)
|
||||
#[serde(default)]
|
||||
pub id: Vec<String>,
|
||||
/// Size of each dimension
|
||||
#[serde(default)]
|
||||
pub size: Vec<usize>,
|
||||
/// One entry per dimension, keyed by dimension code
|
||||
#[serde(flatten)]
|
||||
pub dimensions: HashMap<String, Dimension>,
|
||||
}
|
||||
|
||||
/// A single dimension in the JSON-stat2 response.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Dimension {
|
||||
/// Display label
|
||||
#[serde(default)]
|
||||
pub label: String,
|
||||
/// Category info with index and label maps
|
||||
pub category: CategoryInfo,
|
||||
}
|
||||
|
||||
/// Category info containing value lists for each dimension.
|
||||
/// Category info containing index and label maps.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CategoryInfo {
|
||||
pub label: HashMap<String, Vec<String>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub index: Option<HashMap<String, Vec<usize>>>,
|
||||
/// Maps category code → numeric position in the dimension
|
||||
pub index: HashMap<String, usize>,
|
||||
/// Maps category code → display label
|
||||
#[serde(default)]
|
||||
pub label: HashMap<String, String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal data models
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Decoded row of landing data with labeled dimensions.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DataRow {
|
||||
@@ -126,7 +223,7 @@ pub struct DataRow {
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
/// Structured representation of a single landing record.
|
||||
/// Structured representation of a single landing record for DB insertion.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Landing {
|
||||
pub month: String,
|
||||
@@ -141,6 +238,10 @@ pub struct Landing {
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Errors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Error types for ingestion module.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum IngestError {
|
||||
@@ -153,18 +254,21 @@ pub enum IngestError {
|
||||
#[error("Missing dimension in response: {0}")]
|
||||
MissingDimension(String),
|
||||
|
||||
#[error("Invalid value code: {0}")]
|
||||
#[error("Invalid value: {0}")]
|
||||
InvalidValueCode(String),
|
||||
|
||||
#[error("API returned empty data set")]
|
||||
EmptyDataset,
|
||||
|
||||
#[error("Unicode decode error: {0}")]
|
||||
UnicodeError(String),
|
||||
#[error("Dimension '{0}' not found in metadata")]
|
||||
DimensionNotFound(String),
|
||||
}
|
||||
|
||||
/// Lookup map for decoding key arrays into labeled values.
|
||||
/// Keyed by dimension name, contains code → label mappings.
|
||||
// ---------------------------------------------------------------------------
|
||||
// Type aliases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Lookup map: dimension_code → (value_code → display_label).
|
||||
pub type LookupMap = HashMap<String, HashMap<String, String>>;
|
||||
|
||||
/// Result alias using custom error type.
|
||||
|
||||
Reference in New Issue
Block a user