phase 3 complete
This commit is contained in:
+64
-93
@@ -1,12 +1,6 @@
|
||||
//! Type definitions for hagfish data structures.
|
||||
//!
|
||||
//! These types represent the PX-Web API contract and internal data models.
|
||||
//! All public types derive Serialize/Deserialize for JSON (de)serialization.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Configuration loaded from config.json on startup.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Config {
|
||||
pub duckdb_path: String,
|
||||
@@ -27,27 +21,6 @@ impl Default for Config {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metadata types (GET response)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Metadata response from PX-Web API GET request.
|
||||
///
|
||||
/// Confirmed structure from live API:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "title": "AVR01010 ...",
|
||||
/// "variables": [
|
||||
/// {
|
||||
/// "code": "measure",
|
||||
/// "text": "mát",
|
||||
/// "role": null,
|
||||
/// "values": ["MASS", "VALUE"],
|
||||
/// "valueTexts": ["Nøgd", "Virði"]
|
||||
/// }
|
||||
/// ]
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct MetadataResponse {
|
||||
#[serde(default)]
|
||||
@@ -55,34 +28,22 @@ pub struct MetadataResponse {
|
||||
pub variables: Vec<VariableMeta>,
|
||||
}
|
||||
|
||||
/// Variable metadata describing a dimension in the PX-Web table.
|
||||
///
|
||||
/// PxWeb v1 uses parallel string arrays for codes and labels.
|
||||
/// The `role` field is null on this endpoint — do not rely on it.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct VariableMeta {
|
||||
/// Variable identifier used in queries (e.g. "month", "measure",
|
||||
/// "Species (ASFIS2022)")
|
||||
pub code: String,
|
||||
/// Human-readable label in the queried language
|
||||
#[serde(default)]
|
||||
pub text: String,
|
||||
/// Role classification — null on this endpoint
|
||||
#[serde(default)]
|
||||
pub role: Option<String>,
|
||||
/// Code values for this variable (parallel with valueTexts)
|
||||
#[serde(default)]
|
||||
pub values: Vec<String>,
|
||||
/// Display labels for each value (parallel with values)
|
||||
#[serde(default, rename = "valueTexts")]
|
||||
pub value_texts: Vec<String>,
|
||||
/// Catch-all for unknown fields
|
||||
#[serde(flatten)]
|
||||
pub extra: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
impl VariableMeta {
|
||||
/// Build a code→label lookup map from the parallel arrays.
|
||||
pub fn lookup_map(&self) -> HashMap<String, String> {
|
||||
self.values
|
||||
.iter()
|
||||
@@ -92,42 +53,24 @@ impl VariableMeta {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Query types (POST request body)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Query body sent to PX-Web API POST endpoint.
|
||||
///
|
||||
/// Confirmed format:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "query": [{"code": "month", "selection": {"filter": "item", "values": [...]}}],
|
||||
/// "response": {"format": "json-stat2"}
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Query {
|
||||
pub query: Vec<QueryItem>,
|
||||
pub response: QueryResponse,
|
||||
}
|
||||
|
||||
/// Single query item representing a dimension selection.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct QueryItem {
|
||||
pub code: String,
|
||||
pub selection: Selection,
|
||||
}
|
||||
|
||||
/// Selection within a query item.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Selection {
|
||||
/// "item" for explicit values, "all" for wildcard, "top" for top-N
|
||||
pub filter: String,
|
||||
/// Values to select. For "all" filter, use ["*"].
|
||||
pub values: Vec<String>,
|
||||
}
|
||||
|
||||
/// Response format specification.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct QueryResponse {
|
||||
pub format: String,
|
||||
@@ -141,68 +84,43 @@ impl Default for QueryResponse {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Data response types (JSON-stat2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Raw data response from PX-Web API POST request.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DataResponse {
|
||||
pub dataset: Dataset,
|
||||
}
|
||||
|
||||
/// Dataset wrapper containing dimensions and actual values.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Dataset {
|
||||
pub dimension: DimInfo,
|
||||
pub value: Vec<Option<f64>>,
|
||||
/// Status codes per value (optional)
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub status: Vec<String>,
|
||||
}
|
||||
|
||||
/// Dimension metadata in JSON-stat2 format.
|
||||
///
|
||||
/// Uses a flat `id` array for ordering and a `size` array for cardinality.
|
||||
/// Each dimension is keyed by its code in the `dimensions` map.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DimInfo {
|
||||
/// Ordered dimension IDs (defines cube layout)
|
||||
#[serde(default)]
|
||||
pub id: Vec<String>,
|
||||
/// Size of each dimension
|
||||
#[serde(default)]
|
||||
pub size: Vec<usize>,
|
||||
/// One entry per dimension, keyed by dimension code
|
||||
#[serde(flatten)]
|
||||
pub dimensions: HashMap<String, Dimension>,
|
||||
}
|
||||
|
||||
/// A single dimension in the JSON-stat2 response.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Dimension {
|
||||
/// Display label
|
||||
#[serde(default)]
|
||||
pub label: String,
|
||||
/// Category info with index and label maps
|
||||
pub category: CategoryInfo,
|
||||
}
|
||||
|
||||
/// Category info containing index and label maps.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CategoryInfo {
|
||||
/// Maps category code → numeric position in the dimension
|
||||
pub index: HashMap<String, usize>,
|
||||
/// Maps category code → display label
|
||||
#[serde(default)]
|
||||
pub label: HashMap<String, String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal data models
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Decoded row of landing data with labeled dimensions.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DataRow {
|
||||
pub month: String,
|
||||
@@ -223,7 +141,6 @@ pub struct DataRow {
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
/// Structured representation of a single landing record for DB insertion.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Landing {
|
||||
pub month: String,
|
||||
@@ -238,11 +155,71 @@ pub struct Landing {
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Errors
|
||||
// ---------------------------------------------------------------------------
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SpeciesDto {
|
||||
pub code: String,
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LandingDto {
|
||||
pub month: String,
|
||||
pub species_code: String,
|
||||
pub species_label: String,
|
||||
pub gear_code: String,
|
||||
pub zone_code: String,
|
||||
pub processing_code: String,
|
||||
pub preservation_code: String,
|
||||
pub shipsize_code: String,
|
||||
pub measure_code: String,
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
impl From<Landing> for LandingDto {
|
||||
fn from(l: Landing) -> Self {
|
||||
Self {
|
||||
month: l.month,
|
||||
species_code: l.species_code,
|
||||
species_label: l.species_label,
|
||||
gear_code: l.gear_code,
|
||||
zone_code: l.zone_code,
|
||||
processing_code: l.processing_code,
|
||||
preservation_code: l.preservation_code,
|
||||
shipsize_code: l.shipsize_code,
|
||||
measure_code: l.measure_code,
|
||||
value: l.value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MonthlyAggregate {
|
||||
pub month: String,
|
||||
pub total_mass: Option<f64>,
|
||||
pub total_value: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TopSpecies {
|
||||
pub species_code: String,
|
||||
pub species_label: String,
|
||||
pub total_value: Option<f64>,
|
||||
pub total_mass: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PriceTrend {
|
||||
pub month: String,
|
||||
pub price_per_kg: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SummaryDto {
|
||||
pub monthly: Vec<MonthlyAggregate>,
|
||||
pub top_species: Vec<TopSpecies>,
|
||||
pub price_trend: Vec<PriceTrend>,
|
||||
}
|
||||
|
||||
/// Error types for ingestion module.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum IngestError {
|
||||
#[error("HTTP request failed: {0}")]
|
||||
@@ -267,12 +244,6 @@ pub enum IngestError {
|
||||
UnicodeError(String),
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Type aliases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Lookup map: dimension_code → (value_code → display_label).
|
||||
pub type LookupMap = HashMap<String, HashMap<String, String>>;
|
||||
|
||||
/// Result alias using custom error type.
|
||||
pub type Result<T> = std::result::Result<T, IngestError>;
|
||||
|
||||
Reference in New Issue
Block a user