Compare commits
62
Commits
24d5a4e57a
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9a5902460 | ||
|
|
a1a93fddfd | ||
|
|
ae8a48780d | ||
|
|
24115272cd | ||
|
|
f2bfe5c1d6 | ||
|
|
6ebf123368 | ||
|
|
dfa9911ded | ||
|
|
55c190999a | ||
|
|
a88efaacd8 | ||
|
|
60398645d7 | ||
|
|
011920e2f9 | ||
|
|
3090d5ed5a | ||
|
|
ab48ccffe4 | ||
|
|
1cd305e4c3 | ||
|
|
25dd518eaa | ||
|
|
ecb2afe709 | ||
|
|
014c1d7eec | ||
|
|
b8950393f9 | ||
|
|
f4f2f2c76f | ||
|
|
afe25bc3a3 | ||
|
|
a421751d41 | ||
|
|
1c281c4dc8 | ||
|
|
f9cfb99916 | ||
|
|
3220d1d9a4 | ||
|
|
3ca28677ec | ||
|
|
225aa9b1c6 | ||
|
|
e4e09df834 | ||
|
|
665ad28364 | ||
|
|
8f27c3658c | ||
|
|
af2092d8f8 | ||
|
|
17102a120b | ||
|
|
02f143b9b9 | ||
|
|
f85ddb0c66 | ||
|
|
7cb28861e5 | ||
|
|
e95a5b1a9f | ||
|
|
a7cd06d40d | ||
|
|
30a4476643 | ||
|
|
0dcb7d1414 | ||
|
|
1af25a20fe | ||
|
|
8630a61abd | ||
|
|
6bedac526f | ||
|
|
c0cf0d8495 | ||
|
|
8fb8d507a4 | ||
|
|
20786a9e14 | ||
|
|
42a697aa57 | ||
|
|
bd7459e455 | ||
|
|
fed7d1af0e | ||
|
|
2879fecf70 | ||
|
|
4f3995da17 | ||
|
|
4ef3ca706a | ||
|
|
06886a2286 | ||
|
|
338e6c830a | ||
|
|
53e553e155 | ||
|
|
b072e2c3dc | ||
|
|
1a6840d16c | ||
|
|
5e80204e34 | ||
|
|
d52bcf2d64 | ||
|
|
f639b19c13 | ||
|
|
7a64f00014 | ||
|
|
bd63b7e788 | ||
|
|
5db5c847b2 | ||
|
|
5ec60a71c3 |
@@ -1 +1,3 @@
|
|||||||
/target
|
/target
|
||||||
|
*.db*
|
||||||
|
*.log*
|
||||||
|
|||||||
Generated
+3094
File diff suppressed because it is too large
Load Diff
+13
-3
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "hagfish"
|
name = "hagfisk"
|
||||||
version = "0.1.0"
|
version = "0.0.5"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
description = "Faroese fisheries data pipeline and dashboard"
|
description = "Faroese fisheries data pipeline and dashboard"
|
||||||
authors = ["Bartal Læarsson"]
|
authors = ["Bartal Læarsson"]
|
||||||
@@ -12,9 +12,19 @@ serde = { version = "1", features = ["derive"] }
|
|||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||||
|
tracing-appender = "0.2"
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
|
duckdb = { version = "1.10505", features = ["bundled"] }
|
||||||
|
axum = "0.8"
|
||||||
|
tower-http = { version = "0.6", features = ["cors", "trace"] }
|
||||||
|
rust-embed = "8.12"
|
||||||
|
mime_guess = "2"
|
||||||
|
tokio-util = { version = "0.7", features = ["io"] }
|
||||||
|
tempfile = "3"
|
||||||
|
clap = { version = "4", features = ["derive"] }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
mockito = "1"
|
mockito = "1"
|
||||||
tokio-test = "0.4"
|
tokio-test = "0.4"
|
||||||
|
reqwest = "0.12"
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
# Hagfisk
|
||||||
|
Faroese fisheries data pipeline and dashboard. Pulls monthly landing statistics from Hagstova Føroya's PX-Web API, stuffs them into a local DuckDB, and serves a filtered ECharts dashboard — all from a single Rust binary.
|
||||||
|
|
||||||
|
Named after the hagfish: a deep-sea creature famous for producing copious amounts of slime and being generally unpleasant. The name stuck anyway. The fish has no backbone. Neither does this README.
|
||||||
|
|
||||||
|
## What It Does
|
||||||
|

|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
- Fetches fisheries data (mass + value by species, gear, zone, month) from `statbank.hagstova.fo`
|
||||||
|
- Stores everything in a local DuckDB file
|
||||||
|
- Serves a REST API with filtered queries, summaries, and Parquet export
|
||||||
|
- Renders an interactive dashboard with line, price-trend, and stacked bar charts
|
||||||
|
- Runs monthly ingestion automatically via systemd timer (28th of each month, 03:00)
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
|
||||||
|
| Thing | Choice | Why |
|
||||||
|
|---|---|---|
|
||||||
|
| Language | Rust 2024 | Fast, safe, single static binary. No runtime, no VM, no nonsense. |
|
||||||
|
| Web framework | Axum | Async, typed, plays well with Tokio. Doesn't fight you. |
|
||||||
|
| Database | DuckDB | Embedded OLAP. No server, no config, just a file. Perfect for analytics on a single machine. |
|
||||||
|
| Charts | ECharts | Powerful, flexible, no build step. Vanilla JS, no framework tax. |
|
||||||
|
| Static assets | rust-embed | Compiled into the binary. One artifact to deploy, nothing to forget. |
|
||||||
|
| Logging | tracing + tracing-subscriber | Structured logs to file and journal. |
|
||||||
|
| Deployment | Bare metal + systemd | No containers. No compose files. No port mappings. Just a binary and a `.service` unit. |
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --release
|
||||||
|
./target/release/hagfisk ingest --full
|
||||||
|
./target/release/hagfisk serve
|
||||||
|
```
|
||||||
|
|
||||||
|
Dashboard lives at `http://localhost:8090`.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
`config.json` next to the binary:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"duckdb_path": "hagfish.db",
|
||||||
|
"bind_address": "127.0.0.1:8090",
|
||||||
|
"data_source_url": "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px",
|
||||||
|
"log_file_path": "hagfish.log",
|
||||||
|
"allowed_origins": []
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## CLI
|
||||||
|
|
||||||
|
```
|
||||||
|
hagfisk serve Start the web server
|
||||||
|
hagfisk ingest [--full] Fetch data (incremental or full backfill)
|
||||||
|
hagfisk export -o FILE Export landings to Parquet
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
The `justfile` handles everything, but you'll need to [install](https://just.systems/man/en/installation.html) it on your machine.
|
||||||
|
|
||||||
|
```
|
||||||
|
just deploy # Build, rsync, restore SELinux contexts, restart
|
||||||
|
just deploy-reingest # Same + full re-ingest
|
||||||
|
just logs-app # Tail application logs
|
||||||
|
just ps-app # Check service status
|
||||||
|
```
|
||||||
|
|
||||||
|
Systemd units live in `systemd/` — drop them in `~/.config/systemd/user/` and enable the timer for monthly ingestion.
|
||||||
|
|
||||||
|
## API
|
||||||
|
|
||||||
|
| Endpoint | What |
|
||||||
|
|---|---|
|
||||||
|
| `GET /healthz` | Alive check |
|
||||||
|
| `GET /api/species` | Species codes + Faroese names |
|
||||||
|
| `GET /api/zones` | Economic zone codes |
|
||||||
|
| `GET /api/gear` | Fishing gear codes |
|
||||||
|
| `GET /api/landings` | Filtered landing records (month range, species, gear, zone, measure) |
|
||||||
|
| `GET /api/summary` | Monthly aggregates, top 10 species, price/kg trend |
|
||||||
|
| `GET /api/summary/monthly-breakdown` | Per-month per-species breakdown |
|
||||||
|
| `GET /api/available-filters` | Cascading filter options based on active selections |
|
||||||
|
| `GET /api/export.parquet` | Full dataset as Parquet download |
|
||||||
|
|
||||||
|
Full PX-Web API reference in `docs/API.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Like the hagfish itself: not glamorous, but it works.**
|
||||||
|
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"duckdb_path": "hagfish.db",
|
||||||
|
"bind_address": "127.0.0.1:8090",
|
||||||
|
"data_source_url": "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px",
|
||||||
|
"log_file_path": "hagfish.log",
|
||||||
|
"allowed_origins": []
|
||||||
|
}
|
||||||
+249
@@ -0,0 +1,249 @@
|
|||||||
|
# API Documentation
|
||||||
|
|
||||||
|
This document describes the PX-Web API used by hagfish to fetch Faroese fisheries statistics from the official Statbank.
|
||||||
|
|
||||||
|
## EndpointBase URL: https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px
|
||||||
|
- **GET**: Returns metadata (table structure, dimension codes, labels)
|
||||||
|
- **POST**: Returns data in JSON-stat2 format
|
||||||
|
|
||||||
|
## Endpoint Reference
|
||||||
|
|
||||||
|
### Health Check
|
||||||
|
- **GET** `/healthz` - Returns 200 OK if service is running
|
||||||
|
|
||||||
|
### Species Lookup
|
||||||
|
- **GET** `/api/species` - Returns list of all species codes and Faroese names
|
||||||
|
|
||||||
|
### Zones Lookup
|
||||||
|
- **GET** `/api/zones` - Returns list of all economic zone codes and labels
|
||||||
|
|
||||||
|
### Gear Lookup
|
||||||
|
- **GET** `/api/gear` - Returns list of all fishing gear codes and labels
|
||||||
|
|
||||||
|
### Landings Data
|
||||||
|
- **GET** `/api/landings` - Returns filtered landing records
|
||||||
|
|
||||||
|
**Query Parameters:**
|
||||||
|
|
||||||
|
| Parameter | Type | Required | Description |
|
||||||
|
|-----------|------|----------|-------------|
|
||||||
|
| `month` | string | No | Single month filter (legacy, e.g., `2024M01`) |
|
||||||
|
| `month_from` | string | No | Start of date range (inclusive, e.g., `2024M01`) |
|
||||||
|
| `month_to` | string | No | End of date range (inclusive, e.g., `2024M12`) |
|
||||||
|
| `species` | string | No | Filter by species code (e.g., `COD`) |
|
||||||
|
| `gear` | string | No | Filter by gear code (e.g., `TR1`) |
|
||||||
|
| `zone` | string | No | Filter by zone code (e.g., `FO`) |
|
||||||
|
| `measure` | string | No | Filter by measure type (`MASS` or `VALUE`) |
|
||||||
|
| `limit` | integer | No | Max results (default: 10000, max: 10000) |
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
|
||||||
|
bash curl "http://localhost:8090/api/landings?month_from=2024M01&month_to=2024M06&species=COD&limit=1000"
|
||||||
|
|
||||||
|
|
||||||
|
### Summary Aggregates
|
||||||
|
- **GET** `/api/summary` - Returns aggregated statistics
|
||||||
|
|
||||||
|
**Query Parameters:**
|
||||||
|
|
||||||
|
| Parameter | Type | Required | Description |
|
||||||
|
|-----------|------|----------|-------------|
|
||||||
|
| `species` | string | No | Filter aggregations by species code |
|
||||||
|
|
||||||
|
**Response Fields:**
|
||||||
|
- `monthly`: Array of `{month, total_mass, total_value}`
|
||||||
|
- `top_species`: Array of top 10 species by value
|
||||||
|
- `price_trend`: Array of `{month, price_per_kg}`
|
||||||
|
|
||||||
|
### Parquet Export
|
||||||
|
- **GET** `/api/export.parquet` - Downloads full dataset as Parquet file
|
||||||
|
|
||||||
|
**Note:** Triggers temp file creation in system temp directory with automatic cleanup after 300 seconds.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Metadata Request (GET)
|
||||||
|
|
||||||
|
### Requestbash
|
||||||
|
curl -s "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
|
||||||
|
### Response Structurejson
|
||||||
|
{
|
||||||
|
"title": "AVR01010 Avreiðingar í nøgd og virði...",
|
||||||
|
"variables": [
|
||||||
|
{
|
||||||
|
"code": "measure",
|
||||||
|
"text": "mát",
|
||||||
|
"role": null,
|
||||||
|
"values": ["MASS", "VALUE"],
|
||||||
|
"valueTexts": ["Nøgd", "Virði"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
### Fields
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `title` | string | Table name in Faroese |
|
||||||
|
| `variables[].code` | string | Dimension identifier (used in queries) |
|
||||||
|
| `variables[].text` | string | Human-readable label in Faroese |
|
||||||
|
| `variables[].role` | null/string | Classification (always `null` on this endpoint) |
|
||||||
|
| `variables[].values` | string[] | Valid dimension codes |
|
||||||
|
| `variables[].valueTexts` | string[] | Display labels (parallel with `values`) |
|
||||||
|
|
||||||
|
### Verified Dimension Codes
|
||||||
|
|
||||||
|
| Variable Code | Faroese Label | Values |
|
||||||
|
|---------------|---------------|--------|
|
||||||
|
| `measure` | mát | `MASS`, `VALUE` |
|
||||||
|
| `Species (ASFIS2022)` | Fiskaslag (ASFIS2022) | `TOTAL`, `148XXXXXXX00000`, ... (72 total) |
|
||||||
|
| `Fishing Gear (ISSCFG2016)` | Reiðskapur (ISSCFG2016) | `TOTAL`, ... (12 total) |
|
||||||
|
| `Economic Zone (GEONOM2023)` | Búskapar øki (GEONOM2023) | `TOTAL`, ... (10 total) |
|
||||||
|
| `Processing (EUMOFAPresentation)` | Virking (EUMOFAPresentation) | `TOTAL`, ... (6 total) |
|
||||||
|
| `Preservation (EUMOFAPreservation)` | Viðgerð (EUMOFAPreservation) | `TOTAL`, ... (9 total) |
|
||||||
|
| `Shipsize` | Skipastødd | `TOTAL`, ... (11 total) |
|
||||||
|
| `month` | mánaður | `2015M01` through `2026M05` (137 months) |
|
||||||
|
|
||||||
|
**Note**: `role` is always `null`. Time dimension identification must use `code == "month"`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Request (POST)
|
||||||
|
|
||||||
|
### Request Body Formatjson
|
||||||
|
{
|
||||||
|
"query": [
|
||||||
|
{
|
||||||
|
"code": "dimension_code",
|
||||||
|
"selection": {
|
||||||
|
"filter": "item",
|
||||||
|
"values": ["value1", "value2"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"response": {
|
||||||
|
"format": "json-stat2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
### Filter Types
|
||||||
|
|
||||||
|
| Filter | Values | Description |
|
||||||
|
|--------|--------|-------------|
|
||||||
|
| `item` | explicit codes | Select specific values |
|
||||||
|
| `all` | `["*"]` | Wildcard — select all values |
|
||||||
|
| `top` | `["TOP_N"]` | Top N values by magnitude |
|
||||||
|
|
||||||
|
### Example Querybash
|
||||||
|
curl -s -X POST
|
||||||
|
-H "Content-Type: application/json"
|
||||||
|
-d '{
|
||||||
|
"query": [
|
||||||
|
{"code": "month", "selection": {"filter": "item", "values": ["2024M01", "2024M02"]}},
|
||||||
|
{"code": "Species (ASFIS2022)", "selection": {"filter": "all", "values": [""]}},
|
||||||
|
{"code": "Fishing Gear (ISSCFG2016)", "selection": {"filter": "all", "values": [""]}},
|
||||||
|
{"code": "Economic Zone (GEONOM2023)", "selection": {"filter": "all", "values": ["*"]}},
|
||||||
|
{"code": "Processing (EUMOFAPresentation)", "selection": {"filter": "item", "values": ["TOTAL"]}},
|
||||||
|
{"code": "Preservation (EUMOFAPreservation)", "selection": {"filter": "item", "values": ["TOTAL"]}},
|
||||||
|
{"code": "Shipsize", "selection": {"filter": "item", "values": ["TOTAL"]}},
|
||||||
|
{"code": "measure", "selection": {"filter": "item", "values": ["MASS", "VALUE"]}}
|
||||||
|
],
|
||||||
|
"response": {"format": "json-stat2"}
|
||||||
|
}'
|
||||||
|
"https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
|
||||||
|
### Cell Limit
|
||||||
|
|
||||||
|
- Maximum cells per query: ~8,000,000
|
||||||
|
- Recommended maximum: 1,000,000 for reliability
|
||||||
|
- To fetch full dataset, paginate by month or use smaller dimension selections
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Response Format (JSON-stat2)
|
||||||
|
|
||||||
|
### Structurejson
|
||||||
|
{
|
||||||
|
"class": "dataset",
|
||||||
|
"label": "...",
|
||||||
|
"id": ["measure", "Species (ASFIS2022)", ..., "month"],
|
||||||
|
"size": [2, 72, 12, 10, 1, 1, 1, 2],
|
||||||
|
"dimension": {
|
||||||
|
"measure": {
|
||||||
|
"label": "mát",
|
||||||
|
"category": {
|
||||||
|
"index": {"MASS": 0, "VALUE": 1},
|
||||||
|
"label": {"MASS": "Nøgd", "VALUE": "Virði"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"value": [73653738, 1234567, ...]
|
||||||
|
}
|
||||||
|
### Fields
|
||||||
|
|
||||||
|
| Field | Description |
|
||||||
|
|-------|-------------|
|
||||||
|
| `id` | Array of dimension codes in order (defines cube layout) |
|
||||||
|
| `size` | Cardinality per dimension (parallel with `id`) |
|
||||||
|
| `dimension.<code>.category.index` | Maps value code → numeric position in dimension |
|
||||||
|
| `dimension.<code>.category.label` | Maps value code → display text |
|
||||||
|
| `value` | Flattened array of measurements (row-major order) |
|
||||||
|
|
||||||
|
### Row-Major Index Decoding
|
||||||
|
|
||||||
|
Values are stored in row-major order: the last dimension (`month`) varies fastest.
|
||||||
|
|
||||||
|
Given `size = [2, 72, 12, 10, 1, 1, 1, 2]`:
|
||||||
|
|
||||||
|
- Flat index `0` → `[0,0,0,0,0,0,0,0]` = measure=MASS, species=TOTAL, ..., month=2024M01
|
||||||
|
- Flat index `1` → `[0,0,0,0,0,0,0,1]` = measure=MASS, species=TOTAL, ..., month=2024M02
|
||||||
|
- Flat index `2` → `[0,0,0,0,0,0,1,0]` = measure=MASS, species=SPECIES_1, ..., month=2024M01
|
||||||
|
|
||||||
|
Decoding algorithm (reverse modulo):rust
|
||||||
|
fn decode(flat_index: usize, sizes: &[usize]) -> Vec<usize> {
|
||||||
|
let mut indices = Vec::new();
|
||||||
|
let mut remaining = flat_index;
|
||||||
|
for &size in sizes.iter().rev() {
|
||||||
|
indices.push(remaining % size);
|
||||||
|
remaining /= size;
|
||||||
|
}
|
||||||
|
indices.reverse();
|
||||||
|
indices
|
||||||
|
}
|
||||||
|
### Sentinel Values
|
||||||
|
|
||||||
|
| Value | Meaning |
|
||||||
|
|-------|---------|
|
||||||
|
| `-1.0` | Missing/no data (treated as `NULL`) |
|
||||||
|
| `null` | Missing/no data (already nullable in JSON) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Known Constraints
|
||||||
|
|
||||||
|
1. **Cell limit**: ~8 million cells per query
|
||||||
|
2. **Rate limits**: Unknown — assume reasonable backoff for large downloads
|
||||||
|
3. **Language**: API responds in requested language (`fo` or `en`)
|
||||||
|
4. **Time zone**: No timezone specified — treat timestamps as local
|
||||||
|
5. **Updates**: Data updated irregularly — metadata timestamp in response header
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Integration Checklist
|
||||||
|
|
||||||
|
- [x] Metadata endpoint reachable via GET
|
||||||
|
- [x] POST queries return valid JSON-stat2
|
||||||
|
- [x] Dimension codes match `variables[].code` from metadata
|
||||||
|
- [x] Row-major index decoding verified
|
||||||
|
- [x] Sentinel value coercion (`-1.0` → `None`) implemented
|
||||||
|
- [x] Unicode (Faroese characters) handled correctly
|
||||||
|
- [ ] Incremental ingestion logic tested
|
||||||
|
- [ ] Parquet export tested
|
||||||
|
- [ ] Error handling for network timeouts implemented
|
||||||
|
|
||||||
|
Summary of Changes
|
||||||
|
File Change Reason
|
||||||
|
src/api.rs Added MAX_LIMIT constant and hard cap Prevent unlimited query results
|
||||||
|
src/api.rs Replaced CorsLayer::permissive() with explicit origins Security hardening
|
||||||
|
src/api.rs Updated tempfile::Builder with prefix Ensure cleanup function finds files
|
||||||
|
src/api.rs Added test_get_landings_range_filter test Cover new range-filtering logic
|
||||||
|
src/types.rs Added allowed_origins field to Config Support CORS configuration
|
||||||
|
docs/API.md Added endpoint reference table Document new parameters
|
||||||
|
|
||||||
+54
-37
@@ -1,4 +1,4 @@
|
|||||||
# HAGFISH Project todo
|
# HAGFISH Project TODO
|
||||||
|
|
||||||
`GET statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px — metadata`
|
`GET statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px — metadata`
|
||||||
POST same URL — JSON-stat2 query
|
POST same URL — JSON-stat2 query
|
||||||
@@ -59,55 +59,72 @@ hagfish/
|
|||||||
|
|
||||||
### Phase 1: Types & Ingestion
|
### Phase 1: Types & Ingestion
|
||||||
|
|
||||||
- [ ] 1.1 Define types in types.rs: MetadataResponse, VariableMeta, DataResponse, DataRow, Query, Selection, QueryItem, Config — all with serde derives
|
- [x] 1.1 Define types in types.rs: MetadataResponse, VariableMeta, DataResponse, DataRow, Query, Selection, QueryItem, Config — all with serde derives
|
||||||
- [ ] 1.2 Implement ingest.rs::fetch_metadata(url) — GET request, parse JSON, return HashMap<(variable_code, value_code), faroese_label>
|
- [x] 1.2 Implement ingest.rs::fetch_metadata(url) — GET request, parse JSON, return HashMap<(variable_code, value_code), faroese_label>
|
||||||
- [ ] 1.3 Implement ingest.rs::build_query(months: &[String]) — construct POST body with all species/gear/zones set to "*", processing/preservation/shipsize set to TOTAL, measure set to both MASS and VALUE
|
- [x] 1.3 Implement ingest.rs::build_query(months: &[String]) — construct POST body with all species/gear/zones set to "*", processing/preservation/shipsize set to TOTAL, measure set to both MASS and VALUE
|
||||||
- [ ] 1.4 Implement ingest.rs::fetch_data(url, query) — POST request, parse JSON-stat2 response, return Vec<DataRow>
|
- [x] 1.4 Implement ingest.rs::fetch_data(url, query) — POST request, parse JSON-stat2 response, return Vec<DataRow>
|
||||||
- [ ] 1.5 Implement ingest.rs::parse_row(row, lookup_maps) — decode key[] positions into labeled Landing struct. Handle "-" → None.
|
- [x] 1.5 Implement ingest.rs::parse_row(row, lookup_maps) — decode key[] positions into labeled Landing struct. Handle "-" → None.
|
||||||
- [ ] 1.6 Write unit tests: mock JSON-stat2 response, verify key-to-label mapping, verify "-" handling, verify Faroese Unicode characters in species names (ð, á, í, ý, ø, ó)
|
- [x] 1.6 Write unit tests: mock JSON-stat2 response, verify key-to-label mapping, verify "-" handling, verify Faroese Unicode characters in species names (ð, á, í, ý, ø, ó)
|
||||||
|
|
||||||
### Phase 2: DuckDB Storage
|
### Phase 2: DuckDB Storage
|
||||||
|
|
||||||
- [ ] 2.1 Add duckdb crate dependency (bundled feature)
|
- [x] 2.1 Add duckdb crate dependency (bundled feature)
|
||||||
- [ ] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code.
|
- [x] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code.
|
||||||
- [ ] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE
|
- [x] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE
|
||||||
- [ ] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response
|
- [x] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response
|
||||||
- [ ] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion
|
- [x] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion
|
||||||
- [ ] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month
|
- [x] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month
|
||||||
- [ ] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling
|
- [x] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling
|
||||||
|
|
||||||
### Phase 3: API (Axum)
|
### Phase 3: API (Axum)
|
||||||
|
|
||||||
- [ ] 3.1 Set up Axum router in main.rs with Tokio runtime. AppState holds DuckDB connection wrapped in Mutex.
|
- [x] 3.1 Set up Axum router in main.rs with Tokio runtime. AppState holds DuckDB connection wrapped in Mutex.
|
||||||
- [ ] 3.2 Implement GET /api/species — query species lookup table, return JSON array
|
- [x] 3.2 Implement GET /api/species — query species lookup table, return JSON array
|
||||||
- [ ] 3.3 Implement GET /api/landings — parse query params, build DuckDB SQL with WHERE clauses. Support: months, species, gear, zone, measure filters.
|
- [x] 3.3 Implement GET /api/landings — parse query params, build DuckDB SQL with WHERE clauses. Support: months, species, gear, zone, measure filters.
|
||||||
- [ ] 3.4 Implement GET /api/summary — aggregate query: total mass + value by month, top 10 species by value, price/kg trend
|
- [x] 3.4 Implement GET /api/summary — aggregate query: total mass + value by month, top 10 species by value, price/kg trend
|
||||||
- [ ] 3.5 Implement GET /api/export.parquet — generate and stream Parquet via DuckDB COPY
|
- [x] 3.5 Implement GET /api/export.parquet — generate and stream Parquet via DuckDB COPY
|
||||||
- [ ] 3.6 Implement GET /healthz
|
- [x] 3.6 Implement GET /healthz
|
||||||
- [ ] 3.7 Serve static files via rust-embed
|
- [x] 3.7 Serve static files via rust-embed
|
||||||
- [ ] 3.8 Write API tests
|
- [x] 3.8 Write API tests
|
||||||
|
|
||||||
|
#### Production Hardening (Deferred to Post-Launch)
|
||||||
|
|
||||||
|
- [ ] 🔒 DB operation timeouts — Add 30s `tokio::time::timeout()` wrapper around all `spawn_blocking` DB calls
|
||||||
|
- [ ] 🔒 CORS hardening — Replace `CorsLayer::permissive()` with explicit allowed origins before production deployment
|
||||||
|
- [ ] 🔒 Rate limiting — Optional: add `tower_governor` middleware to prevent abuse on public deployments
|
||||||
|
- [ ] ⚠️ Startup config validation — Verify `data_source_url` is reachable, `duckdb_path` is writable before accepting connections
|
||||||
|
|
||||||
### Phase 4: Frontend
|
### Phase 4: Frontend
|
||||||
|
|
||||||
- [ ] 4.1 index.html — dropdown filters (species, zone, gear, month range) and 3 chart containers
|
- [x] 4.1 index.html — dropdown filters (species, zone, gear, month range) and 3 chart containers
|
||||||
- [ ] 4.2 app.js — fetch species list on load, populate dropdowns, fetch /api/landings, render charts
|
- [x] 4.2 app.js — fetch species list on load, populate dropdowns, fetch /api/landings, render charts
|
||||||
- [ ] 4.3 ECharts line chart: x=month, y=mass/value toggle
|
- [x] 4.3 ECharts line chart: x=month, y=mass/value toggle
|
||||||
- [ ] 4.4 ECharts stacked bar: x=month, y=value by species (top 10 + "other")
|
- [x] 4.4 ECharts stacked bar: x=month, y=value by species (top 10 + "other")
|
||||||
- [ ] 4.5 ECharts donut: species distribution for selected month
|
- [x] 4.5 ECharts donut: species distribution for selected month
|
||||||
- [ ] 4.6 Loading states, error handling, empty state
|
- [x] 4.6 Loading states, error handling, empty state
|
||||||
- [ ] 4.7 Responsive layout, plain CSS
|
- [x] 4.7 Responsive layout, plain CSS
|
||||||
|
|
||||||
### Phase 5: CLI & Scheduling
|
### Phase 5: CLI & Scheduling
|
||||||
|
|
||||||
- [ ] 5.1 Add clap derive subcommands: hagfish ingest [--full] and hagfish serve
|
- [x] 5.1 Add clap derive subcommands: hagfish ingest [--full] and hagfish serve
|
||||||
- [ ] 5.2 Implement incremental logic: read get_last_month(), compute remaining months from metadata, fetch in batches if >12 months
|
- [x] 5.2 Implement incremental logic: read get_last_month(), compute remaining months from metadata, fetch in batches if >12 months
|
||||||
- [ ] 5.3 Log ingestion runs with slog (rows inserted, duration, errors)
|
- [x] 5.3 Log ingestion runs with tracing (rows inserted, duration, errors)
|
||||||
- [ ] 5.4 Add hagfish export --out /path/to/parquet subcommand
|
- [x] 5.4 Add hagfish export --out /path/to/parquet subcommand
|
||||||
- [ ] 5.5 Load config.json on startup (DuckDB path, bind address, data source URL, log file path)
|
- [x] 5.5 Load config.json on startup (DuckDB path, bind address, data source URL, log file path)
|
||||||
|
|
||||||
### Phase 6: Bare Metal Deployment
|
### Phase 6: Bare Metal Deployment
|
||||||
|
|
||||||
- [ ] 6.1 Write systemd service unit file (hagfish.service) — ExecStart=/usr/local/bin/hagfish serve, restart policy
|
- [x] 6.1 Write systemd service unit file (hagfish.service) — ExecStart=/usr/local/bin/hagfish serve, restart policy
|
||||||
- [ ] 6.2 Write systemd timer (hagfish-ingest.timer + hagfish-ingest.service) — monthly, runs hagfish ingest
|
- [x] 6.2 Write systemd timer (hagfish-ingest.timer + hagfish-ingest.service) — monthly, runs hagfish ingest
|
||||||
- [ ] 6.3 Taskfile: build (release, static), deploy (rsync binary + config + units, ssh reload)
|
- [x] 6.3 Taskfile: build (release, static), deploy (rsync binary + config + units, ssh reload)
|
||||||
- [ ] 6.4 README with ELI5 Technology Choices section (why DuckDB, why Rust, why embedded static assets)
|
- [ ] 6.4 README with ELI5 Technology Choices section (why DuckDB, why Rust, why embedded static assets)
|
||||||
|
|
||||||
|
### Phase 7: post deploy fixes
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current Status
|
||||||
|
|
||||||
|
**Bug Fixes**: Next up
|
||||||
|
**Production Hardening**: Deferred ⏸️
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
[[ FO ]]
|
||||||
|
- Lýsing
|
||||||
|
Sum maðurin segði; tað frálíkt at vísa á, at ein dugur at heinta dátur úr einum API og vísa tað á einari heimasíðu har brúkarar kunna filtrera visualiseringina.
|
||||||
|
Her nýta vit fiskahagtøl sum verða dagførd einaferð um mánaðin. So kunna vit líkasum síggja hvat hendir har, ha? Eg kalli hesa applikatión 'hagfisk'. Altso hagtøl + fisk. Hagfisk. Hagfisk, havi eg funni útav, er eisini ein heilt ótrúligani ólekkur og slímutur fiskur.
|
||||||
|
[INSERT PHOTO].
|
||||||
|
Men ja, eg fór við tí allíkavæl.
|
||||||
|
Anyways, her er ikki so nógv at siga: taka dátur frá hagstovu-API, skriva til ein lokalan DuckDB OLTP-dátugrunn á servara, servera API til web dátur, front end til at visualisera dáturnar.
|
||||||
|
|
||||||
|
- Leiðbeining
|
||||||
|
Far inn á síðuna, hygg og filtrera.
|
||||||
|
|
||||||
|
- Tøkni
|
||||||
|
Rust - axum, serde, tokio, tower-http, reqwest, rust-embed
|
||||||
|
DuckDB
|
||||||
|
html
|
||||||
|
js
|
||||||
|
css
|
||||||
|
echarts
|
||||||
|
|
||||||
|
- Keldu kota
|
||||||
|
https://git.flo.fo/FLO/hagfisk
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- Leinkja
|
||||||
|
https://hagfisk.poc.fló.fo
|
||||||
|
|
||||||
|
|
||||||
|
[[ DA ]]
|
||||||
|
|
||||||
|
## Beskrivelse
|
||||||
|
|
||||||
|
Som den gamle mand sagde: Det er jo meget fint at vise, at man kan hente data fra et API og visualisere dem på en hjemmeside, hvor brugerne kan filtrere visningen.
|
||||||
|
|
||||||
|
Her bruger vi fiskeristatistik, som bliver opdateret én gang om måneden. Så kan vi ligesom se, hvad der sker, ikke? Jeg kalder denne applikation 'hagfisk'. Altså statistik (FO = hagtøl) + fisk. Hagfisk. Jeg har fundet ud af, at hagfisk også er en helt utrolig modbydelig og slimet fisk.
|
||||||
|
|
||||||
|
[INDSÆT FOTO].
|
||||||
|
|
||||||
|
Men ja, jeg gik med navnet alligevel.
|
||||||
|
|
||||||
|
Anyways, der er ikke så meget mere at sige: Hent data fra Hagstova-API'et, gem dem i en lokal DuckDB-database på serveren, eksponér dem via et API til webfrontend'en, og visualisér dataene.
|
||||||
|
|
||||||
|
## Vejledning
|
||||||
|
|
||||||
|
Gå ind på siden, kig dig omkring og filtrér.
|
||||||
|
|
||||||
|
## Teknologi
|
||||||
|
|
||||||
|
Rust – axum, serde, tokio, tower-http, reqwest, rust-embed
|
||||||
|
DuckDB
|
||||||
|
HTML
|
||||||
|
JS
|
||||||
|
CSS
|
||||||
|
ECharts
|
||||||
|
|
||||||
|
## Kildekode
|
||||||
|
|
||||||
|
https://git.flo.fo/FLO/hagfisk
|
||||||
|
|
||||||
|
## Link
|
||||||
|
|
||||||
|
https://hagfisk.poc.fló.fo
|
||||||
|
|
||||||
|
|
||||||
|
[[ EN ]]
|
||||||
|
|
||||||
|
## Description
|
||||||
|
|
||||||
|
As the old man said: It’s pretty great to show that you can pull data from an API and visualise it on a website where users can filter the view.
|
||||||
|
|
||||||
|
Here we’re using fisheries statistics, updated once a month. So we can kind of see what’s going on, right? I’m calling this application ‘hagfisk’. Basically, statistics + fish. Hagfisk. I’ve since discovered that hagfish is also an incredibly gross and slimy fish.
|
||||||
|
|
||||||
|
[INSERT PHOTO].
|
||||||
|
|
||||||
|
But yeah, I went with the name anyway.
|
||||||
|
|
||||||
|
Anyways, there’s not much more to say: fetch data from the Hagstova API, store it in a local DuckDB database on the server, expose it through an API to the web frontend, and visualise the data.
|
||||||
|
|
||||||
|
## Guide
|
||||||
|
|
||||||
|
Go to the page, have a look around, and filter.
|
||||||
|
|
||||||
|
## Technology
|
||||||
|
|
||||||
|
Rust – axum, serde, tokio, tower-http, reqwest, rust-embed
|
||||||
|
DuckDB
|
||||||
|
HTML
|
||||||
|
JS
|
||||||
|
CSS
|
||||||
|
ECharts
|
||||||
|
|
||||||
|
## Source Code
|
||||||
|
|
||||||
|
https://git.flo.fo/FLO/hagfisk
|
||||||
|
|
||||||
|
## Link
|
||||||
|
|
||||||
|
https://hagfisk.poc.fló.fo
|
||||||
|
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 MiB |
@@ -0,0 +1,61 @@
|
|||||||
|
host := "bl@flo-homeserver"
|
||||||
|
hagfisk_dir := "~/hagfisk"
|
||||||
|
|
||||||
|
# Build binary locally
|
||||||
|
build:
|
||||||
|
echo "Building hagfisk binary..."
|
||||||
|
cargo build --release
|
||||||
|
|
||||||
|
# Stop app service (must stop before DB operations)
|
||||||
|
stop-app:
|
||||||
|
ssh {{host}} "systemctl --user stop hagfisk.service"
|
||||||
|
|
||||||
|
# Start app service
|
||||||
|
start-app:
|
||||||
|
ssh {{host}} "systemctl --user start hagfisk.service"
|
||||||
|
|
||||||
|
# Restart app service
|
||||||
|
restart-app:
|
||||||
|
ssh {{host}} "systemctl --user restart hagfisk.service"
|
||||||
|
|
||||||
|
# Deploy and restart (safe for regular updates)
|
||||||
|
deploy: build
|
||||||
|
ssh -q {{host}} "rm -f {{hagfisk_dir}}/hagfisk"
|
||||||
|
echo "Syncing binary to remote..."
|
||||||
|
rsync -avz --progress ./target/release/hagfisk {{host}}:{{hagfisk_dir}}/hagfisk
|
||||||
|
echo "Restoring SELinux contexts..."
|
||||||
|
ssh -t -q {{host}} "sudo restorecon -RF {{hagfisk_dir}}"
|
||||||
|
ssh -q {{host}} "systemctl --user daemon-reload"
|
||||||
|
ssh -q {{host}} "systemctl --user restart hagfisk.service"
|
||||||
|
|
||||||
|
# Deploy with full re-ingest (stops service first)
|
||||||
|
deploy-reingest: build stop-app
|
||||||
|
echo "Syncing binary to remote..."
|
||||||
|
rsync -avz --progress ./target/release/hagfisk {{host}}:{{hagfisk_dir}}/hagfisk
|
||||||
|
echo "Restoring SELinux contexts..."
|
||||||
|
ssh -t -q {{host}} "sudo restorecon -RF {{hagfisk_dir}}"
|
||||||
|
ssh -q {{host}} "systemctl --user daemon-reload"
|
||||||
|
echo "Running full re-ingest..."
|
||||||
|
ssh {{host}} "cd {{hagfisk_dir}} && ./hagfisk ingest --full"
|
||||||
|
echo "Starting service..."
|
||||||
|
ssh {{host}} "systemctl --user start hagfisk.service"
|
||||||
|
|
||||||
|
# Manual full ingest (assumes service is stopped)
|
||||||
|
ingest-full:
|
||||||
|
ssh {{host}} "cd {{hagfisk_dir}} && ./hagfisk ingest --full"
|
||||||
|
|
||||||
|
# Stream logs from remote app
|
||||||
|
logs-app:
|
||||||
|
ssh {{host}} "journalctl --user -u hagfisk -f"
|
||||||
|
|
||||||
|
# Show status of app
|
||||||
|
ps-app:
|
||||||
|
ssh {{host}} "systemctl --user status hagfisk.service"
|
||||||
|
|
||||||
|
# Stream logs from remote ingest (manual run)
|
||||||
|
logs-ingest:
|
||||||
|
ssh {{host}} "journalctl --user -u hagfisk-ingest -f"
|
||||||
|
|
||||||
|
# Show status of ingest
|
||||||
|
ps-ingest:
|
||||||
|
ssh {{host}} "systemctl --user status hagfisk-ingest.service"
|
||||||
+1493
File diff suppressed because it is too large
Load Diff
+25
@@ -0,0 +1,25 @@
|
|||||||
|
use clap::{Parser, Subcommand};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(name = "hagfish", version, about)]
|
||||||
|
pub struct Cli {
|
||||||
|
#[arg(short, long, value_name = "FILE")]
|
||||||
|
pub config: Option<PathBuf>,
|
||||||
|
|
||||||
|
#[command(subcommand)]
|
||||||
|
pub command: Command,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Subcommand)]
|
||||||
|
pub enum Command {
|
||||||
|
Ingest {
|
||||||
|
#[arg(long)]
|
||||||
|
full: bool,
|
||||||
|
},
|
||||||
|
Serve,
|
||||||
|
Export {
|
||||||
|
#[arg(short, long, value_name = "FILE")]
|
||||||
|
out: PathBuf,
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -0,0 +1,596 @@
|
|||||||
|
use crate::types::{Landing, LookupMap};
|
||||||
|
use duckdb::{Connection, params};
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use thiserror::Error;
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
const LOOKUP_TABLES: &[(&str, &str)] = &[
|
||||||
|
("Species (ASFIS2022)", "species"),
|
||||||
|
("Fishing Gear (ISSCFG2016)", "gear"),
|
||||||
|
("Economic Zone (GEONOM2023)", "zone"),
|
||||||
|
("Processing (EUMOFAPresentation)", "processing"),
|
||||||
|
("Preservation (EUMOFAPreservation)", "preservation"),
|
||||||
|
("Shipsize", "shipsize"),
|
||||||
|
];
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum DbError {
|
||||||
|
#[error("DuckDB error: {0}")]
|
||||||
|
Duckdb(#[from] duckdb::Error),
|
||||||
|
#[error("invalid export path: {0}")]
|
||||||
|
InvalidPath(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub type Result<T> = std::result::Result<T, DbError>;
|
||||||
|
|
||||||
|
pub fn init(path: &str) -> Result<Connection> {
|
||||||
|
let conn = Connection::open(path)?;
|
||||||
|
init_schema(&conn)?;
|
||||||
|
info!("Initialized DuckDB at {}", path);
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init_schema(conn: &Connection) -> Result<()> {
|
||||||
|
conn.execute_batch(
|
||||||
|
"
|
||||||
|
CREATE TABLE IF NOT EXISTS species (
|
||||||
|
code TEXT PRIMARY KEY,
|
||||||
|
label TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS gear (
|
||||||
|
code TEXT PRIMARY KEY,
|
||||||
|
label TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS zone (
|
||||||
|
code TEXT PRIMARY KEY,
|
||||||
|
label TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS processing (
|
||||||
|
code TEXT PRIMARY KEY,
|
||||||
|
label TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS preservation (
|
||||||
|
code TEXT PRIMARY KEY,
|
||||||
|
label TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS shipsize (
|
||||||
|
code TEXT PRIMARY KEY,
|
||||||
|
label TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS landings (
|
||||||
|
month TEXT NOT NULL,
|
||||||
|
species_code TEXT NOT NULL,
|
||||||
|
species_label TEXT,
|
||||||
|
gear_code TEXT NOT NULL,
|
||||||
|
zone_code TEXT NOT NULL,
|
||||||
|
processing_code TEXT NOT NULL,
|
||||||
|
preservation_code TEXT NOT NULL,
|
||||||
|
shipsize_code TEXT NOT NULL,
|
||||||
|
measure_code TEXT NOT NULL,
|
||||||
|
value DOUBLE,
|
||||||
|
PRIMARY KEY (month, species_code, gear_code, zone_code,
|
||||||
|
processing_code, preservation_code, shipsize_code,
|
||||||
|
measure_code)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_landings_month
|
||||||
|
ON landings(month);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_landings_species
|
||||||
|
ON landings(species_code);
|
||||||
|
",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn update_lookups(conn: &Connection, lookup_maps: &LookupMap) -> Result<()> {
|
||||||
|
for (dim_code, table_name) in LOOKUP_TABLES {
|
||||||
|
if let Some(codes) = lookup_maps.get(*dim_code) {
|
||||||
|
let sql = format!(
|
||||||
|
"INSERT INTO {} (code, label) VALUES (?, ?) \
|
||||||
|
ON CONFLICT(code) DO UPDATE SET label = excluded.label",
|
||||||
|
table_name
|
||||||
|
);
|
||||||
|
for (code, label) in codes {
|
||||||
|
conn.execute(&sql, params![code, label])?;
|
||||||
|
}
|
||||||
|
info!(
|
||||||
|
"Updated {} entries in '{}' lookup table",
|
||||||
|
codes.len(),
|
||||||
|
table_name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn upsert_landings(conn: &Connection, rows: &[Landing]) -> Result<usize> {
|
||||||
|
if rows.is_empty() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let months: HashSet<&str> = rows.iter().map(|r| r.month.as_str()).collect();
|
||||||
|
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
|
||||||
|
for month in &months {
|
||||||
|
tx.execute("DELETE FROM landings WHERE month = ?", params![month])?;
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
let mut app = tx.appender("landings")?;
|
||||||
|
for row in rows {
|
||||||
|
app.append_row(params![
|
||||||
|
row.month,
|
||||||
|
row.species_code,
|
||||||
|
row.species_label,
|
||||||
|
row.gear_code,
|
||||||
|
row.zone_code,
|
||||||
|
row.processing_code,
|
||||||
|
row.preservation_code,
|
||||||
|
row.shipsize_code,
|
||||||
|
row.measure_code,
|
||||||
|
row.value,
|
||||||
|
])?;
|
||||||
|
}
|
||||||
|
app.flush()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit()?;
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"Upserted {} rows across {} month(s)",
|
||||||
|
rows.len(),
|
||||||
|
months.len()
|
||||||
|
);
|
||||||
|
Ok(rows.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_last_month(conn: &Connection) -> Result<Option<String>> {
|
||||||
|
let result = conn.query_row("SELECT MAX(month) FROM landings", [], |row| {
|
||||||
|
let val: Option<String> = row.get(0)?;
|
||||||
|
Ok(val)
|
||||||
|
})?;
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn export_parquet(conn: &Connection, path: &str) -> Result<()> {
|
||||||
|
if path.contains('\'') || path.contains('\0') {
|
||||||
|
return Err(DbError::InvalidPath(path.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !path
|
||||||
|
.chars()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '/' | '-' | '_' | ' '))
|
||||||
|
{
|
||||||
|
return Err(DbError::InvalidPath(format!(
|
||||||
|
"Path contains disallowed characters: {path}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
&format!(
|
||||||
|
"COPY (SELECT * FROM landings) TO '{}' (FORMAT PARQUET)",
|
||||||
|
path
|
||||||
|
),
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
info!("Exported landings to Parquet at {}", path);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::types::Landing;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
fn test_conn() -> Connection {
|
||||||
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
|
init_schema(&conn).unwrap();
|
||||||
|
conn
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_landings() -> Vec<Landing> {
|
||||||
|
vec![
|
||||||
|
Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "Toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(1234.5),
|
||||||
|
},
|
||||||
|
Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "VALUE".to_string(),
|
||||||
|
value: None,
|
||||||
|
},
|
||||||
|
Landing {
|
||||||
|
month: "2024M02".to_string(),
|
||||||
|
species_code: "HER".to_string(),
|
||||||
|
species_label: "Sild".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(5678.9),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_init_creates_all_tables() {
|
||||||
|
let conn = test_conn();
|
||||||
|
|
||||||
|
for table in &[
|
||||||
|
"species",
|
||||||
|
"gear",
|
||||||
|
"zone",
|
||||||
|
"processing",
|
||||||
|
"preservation",
|
||||||
|
"shipsize",
|
||||||
|
"landings",
|
||||||
|
] {
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|_| panic!("Failed to query {}", table));
|
||||||
|
assert_eq!(count, 0, "Table {} should be empty", table);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_init_is_idempotent() {
|
||||||
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
|
init_schema(&conn).unwrap();
|
||||||
|
init_schema(&conn).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_upsert_insert_and_idempotency() {
|
||||||
|
let conn = test_conn();
|
||||||
|
let rows = sample_landings();
|
||||||
|
|
||||||
|
let inserted = upsert_landings(&conn, &rows).unwrap();
|
||||||
|
assert_eq!(inserted, 3);
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 3);
|
||||||
|
|
||||||
|
let inserted = upsert_landings(&conn, &rows).unwrap();
|
||||||
|
assert_eq!(inserted, 3);
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_upsert_replaces_month_data() {
|
||||||
|
let conn = test_conn();
|
||||||
|
upsert_landings(&conn, &sample_landings()).unwrap();
|
||||||
|
|
||||||
|
let modified = vec![Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "Toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(9999.9),
|
||||||
|
}];
|
||||||
|
upsert_landings(&conn, &modified).unwrap();
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 2);
|
||||||
|
|
||||||
|
let val: f64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT value FROM landings
|
||||||
|
WHERE month = '2024M01' AND measure_code = 'MASS'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!((val - 9999.9).abs() < f64::EPSILON);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_upsert_empty_input() {
|
||||||
|
let conn = test_conn();
|
||||||
|
let result = upsert_landings(&conn, &[]);
|
||||||
|
assert!(result.is_ok());
|
||||||
|
assert_eq!(result.unwrap(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_upsert_multi_month_atomic() {
|
||||||
|
let conn = test_conn();
|
||||||
|
upsert_landings(&conn, &sample_landings()).unwrap();
|
||||||
|
|
||||||
|
let rows = vec![
|
||||||
|
Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "Toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(111.0),
|
||||||
|
},
|
||||||
|
Landing {
|
||||||
|
month: "2024M02".to_string(),
|
||||||
|
species_code: "HER".to_string(),
|
||||||
|
species_label: "Sild".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(222.0),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
upsert_landings(&conn, &rows).unwrap();
|
||||||
|
|
||||||
|
let jan_val: f64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT value FROM landings WHERE month = '2024M01' AND measure_code = 'MASS'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!((jan_val - 111.0).abs() < f64::EPSILON);
|
||||||
|
|
||||||
|
let feb_val: f64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT value FROM landings WHERE month = '2024M02' AND measure_code = 'MASS'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!((feb_val - 222.0).abs() < f64::EPSILON);
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_null_preserved_on_insert() {
|
||||||
|
let conn = test_conn();
|
||||||
|
let rows = vec![Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "VALUE".to_string(),
|
||||||
|
value: None,
|
||||||
|
}];
|
||||||
|
upsert_landings(&conn, &rows).unwrap();
|
||||||
|
|
||||||
|
let val: Option<f64> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT value FROM landings WHERE measure_code = 'VALUE'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(val.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_null_preserved_after_upsert() {
|
||||||
|
let conn = test_conn();
|
||||||
|
upsert_landings(&conn, &sample_landings()).unwrap();
|
||||||
|
|
||||||
|
let val: Option<f64> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT value FROM landings
|
||||||
|
WHERE month = '2024M01' AND measure_code = 'VALUE'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(val.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_last_month_empty_table() {
|
||||||
|
let conn = test_conn();
|
||||||
|
let result = get_last_month(&conn).unwrap();
|
||||||
|
assert!(result.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_last_month_populated() {
|
||||||
|
let conn = test_conn();
|
||||||
|
upsert_landings(&conn, &sample_landings()).unwrap();
|
||||||
|
|
||||||
|
let last = get_last_month(&conn).unwrap();
|
||||||
|
assert_eq!(last.as_deref(), Some("2024M02"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_last_month_single_month() {
|
||||||
|
let conn = test_conn();
|
||||||
|
let rows = vec![Landing {
|
||||||
|
month: "2015M03".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "Toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(100.0),
|
||||||
|
}];
|
||||||
|
upsert_landings(&conn, &rows).unwrap();
|
||||||
|
|
||||||
|
let last = get_last_month(&conn).unwrap();
|
||||||
|
assert_eq!(last.as_deref(), Some("2015M03"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_update_lookups_populates_tables() {
|
||||||
|
let conn = test_conn();
|
||||||
|
|
||||||
|
let mut maps = LookupMap::new();
|
||||||
|
|
||||||
|
let mut species = HashMap::new();
|
||||||
|
species.insert("COD".to_string(), "Toskur".to_string());
|
||||||
|
species.insert("HER".to_string(), "Sild".to_string());
|
||||||
|
maps.insert("Species (ASFIS2022)".to_string(), species);
|
||||||
|
|
||||||
|
let mut gear = HashMap::new();
|
||||||
|
gear.insert("TOTAL".to_string(), "Tilsamans".to_string());
|
||||||
|
maps.insert("Fishing Gear (ISSCFG2016)".to_string(), gear);
|
||||||
|
|
||||||
|
update_lookups(&conn, &maps).unwrap();
|
||||||
|
|
||||||
|
let label: String = conn
|
||||||
|
.query_row("SELECT label FROM species WHERE code = 'COD'", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(label, "Toskur");
|
||||||
|
|
||||||
|
let label: String = conn
|
||||||
|
.query_row("SELECT label FROM gear WHERE code = 'TOTAL'", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(label, "Tilsamans");
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_update_lookups_upsert_existing() {
|
||||||
|
let conn = test_conn();
|
||||||
|
|
||||||
|
let mut maps = LookupMap::new();
|
||||||
|
let mut species = HashMap::new();
|
||||||
|
species.insert("COD".to_string(), "Old label".to_string());
|
||||||
|
maps.insert("Species (ASFIS2022)".to_string(), species);
|
||||||
|
|
||||||
|
update_lookups(&conn, &maps).unwrap();
|
||||||
|
|
||||||
|
let mut species2 = HashMap::new();
|
||||||
|
species2.insert("COD".to_string(), "Toskur".to_string());
|
||||||
|
let mut maps2 = LookupMap::new();
|
||||||
|
maps2.insert("Species (ASFIS2022)".to_string(), species2);
|
||||||
|
|
||||||
|
update_lookups(&conn, &maps2).unwrap();
|
||||||
|
|
||||||
|
let label: String = conn
|
||||||
|
.query_row("SELECT label FROM species WHERE code = 'COD'", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(label, "Toskur");
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_update_lookups_skips_missing_dims() {
|
||||||
|
let conn = test_conn();
|
||||||
|
|
||||||
|
let mut maps = LookupMap::new();
|
||||||
|
let mut species = HashMap::new();
|
||||||
|
species.insert("COD".to_string(), "Toskur".to_string());
|
||||||
|
maps.insert("Species (ASFIS2022)".to_string(), species);
|
||||||
|
|
||||||
|
update_lookups(&conn, &maps).unwrap();
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row("SELECT COUNT(*) FROM gear", [], |row| row.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_export_parquet() {
|
||||||
|
let conn = test_conn();
|
||||||
|
upsert_landings(&conn, &sample_landings()).unwrap();
|
||||||
|
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
let path = tmp.path().to_str().unwrap().to_string() + ".parquet";
|
||||||
|
|
||||||
|
export_parquet(&conn, &path).unwrap();
|
||||||
|
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
&format!("SELECT COUNT(*) FROM read_parquet('{}')", path),
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_faroese_labels_round_trip() {
|
||||||
|
let conn = test_conn();
|
||||||
|
|
||||||
|
let rows = vec![Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "148XXXXXXX00000".to_string(),
|
||||||
|
species_label: "Hýsa".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(100.0),
|
||||||
|
}];
|
||||||
|
upsert_landings(&conn, &rows).unwrap();
|
||||||
|
|
||||||
|
let label: String = conn
|
||||||
|
.query_row("SELECT species_label FROM landings LIMIT 1", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(label, "Hýsa");
|
||||||
|
assert!(label.contains('ý'));
|
||||||
|
assert!(!label.contains('\u{FFFD}'));
|
||||||
|
}
|
||||||
|
}
|
||||||
+643
-355
File diff suppressed because it is too large
Load Diff
+380
-2
@@ -1,3 +1,381 @@
|
|||||||
fn main() {
|
mod api;
|
||||||
println!("Hello, world!");
|
mod cli;
|
||||||
|
mod db;
|
||||||
|
mod ingest;
|
||||||
|
mod types;
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use axum::serve;
|
||||||
|
use clap::Parser;
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
use tracing_subscriber::EnvFilter;
|
||||||
|
|
||||||
|
struct LoggerGuard {
|
||||||
|
_guard: Option<tracing_appender::non_blocking::WorkerGuard>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
|
||||||
|
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||||
|
let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json");
|
||||||
|
|
||||||
|
match log_file_path {
|
||||||
|
Some(path) => {
|
||||||
|
let file = std::fs::File::create(path).expect("Failed to create log file");
|
||||||
|
let (non_blocking, guard) = tracing_appender::non_blocking(file);
|
||||||
|
|
||||||
|
let subscriber = tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(filter)
|
||||||
|
.with_writer(non_blocking);
|
||||||
|
|
||||||
|
if is_json {
|
||||||
|
subscriber.json().init();
|
||||||
|
} else {
|
||||||
|
subscriber.init();
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggerGuard {
|
||||||
|
_guard: Some(guard),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
let subscriber = tracing_subscriber::fmt().with_env_filter(filter);
|
||||||
|
|
||||||
|
if is_json {
|
||||||
|
subscriber.json().init();
|
||||||
|
} else {
|
||||||
|
subscriber.init();
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggerGuard { _guard: None }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_stale_parquet_files() {
|
||||||
|
let tmp_dir = std::env::temp_dir();
|
||||||
|
match std::fs::read_dir(&tmp_dir) {
|
||||||
|
Ok(entries) => {
|
||||||
|
for entry in entries.filter_map(|e| e.ok()) {
|
||||||
|
if let Some(name) = entry.file_name().to_str()
|
||||||
|
&& name.starts_with("hagfish-")
|
||||||
|
&& name.ends_with(".parquet")
|
||||||
|
&& let Ok(metadata) = entry.metadata()
|
||||||
|
&& let Ok(modified) = metadata.modified()
|
||||||
|
{
|
||||||
|
let age = std::time::SystemTime::now().duration_since(modified);
|
||||||
|
if age.map(|d| d.as_secs() > 3600).unwrap_or(false) {
|
||||||
|
let _ = std::fs::remove_file(entry.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => tracing::warn!(error = %e, "Failed to list temp dir"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
let cli = cli::Cli::parse();
|
||||||
|
|
||||||
|
let config = match &cli.config {
|
||||||
|
Some(path) => types::Config::load(path)?,
|
||||||
|
None => {
|
||||||
|
let default_path = std::path::Path::new("config.json");
|
||||||
|
if default_path.exists() {
|
||||||
|
types::Config::load(default_path)?
|
||||||
|
} else {
|
||||||
|
eprintln!("WARN: No config file found, using defaults");
|
||||||
|
types::Config::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let _logger_guard = init_logging(config.log_file_path.as_deref());
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
duckdb_path = %config.duckdb_path,
|
||||||
|
bind_address = %config.bind_address,
|
||||||
|
"configuration loaded"
|
||||||
|
);
|
||||||
|
|
||||||
|
match cli.command {
|
||||||
|
cli::Command::Serve => run_serve(&config).await,
|
||||||
|
cli::Command::Ingest { full } => run_ingest(&config, full).await,
|
||||||
|
cli::Command::Export { out } => run_export(&config, &out),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_serve(config: &types::Config) -> anyhow::Result<()> {
|
||||||
|
cleanup_stale_parquet_files();
|
||||||
|
|
||||||
|
let conn = db::init(&config.duckdb_path)?;
|
||||||
|
|
||||||
|
let state = api::AppState {
|
||||||
|
conn: Arc::new(tokio::sync::Mutex::new(conn)),
|
||||||
|
config: Arc::new(config.clone()),
|
||||||
|
};
|
||||||
|
|
||||||
|
let app = api::build_router(state);
|
||||||
|
|
||||||
|
let listener = TcpListener::bind(&config.bind_address).await?;
|
||||||
|
tracing::info!(address = %config.bind_address, "hagfish server started");
|
||||||
|
|
||||||
|
serve(listener, app).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
|
||||||
|
let run_start = Instant::now();
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
mode = if full { "full" } else { "incremental" },
|
||||||
|
"starting ingestion run"
|
||||||
|
);
|
||||||
|
|
||||||
|
let conn = db::init(&config.duckdb_path)?;
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(std::time::Duration::from_secs(60))
|
||||||
|
.build()?;
|
||||||
|
|
||||||
|
let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url)
|
||||||
|
.await
|
||||||
|
.inspect_err(|e| {
|
||||||
|
tracing::error!(error = %e, stage = "fetch_metadata", "ingestion failed");
|
||||||
|
})?;
|
||||||
|
|
||||||
|
db::update_lookups(&conn, &lookup_maps)?;
|
||||||
|
|
||||||
|
let species_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_SPECIES);
|
||||||
|
let gear_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_GEAR);
|
||||||
|
let zone_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_ZONE);
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
species_count = species_codes.len(),
|
||||||
|
gear_count = gear_codes.len(),
|
||||||
|
zone_count = zone_codes.len(),
|
||||||
|
"Extracted non-TOTAL dimension values for leaf-only query"
|
||||||
|
);
|
||||||
|
|
||||||
|
let all_months = ingest::extract_available_months(&metadata);
|
||||||
|
|
||||||
|
if full {
|
||||||
|
conn.execute("DELETE FROM landings", [])?;
|
||||||
|
tracing::info!("Cleared landings table for full re-ingest");
|
||||||
|
}
|
||||||
|
|
||||||
|
let months_to_fetch: Vec<String> = if full {
|
||||||
|
all_months
|
||||||
|
} else {
|
||||||
|
let last = db::get_last_month(&conn)?;
|
||||||
|
match last {
|
||||||
|
Some(last_month) => all_months
|
||||||
|
.into_iter()
|
||||||
|
.filter(|m| m.as_str() > last_month.as_str())
|
||||||
|
.collect(),
|
||||||
|
None => all_months,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if months_to_fetch.is_empty() {
|
||||||
|
let elapsed = run_start.elapsed();
|
||||||
|
tracing::info!(
|
||||||
|
mode = if full { "full" } else { "incremental" },
|
||||||
|
months_requested = 0,
|
||||||
|
total_rows = 0,
|
||||||
|
batches = 0,
|
||||||
|
errors = 0,
|
||||||
|
elapsed_ms = elapsed.as_millis() as u64,
|
||||||
|
"ingestion complete — no new months"
|
||||||
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
const BATCH_SIZE: usize = 12;
|
||||||
|
let batches = ingest::chunk_months(&months_to_fetch, BATCH_SIZE);
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
mode = if full { "full" } else { "incremental" },
|
||||||
|
months_requested = months_to_fetch.len(),
|
||||||
|
batches = batches.len(),
|
||||||
|
"beginning batched ingestion"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut total_rows = 0;
|
||||||
|
let mut errors = 0;
|
||||||
|
|
||||||
|
for (i, batch) in batches.iter().enumerate() {
|
||||||
|
let batch_start = Instant::now();
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
batch = i + 1,
|
||||||
|
total_batches = batches.len(),
|
||||||
|
months_in_batch = batch.len(),
|
||||||
|
months = %batch.join(", "),
|
||||||
|
"fetching batch"
|
||||||
|
);
|
||||||
|
|
||||||
|
let query = ingest::build_query(batch, &species_codes, &gear_codes, &zone_codes)?;
|
||||||
|
|
||||||
|
let data = match ingest::fetch_data(&client, &config.data_source_url, &query).await {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(
|
||||||
|
batch = i + 1,
|
||||||
|
error = %e,
|
||||||
|
stage = "fetch_data",
|
||||||
|
"batch failed, continuing to next batch"
|
||||||
|
);
|
||||||
|
errors += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut landings = Vec::with_capacity(data.dataset.value.len());
|
||||||
|
let mut parse_errors = 0;
|
||||||
|
|
||||||
|
for j in 0..data.dataset.value.len() {
|
||||||
|
match ingest::parse_row(j, &data.dataset, &lookup_maps) {
|
||||||
|
Ok(row) => {
|
||||||
|
if row.species_code == "TOTAL"
|
||||||
|
|| row.gear_code == "TOTAL"
|
||||||
|
|| row.zone_code == "TOTAL"
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
landings.push(ingest::data_row_to_landing(row));
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
batch = i + 1,
|
||||||
|
row_index = j,
|
||||||
|
error = %e,
|
||||||
|
"failed to parse row"
|
||||||
|
);
|
||||||
|
parse_errors += 1;
|
||||||
|
errors += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let count = match db::upsert_landings(&conn, &landings) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(
|
||||||
|
batch = i + 1,
|
||||||
|
error = %e,
|
||||||
|
stage = "upsert_landings",
|
||||||
|
"batch upsert failed"
|
||||||
|
);
|
||||||
|
errors += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
total_rows += count;
|
||||||
|
let batch_elapsed = batch_start.elapsed();
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
batch = i + 1,
|
||||||
|
total_batches = batches.len(),
|
||||||
|
rows_inserted = count,
|
||||||
|
parse_errors = parse_errors,
|
||||||
|
elapsed_ms = batch_elapsed.as_millis() as u64,
|
||||||
|
"batch complete"
|
||||||
|
);
|
||||||
|
|
||||||
|
if i < batches.len() - 1 {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let elapsed = run_start.elapsed();
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
mode = if full { "full" } else { "incremental" },
|
||||||
|
months_requested = months_to_fetch.len(),
|
||||||
|
batches = batches.len(),
|
||||||
|
total_rows = total_rows,
|
||||||
|
errors = errors,
|
||||||
|
elapsed_ms = elapsed.as_millis() as u64,
|
||||||
|
"ingestion run complete"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_export(config: &types::Config, out: &Path) -> anyhow::Result<()> {
|
||||||
|
let run_start = Instant::now();
|
||||||
|
|
||||||
|
let conn = db::init(&config.duckdb_path)?;
|
||||||
|
let path_str = out
|
||||||
|
.to_str()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("Output path contains invalid UTF-8"))?;
|
||||||
|
|
||||||
|
db::export_parquet(&conn, path_str)?;
|
||||||
|
|
||||||
|
let elapsed = run_start.elapsed();
|
||||||
|
tracing::info!(
|
||||||
|
path = path_str,
|
||||||
|
elapsed_ms = elapsed.as_millis() as u64,
|
||||||
|
"export complete"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::types::Config;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_default_values() {
|
||||||
|
let config = Config::default();
|
||||||
|
assert_eq!(config.duckdb_path, "hagfish.db");
|
||||||
|
assert_eq!(config.bind_address, "127.0.0.1:8090");
|
||||||
|
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_run_export_writes_valid_parquet() {
|
||||||
|
let tmp_dir = tempfile::tempdir().unwrap();
|
||||||
|
let db_path = tmp_dir.path().join("test.db");
|
||||||
|
let parquet_path = tmp_dir.path().join("export.parquet");
|
||||||
|
|
||||||
|
let conn = db::init(db_path.to_str().unwrap()).unwrap();
|
||||||
|
|
||||||
|
let rows = vec![crate::types::Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "Toskur".to_string(),
|
||||||
|
gear_code: "TR1".to_string(),
|
||||||
|
zone_code: "FO".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(1000.0),
|
||||||
|
}];
|
||||||
|
|
||||||
|
db::upsert_landings(&conn, &rows).unwrap();
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
|
let config = Config {
|
||||||
|
duckdb_path: db_path.to_str().unwrap().to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = run_export(&config, &parquet_path);
|
||||||
|
assert!(result.is_ok());
|
||||||
|
|
||||||
|
let metadata = std::fs::metadata(&parquet_path).unwrap();
|
||||||
|
assert!(metadata.len() > 0);
|
||||||
|
|
||||||
|
let file_bytes = std::fs::read(&parquet_path).unwrap();
|
||||||
|
assert_eq!(&file_bytes[..4], b"PAR1");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+234
-54
@@ -1,18 +1,19 @@
|
|||||||
//! Type definitions for hagfish data structures.
|
|
||||||
//!
|
|
||||||
//! These types represent the PX-Web API contract and internal data models.
|
|
||||||
//! All public types derive Serialize/Deserialize for JSON (de)serialization.
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
/// Configuration loaded from config.json on startup.
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub duckdb_path: String,
|
pub duckdb_path: String,
|
||||||
pub bind_address: String,
|
pub bind_address: String,
|
||||||
pub data_source_url: String,
|
pub data_source_url: String,
|
||||||
pub log_file_path: Option<String>,
|
pub log_file_path: Option<String>,
|
||||||
|
#[serde(default = "default_allowed_origins")]
|
||||||
|
pub allowed_origins: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_allowed_origins() -> Vec<String> {
|
||||||
|
vec![]
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for Config {
|
impl Default for Config {
|
||||||
@@ -20,92 +21,128 @@ impl Default for Config {
|
|||||||
Self {
|
Self {
|
||||||
duckdb_path: "hagfish.db".to_string(),
|
duckdb_path: "hagfish.db".to_string(),
|
||||||
bind_address: "127.0.0.1:8090".to_string(),
|
bind_address: "127.0.0.1:8090".to_string(),
|
||||||
data_source_url: "https://statbank.hagstova.fo/data/api/table/fisknv_md".to_string(),
|
data_source_url: "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
|
||||||
|
.to_string(),
|
||||||
log_file_path: Some("hagfish.log".to_string()),
|
log_file_path: Some("hagfish.log".to_string()),
|
||||||
|
allowed_origins: default_allowed_origins(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Metadata response from PX-Web API GET request.
|
impl Config {
|
||||||
/// Contains variable definitions and value codes with labels.
|
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||||
|
let contents = std::fs::read_to_string(path)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to read config file {}: {e}", path.display()))?;
|
||||||
|
let config: Config = serde_json::from_str(&contents)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {e}", path.display()))?;
|
||||||
|
tracing::info!("Loaded config from {}", path.display());
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct MetadataResponse {
|
pub struct MetadataResponse {
|
||||||
|
#[serde(default)]
|
||||||
|
pub title: Option<String>,
|
||||||
pub variables: Vec<VariableMeta>,
|
pub variables: Vec<VariableMeta>,
|
||||||
pub values: HashMap<String, Vec<ValueMeta>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Variable metadata describing a dimension in the PX-Web table.
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct AvailableFiltersDto {
|
||||||
|
pub species: Vec<String>,
|
||||||
|
pub zones: Vec<String>,
|
||||||
|
pub gear: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct VariableMeta {
|
pub struct VariableMeta {
|
||||||
pub id: String,
|
|
||||||
pub text: String,
|
|
||||||
pub role: String,
|
|
||||||
/// Position in the key array (0-indexed)
|
|
||||||
#[serde(rename = "keyPosition", skip_serializing_if = "Option::is_none")]
|
|
||||||
pub key_position: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Value metadata: code → display label mapping for a variable.
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
|
||||||
pub struct ValueMeta {
|
|
||||||
pub code: String,
|
pub code: String,
|
||||||
|
#[serde(default)]
|
||||||
pub text: String,
|
pub text: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub role: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub values: Vec<String>,
|
||||||
|
#[serde(default, rename = "valueTexts")]
|
||||||
|
pub value_texts: Vec<String>,
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub extra: HashMap<String, serde_json::Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VariableMeta {
|
||||||
|
pub fn lookup_map(&self) -> HashMap<String, String> {
|
||||||
|
self.values
|
||||||
|
.iter()
|
||||||
|
.zip(self.value_texts.iter())
|
||||||
|
.map(|(code, label)| (code.clone(), label.clone()))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Query body sent to PX-Web API POST endpoint.
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct Query {
|
pub struct Query {
|
||||||
pub query: Vec<QueryItem>,
|
pub query: Vec<QueryItem>,
|
||||||
pub language: String,
|
pub response: QueryResponse,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Single query item representing a dimension selection.
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct QueryItem {
|
pub struct QueryItem {
|
||||||
pub id: String,
|
pub code: String,
|
||||||
|
pub selection: Selection,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct Selection {
|
||||||
|
pub filter: String,
|
||||||
pub values: Vec<String>,
|
pub values: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Selection helper for building queries programmatically.
|
#[derive(Debug, Clone, Serialize)]
|
||||||
#[derive(Debug, Clone)]
|
pub struct QueryResponse {
|
||||||
pub struct Selection {
|
pub format: String,
|
||||||
/// Dimension name (matches variable.id from metadata)
|
}
|
||||||
pub dimension: String,
|
|
||||||
/// Codes to include, or "*" for all
|
impl Default for QueryResponse {
|
||||||
pub codes: Vec<String>,
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
format: "json-stat2".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Raw data response from PX-Web API POST request.
|
|
||||||
/// JSON-stat2 format with metadata and data sections.
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct DataResponse {
|
pub struct DataResponse {
|
||||||
|
#[serde(flatten)]
|
||||||
pub dataset: Dataset,
|
pub dataset: Dataset,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Dataset wrapper containing dimensions and actual values.
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct Dataset {
|
pub struct Dataset {
|
||||||
pub dimension: DimInfo,
|
#[serde(default)]
|
||||||
|
pub id: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub size: Vec<usize>,
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub extra_fields: HashMap<String, serde_json::Value>,
|
||||||
|
pub dimension: HashMap<String, Dimension>,
|
||||||
pub value: Vec<Option<f64>>,
|
pub value: Vec<Option<f64>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Dimension metadata describing key layout.
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct DimInfo {
|
pub struct Dimension {
|
||||||
#[serde(rename = "key")]
|
#[serde(default)]
|
||||||
pub keys: Vec<String>,
|
pub label: String,
|
||||||
pub category: CategoryInfo,
|
pub category: CategoryInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Category info containing value lists for each dimension.
|
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
pub struct CategoryInfo {
|
pub struct CategoryInfo {
|
||||||
pub label: HashMap<String, Vec<String>>,
|
pub index: HashMap<String, usize>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(default)]
|
||||||
pub index: Option<HashMap<String, Vec<usize>>>,
|
pub label: HashMap<String, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Decoded row of landing data with labeled dimensions.
|
#[allow(dead_code)]
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct DataRow {
|
pub struct DataRow {
|
||||||
pub month: String,
|
pub month: String,
|
||||||
@@ -126,7 +163,6 @@ pub struct DataRow {
|
|||||||
pub value: Option<f64>,
|
pub value: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Structured representation of a single landing record.
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Landing {
|
pub struct Landing {
|
||||||
pub month: String,
|
pub month: String,
|
||||||
@@ -141,7 +177,86 @@ pub struct Landing {
|
|||||||
pub value: Option<f64>,
|
pub value: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Error types for ingestion module.
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SpeciesDto {
|
||||||
|
pub code: String,
|
||||||
|
pub label: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct LookupDto {
|
||||||
|
pub code: String,
|
||||||
|
pub label: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct LandingDto {
|
||||||
|
pub month: String,
|
||||||
|
pub species_code: String,
|
||||||
|
pub species_label: String,
|
||||||
|
pub gear_code: String,
|
||||||
|
pub zone_code: String,
|
||||||
|
pub processing_code: String,
|
||||||
|
pub preservation_code: String,
|
||||||
|
pub shipsize_code: String,
|
||||||
|
pub measure_code: String,
|
||||||
|
pub value: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Landing> for LandingDto {
|
||||||
|
fn from(l: Landing) -> Self {
|
||||||
|
Self {
|
||||||
|
month: l.month,
|
||||||
|
species_code: l.species_code,
|
||||||
|
species_label: l.species_label,
|
||||||
|
gear_code: l.gear_code,
|
||||||
|
zone_code: l.zone_code,
|
||||||
|
processing_code: l.processing_code,
|
||||||
|
preservation_code: l.preservation_code,
|
||||||
|
shipsize_code: l.shipsize_code,
|
||||||
|
measure_code: l.measure_code,
|
||||||
|
value: l.value,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct MonthlyAggregate {
|
||||||
|
pub month: String,
|
||||||
|
pub total_mass: Option<f64>,
|
||||||
|
pub total_value: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct TopSpecies {
|
||||||
|
pub species_code: String,
|
||||||
|
pub species_label: String,
|
||||||
|
pub total_value: Option<f64>,
|
||||||
|
pub total_mass: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct PriceTrend {
|
||||||
|
pub month: String,
|
||||||
|
pub price_per_kg: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SummaryDto {
|
||||||
|
pub monthly: Vec<MonthlyAggregate>,
|
||||||
|
pub top_species: Vec<TopSpecies>,
|
||||||
|
pub price_trend: Vec<PriceTrend>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct MonthlySpeciesBreakdown {
|
||||||
|
pub month: String,
|
||||||
|
pub species_code: String,
|
||||||
|
pub species_label: String,
|
||||||
|
pub total_value: Option<f64>,
|
||||||
|
pub total_mass: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum IngestError {
|
pub enum IngestError {
|
||||||
#[error("HTTP request failed: {0}")]
|
#[error("HTTP request failed: {0}")]
|
||||||
@@ -150,6 +265,9 @@ pub enum IngestError {
|
|||||||
#[error("JSON parsing failed: {0}")]
|
#[error("JSON parsing failed: {0}")]
|
||||||
JsonError(#[from] serde_json::Error),
|
JsonError(#[from] serde_json::Error),
|
||||||
|
|
||||||
|
#[error("API returned status {status}: {body}")]
|
||||||
|
ApiStatus { status: u16, body: String },
|
||||||
|
|
||||||
#[error("Missing dimension in response: {0}")]
|
#[error("Missing dimension in response: {0}")]
|
||||||
MissingDimension(String),
|
MissingDimension(String),
|
||||||
|
|
||||||
@@ -158,14 +276,76 @@ pub enum IngestError {
|
|||||||
|
|
||||||
#[error("API returned empty data set")]
|
#[error("API returned empty data set")]
|
||||||
EmptyDataset,
|
EmptyDataset,
|
||||||
|
|
||||||
#[error("Unicode decode error: {0}")]
|
|
||||||
UnicodeError(String),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lookup map for decoding key arrays into labeled values.
|
|
||||||
/// Keyed by dimension name, contains code → label mappings.
|
|
||||||
pub type LookupMap = HashMap<String, HashMap<String, String>>;
|
pub type LookupMap = HashMap<String, HashMap<String, String>>;
|
||||||
|
|
||||||
/// Result alias using custom error type.
|
|
||||||
pub type Result<T> = std::result::Result<T, IngestError>;
|
pub type Result<T> = std::result::Result<T, IngestError>;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_missing_file_hard_error() {
|
||||||
|
let result = Config::load(Path::new("/nonexistent/path/config.json"));
|
||||||
|
assert!(result.is_err());
|
||||||
|
let err_msg = result.unwrap_err().to_string();
|
||||||
|
assert!(err_msg.contains("Failed to read config file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_invalid_json_error() {
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
std::fs::write(tmp.path(), "not valid json {{{").unwrap();
|
||||||
|
|
||||||
|
let result = Config::load(tmp.path());
|
||||||
|
assert!(result.is_err());
|
||||||
|
let err_msg = result.unwrap_err().to_string();
|
||||||
|
assert!(err_msg.contains("Failed to parse config file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_valid_file() {
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
let content = r#"{
|
||||||
|
"duckdb_path": "test.db",
|
||||||
|
"bind_address": "127.0.0.1:9090",
|
||||||
|
"data_source_url": "http://example.com",
|
||||||
|
"log_file_path": null,
|
||||||
|
"allowed_origins": ["https://example.com"]
|
||||||
|
}"#;
|
||||||
|
std::fs::write(tmp.path(), content).unwrap();
|
||||||
|
|
||||||
|
let config = Config::load(tmp.path()).unwrap();
|
||||||
|
assert_eq!(config.duckdb_path, "test.db");
|
||||||
|
assert_eq!(config.bind_address, "127.0.0.1:9090");
|
||||||
|
assert_eq!(config.data_source_url, "http://example.com");
|
||||||
|
assert!(config.log_file_path.is_none());
|
||||||
|
assert_eq!(config.allowed_origins, vec!["https://example.com"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_defaults_allowed_origins() {
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
let content = r#"{
|
||||||
|
"duckdb_path": "test.db",
|
||||||
|
"bind_address": "127.0.0.1:9090",
|
||||||
|
"data_source_url": "http://example.com"
|
||||||
|
}"#;
|
||||||
|
std::fs::write(tmp.path(), content).unwrap();
|
||||||
|
|
||||||
|
let config = Config::load(tmp.path()).unwrap();
|
||||||
|
assert!(config.allowed_origins.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_default_values() {
|
||||||
|
let config = Config::default();
|
||||||
|
assert_eq!(config.duckdb_path, "hagfish.db");
|
||||||
|
assert_eq!(config.bind_address, "127.0.0.1:8090");
|
||||||
|
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
|
||||||
|
assert!(config.log_file_path.is_some());
|
||||||
|
assert!(config.allowed_origins.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+857
@@ -0,0 +1,857 @@
|
|||||||
|
let chartLine, chartPrice, chartBar;
|
||||||
|
let allMonths = [];
|
||||||
|
const MONTHS_TO_SHOW_DEFAULT = 6;
|
||||||
|
|
||||||
|
function formatMonthDisplay(monthCode) {
|
||||||
|
const monthNames = [
|
||||||
|
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
|
||||||
|
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
|
||||||
|
];
|
||||||
|
const year = monthCode.substring(0, 4);
|
||||||
|
const monthIdx = parseInt(monthCode.substring(5, 7)) - 1;
|
||||||
|
return `${monthNames[monthIdx]} ${year}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDKK(value) {
|
||||||
|
return new Intl.NumberFormat('da-DK', {
|
||||||
|
minimumFractionDigits: 0,
|
||||||
|
maximumFractionDigits: 0,
|
||||||
|
}).format(value) + ' DKK';
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDKKShort(value) {
|
||||||
|
if (value >= 1e9) return (value / 1e9).toFixed(1).replace('.', ',') + ' mia.';
|
||||||
|
if (value >= 1e6) return (value / 1e6).toFixed(1).replace('.', ',') + ' mio.';
|
||||||
|
if (value >= 1e3) return (value / 1e3).toFixed(0) + ' tus.';
|
||||||
|
return value.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatPricePerKg(value) {
|
||||||
|
return new Intl.NumberFormat('da-DK', {
|
||||||
|
minimumFractionDigits: 2,
|
||||||
|
maximumFractionDigits: 2,
|
||||||
|
}).format(value) + ' DKK/kg';
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNextUpdateDate() {
|
||||||
|
const now = new Date();
|
||||||
|
const currentMonth = now.getMonth();
|
||||||
|
const currentYear = now.getFullYear();
|
||||||
|
const today = now.getDate();
|
||||||
|
|
||||||
|
let updateMonth = currentMonth;
|
||||||
|
let updateYear = currentYear;
|
||||||
|
|
||||||
|
if (today > 28) {
|
||||||
|
updateMonth++;
|
||||||
|
if (updateMonth > 11) {
|
||||||
|
updateMonth = 0;
|
||||||
|
updateYear++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const updateDate = new Date(updateYear, updateMonth, 28);
|
||||||
|
const yyyy = updateDate.getFullYear();
|
||||||
|
const mm = String(updateDate.getMonth() + 1).padStart(2, '0');
|
||||||
|
const dd = String(updateDate.getDate()).padStart(2, '0');
|
||||||
|
|
||||||
|
return `${yyyy}-${mm}-${dd}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateNavbarUpdateDate() {
|
||||||
|
const container = document.getElementById('update-date-display');
|
||||||
|
if (container) {
|
||||||
|
container.textContent = `Next update: ${getNextUpdateDate()}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getThemeColors() {
|
||||||
|
const theme = document.documentElement.getAttribute('data-theme');
|
||||||
|
const isDark = theme === 'macchiato';
|
||||||
|
|
||||||
|
if (isDark) {
|
||||||
|
return {
|
||||||
|
isDark: true,
|
||||||
|
textColor: '#cdd6f4',
|
||||||
|
subTextColor: '#bac2de',
|
||||||
|
gridColor: 'rgba(180, 185, 200, 0.12)',
|
||||||
|
accent: '#89b4fa',
|
||||||
|
secondary: '#94e2d5',
|
||||||
|
palette: [
|
||||||
|
'#89b4fa',
|
||||||
|
'#fab387',
|
||||||
|
'#a6e3a1',
|
||||||
|
'#f9e2af',
|
||||||
|
'#f5c2e7',
|
||||||
|
'#cba6f7',
|
||||||
|
'#f38ba8',
|
||||||
|
'#eba0ac',
|
||||||
|
'#94e2d5',
|
||||||
|
'#89dceb',
|
||||||
|
],
|
||||||
|
tooltipBg: 'rgba(30, 32, 48, 0.95)',
|
||||||
|
tooltipBorder: 'rgba(180, 185, 200, 0.2)',
|
||||||
|
bgTransparent: 'transparent',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
isDark: false,
|
||||||
|
textColor: '#45475a',
|
||||||
|
subTextColor: '#7f849c',
|
||||||
|
gridColor: 'rgba(73, 77, 113, 0.08)',
|
||||||
|
accent: '#89b4fa',
|
||||||
|
secondary: '#94e2d5',
|
||||||
|
palette: [
|
||||||
|
'#89b4fa',
|
||||||
|
'#fab387',
|
||||||
|
'#a6e3a1',
|
||||||
|
'#f9e2af',
|
||||||
|
'#f5c2e7',
|
||||||
|
'#cba6f7',
|
||||||
|
'#f38ba8',
|
||||||
|
'#eba0ac',
|
||||||
|
'#94e2d5',
|
||||||
|
'#89dceb',
|
||||||
|
],
|
||||||
|
tooltipBg: 'rgba(255, 255, 255, 0.98)',
|
||||||
|
tooltipBorder: 'rgba(73, 77, 113, 0.15)',
|
||||||
|
bgTransparent: 'transparent',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function initCharts() {
|
||||||
|
if (chartLine) chartLine.dispose();
|
||||||
|
if (chartPrice) chartPrice.dispose();
|
||||||
|
if (chartBar) chartBar.dispose();
|
||||||
|
|
||||||
|
chartLine = echarts.init(document.getElementById('chart-line'));
|
||||||
|
chartPrice = echarts.init(document.getElementById('chart-price'));
|
||||||
|
chartBar = echarts.init(document.getElementById('chart-bar'));
|
||||||
|
|
||||||
|
chartLine.setOption({ series: [] });
|
||||||
|
chartPrice.setOption({ series: [] });
|
||||||
|
chartBar.setOption({ series: [] });
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
chartLine.resize();
|
||||||
|
chartPrice.resize();
|
||||||
|
chartBar.resize();
|
||||||
|
}, 0);
|
||||||
|
|
||||||
|
window.addEventListener('resize', () => {
|
||||||
|
chartLine.resize();
|
||||||
|
chartPrice.resize();
|
||||||
|
chartBar.resize();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderLineChart(months, massData, valueData) {
|
||||||
|
const c = getThemeColors();
|
||||||
|
|
||||||
|
const formattedMonths = months.map(m => formatMonthDisplay(m));
|
||||||
|
const step = months.length > 24 ? 6 : (months.length > 12 ? 3 : 1);
|
||||||
|
|
||||||
|
const xLabels = formattedMonths.map((label, idx) =>
|
||||||
|
idx % step === 0 ? label : ''
|
||||||
|
);
|
||||||
|
|
||||||
|
chartLine.setOption({
|
||||||
|
backgroundColor: c.bgTransparent,
|
||||||
|
textStyle: { fontFamily: 'Noto Sans', color: c.textColor },
|
||||||
|
tooltip: {
|
||||||
|
trigger: 'axis',
|
||||||
|
axisPointer: {
|
||||||
|
type: 'cross',
|
||||||
|
lineStyle: { color: c.gridColor, type: 'dashed' },
|
||||||
|
crossStyle: { color: c.gridColor },
|
||||||
|
},
|
||||||
|
backgroundColor: c.tooltipBg,
|
||||||
|
borderColor: c.tooltipBorder,
|
||||||
|
borderWidth: 1,
|
||||||
|
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
|
||||||
|
},
|
||||||
|
legend: {
|
||||||
|
data: ['Mass (kg)', 'Value (DKK)'],
|
||||||
|
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
|
||||||
|
top: 0,
|
||||||
|
left: 'center',
|
||||||
|
orient: 'horizontal',
|
||||||
|
},
|
||||||
|
grid: {
|
||||||
|
left: '8%',
|
||||||
|
right: '10%',
|
||||||
|
bottom: 50,
|
||||||
|
containLabel: true,
|
||||||
|
top: '18%',
|
||||||
|
},
|
||||||
|
xAxis: {
|
||||||
|
type: 'category',
|
||||||
|
data: xLabels,
|
||||||
|
axisLine: { lineStyle: { color: c.gridColor } },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
interval: 0,
|
||||||
|
rotate: 0,
|
||||||
|
fontSize: 10,
|
||||||
|
margin: 8,
|
||||||
|
overflow: 'break',
|
||||||
|
width: 70,
|
||||||
|
},
|
||||||
|
boundaryGap: false,
|
||||||
|
},
|
||||||
|
yAxis: [
|
||||||
|
{
|
||||||
|
type: 'value',
|
||||||
|
name: 'Mass (kg)',
|
||||||
|
position: 'left',
|
||||||
|
splitLine: { lineStyle: { color: c.gridColor } },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
formatter: (val) => (val >= 1e6 ? (val / 1e6).toFixed(1) + 'M' : val.toLocaleString()),
|
||||||
|
margin: 8,
|
||||||
|
},
|
||||||
|
axisLine: { show: true, lineStyle: { color: c.gridColor } },
|
||||||
|
nameTextStyle: { color: c.accent, padding: [0, 0, 0, 0] },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'value',
|
||||||
|
name: 'Value (DKK)',
|
||||||
|
position: 'right',
|
||||||
|
splitLine: { show: false },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
formatter: (val) => (val >= 1e6 ? (val / 1e6).toFixed(1) + 'M' : val.toLocaleString()),
|
||||||
|
margin: 8,
|
||||||
|
},
|
||||||
|
axisLine: { show: true, lineStyle: { color: c.gridColor } },
|
||||||
|
nameTextStyle: { color: c.secondary, padding: [0, 10, 0, 0] },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
series: [
|
||||||
|
{
|
||||||
|
name: 'Mass (kg)',
|
||||||
|
type: 'line',
|
||||||
|
yAxisIndex: 0,
|
||||||
|
smooth: true,
|
||||||
|
symbol: 'none',
|
||||||
|
lineStyle: { width: 2, opacity: 0.9 },
|
||||||
|
itemStyle: { color: c.accent },
|
||||||
|
areaStyle: {
|
||||||
|
color: {
|
||||||
|
type: 'linear',
|
||||||
|
x: 0, y: 0, x2: 0, y2: 1,
|
||||||
|
colorStops: [
|
||||||
|
{ offset: 0, color: c.isDark ? 'rgba(137, 180, 250, 0.22)' : 'rgba(137, 180, 250, 0.18)' },
|
||||||
|
{ offset: 1, color: 'rgba(137, 180, 250, 0)' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
data: massData,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Value (DKK)',
|
||||||
|
type: 'line',
|
||||||
|
yAxisIndex: 1,
|
||||||
|
smooth: true,
|
||||||
|
symbol: 'none',
|
||||||
|
lineStyle: { width: 2, opacity: 0.9 },
|
||||||
|
itemStyle: { color: c.secondary },
|
||||||
|
areaStyle: {
|
||||||
|
color: {
|
||||||
|
type: 'linear',
|
||||||
|
x: 0, y: 0, x2: 0, y2: 1,
|
||||||
|
colorStops: [
|
||||||
|
{ offset: 0, color: c.isDark ? 'rgba(148, 226, 213, 0.22)' : 'rgba(148, 226, 213, 0.18)' },
|
||||||
|
{ offset: 1, color: 'rgba(148, 226, 213, 0)' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
data: valueData,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderPriceChart(priceTrend) {
|
||||||
|
const c = getThemeColors();
|
||||||
|
|
||||||
|
chartPrice.setOption({
|
||||||
|
backgroundColor: c.bgTransparent,
|
||||||
|
textStyle: { fontFamily: 'Noto Sans', color: c.textColor },
|
||||||
|
tooltip: {
|
||||||
|
trigger: 'axis',
|
||||||
|
axisPointer: { type: 'cross' },
|
||||||
|
backgroundColor: c.tooltipBg,
|
||||||
|
borderColor: c.tooltipBorder,
|
||||||
|
borderWidth: 1,
|
||||||
|
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
|
||||||
|
formatter: (params) => {
|
||||||
|
const p = params[0];
|
||||||
|
return `<strong>${p.axisValue}</strong><br/>${formatPricePerKg(p.value)}`;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
grid: {
|
||||||
|
left: '8%',
|
||||||
|
right: '10%',
|
||||||
|
bottom: 50,
|
||||||
|
containLabel: true,
|
||||||
|
top: '18%',
|
||||||
|
},
|
||||||
|
xAxis: {
|
||||||
|
type: 'category',
|
||||||
|
data: priceTrend.map(d => formatMonthDisplay(d.month)),
|
||||||
|
axisLine: { lineStyle: { color: c.gridColor } },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
interval: priceTrend.length > 24 ? 6 : (priceTrend.length > 12 ? 3 : 1),
|
||||||
|
fontSize: 10,
|
||||||
|
rotate: 0,
|
||||||
|
margin: 8,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
yAxis: {
|
||||||
|
type: 'value',
|
||||||
|
name: 'Price per kg (DKK)',
|
||||||
|
splitLine: { lineStyle: { color: c.gridColor, type: 'dashed' } },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
formatter: (val) => formatPricePerKg(val),
|
||||||
|
margin: 8,
|
||||||
|
},
|
||||||
|
axisLine: { show: true, lineStyle: { color: c.gridColor } },
|
||||||
|
nameTextStyle: { color: c.accent, padding: [0, 0, 0, 0] },
|
||||||
|
},
|
||||||
|
series: [
|
||||||
|
{
|
||||||
|
name: 'Price per kg',
|
||||||
|
type: 'line',
|
||||||
|
smooth: true,
|
||||||
|
symbol: 'none',
|
||||||
|
lineStyle: { width: 3 },
|
||||||
|
itemStyle: { color: c.secondary },
|
||||||
|
areaStyle: {
|
||||||
|
color: {
|
||||||
|
type: 'linear',
|
||||||
|
x: 0, y: 0, x2: 0, y2: 1,
|
||||||
|
colorStops: [
|
||||||
|
{ offset: 0, color: c.isDark ? 'rgba(148, 226, 213, 0.3)' : 'rgba(148, 226, 213, 0.2)' },
|
||||||
|
{ offset: 1, color: 'rgba(148, 226, 213, 0)' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
data: priceTrend.map(d => d.price_per_kg),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderBarChart(months, breakdown, showOthers) {
|
||||||
|
const c = getThemeColors();
|
||||||
|
|
||||||
|
if (months.length === 0 || breakdown.length === 0) {
|
||||||
|
chartBar.setOption({ series: [] });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const formattedMonths = months.map(m => formatMonthDisplay(m));
|
||||||
|
const step = months.length > 36 ? 12 : (months.length > 18 ? 6 : (months.length > 12 ? 3 : 1));
|
||||||
|
|
||||||
|
const xLabels = formattedMonths.map((label, idx) =>
|
||||||
|
idx % step === 0 ? label : ''
|
||||||
|
);
|
||||||
|
|
||||||
|
const speciesMonthlyMap = {};
|
||||||
|
const speciesLabels = {};
|
||||||
|
const speciesTotals = {};
|
||||||
|
|
||||||
|
breakdown.forEach(row => {
|
||||||
|
if (!speciesMonthlyMap[row.species_code]) {
|
||||||
|
speciesMonthlyMap[row.species_code] = {};
|
||||||
|
speciesLabels[row.species_code] = row.species_label || row.species_code;
|
||||||
|
speciesTotals[row.species_code] = 0;
|
||||||
|
}
|
||||||
|
speciesMonthlyMap[row.species_code][row.month] = row.total_value || 0;
|
||||||
|
speciesTotals[row.species_code] += row.total_value || 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
const rankedSpecies = Object.entries(speciesTotals)
|
||||||
|
.sort((a, b) => b[1] - a[1])
|
||||||
|
.slice(0, 5)
|
||||||
|
.map(([code]) => code);
|
||||||
|
|
||||||
|
const otherValues = months.map(m => {
|
||||||
|
return Object.keys(speciesMonthlyMap)
|
||||||
|
.filter(code => !rankedSpecies.includes(code))
|
||||||
|
.reduce((sum, code) => sum + (speciesMonthlyMap[code][m] || 0), 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
let seriesData = rankedSpecies.map(code => ({
|
||||||
|
name: speciesLabels[code],
|
||||||
|
type: 'bar',
|
||||||
|
stack: 'total',
|
||||||
|
emphasis: { focus: 'series' },
|
||||||
|
data: months.map(m => speciesMonthlyMap[code][m] || 0),
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (showOthers) {
|
||||||
|
seriesData.push({
|
||||||
|
name: 'Other species',
|
||||||
|
type: 'bar',
|
||||||
|
stack: 'total',
|
||||||
|
emphasis: { focus: 'series' },
|
||||||
|
data: otherValues,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const chartWidth = chartBar.getWidth();
|
||||||
|
const barMaxWidth = Math.max(15, Math.floor((chartWidth * 0.7) / (months.length * 1.5)));
|
||||||
|
|
||||||
|
chartBar.setOption({
|
||||||
|
backgroundColor: c.bgTransparent,
|
||||||
|
textStyle: { fontFamily: 'Noto Sans', color: c.textColor },
|
||||||
|
tooltip: {
|
||||||
|
trigger: 'axis',
|
||||||
|
axisPointer: { type: 'shadow' },
|
||||||
|
backgroundColor: c.tooltipBg,
|
||||||
|
borderColor: c.tooltipBorder,
|
||||||
|
borderWidth: 1,
|
||||||
|
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
|
||||||
|
formatter: (params) => {
|
||||||
|
const month = params[0]?.axisValue || '';
|
||||||
|
let html = `<strong>${month}</strong><br/>`;
|
||||||
|
[...params].reverse().forEach(p => {
|
||||||
|
const val = p.value || 0;
|
||||||
|
if (val > 0) {
|
||||||
|
html += `${p.marker} ${p.seriesName}: ${formatDKK(val)}<br/>`;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return html;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
legend: {
|
||||||
|
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
|
||||||
|
top: 0,
|
||||||
|
left: 'center',
|
||||||
|
orient: 'horizontal',
|
||||||
|
type: 'scroll',
|
||||||
|
maxItems: 8,
|
||||||
|
padding: [0, 0, 0, 0],
|
||||||
|
itemGap: 10,
|
||||||
|
},
|
||||||
|
grid: {
|
||||||
|
left: '8%',
|
||||||
|
right: '5%',
|
||||||
|
bottom: 65,
|
||||||
|
containLabel: true,
|
||||||
|
top: '25%',
|
||||||
|
},
|
||||||
|
xAxis: {
|
||||||
|
type: 'category',
|
||||||
|
data: xLabels,
|
||||||
|
axisLine: { lineStyle: { color: c.gridColor } },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
interval: 0,
|
||||||
|
fontSize: 9,
|
||||||
|
rotate: 0,
|
||||||
|
margin: 8,
|
||||||
|
overflow: 'break',
|
||||||
|
width: 60,
|
||||||
|
},
|
||||||
|
axisTick: { show: false },
|
||||||
|
},
|
||||||
|
yAxis: {
|
||||||
|
type: 'value',
|
||||||
|
splitLine: { lineStyle: { color: c.gridColor, type: 'dashed' } },
|
||||||
|
axisLabel: {
|
||||||
|
color: c.subTextColor,
|
||||||
|
formatter: (val) => formatDKKShort(val),
|
||||||
|
margin: 8,
|
||||||
|
},
|
||||||
|
nameTextStyle: { padding: [0, 0, 0, 10] },
|
||||||
|
},
|
||||||
|
series: seriesData.map((s, idx) => ({
|
||||||
|
...s,
|
||||||
|
barMaxWidth: barMaxWidth,
|
||||||
|
itemStyle: {
|
||||||
|
color: c.palette[idx % c.palette.length],
|
||||||
|
},
|
||||||
|
label: {
|
||||||
|
show: false,
|
||||||
|
},
|
||||||
|
emphasis: {
|
||||||
|
itemStyle: {
|
||||||
|
shadowBlur: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchData(endpoint) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(endpoint);
|
||||||
|
if (!res.ok) throw new Error(`API Error: ${res.status}`);
|
||||||
|
return await res.json();
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function populateDropdown(containerId, items, codeKey, labelKey) {
|
||||||
|
const container = document.getElementById(containerId);
|
||||||
|
if (!container) return;
|
||||||
|
|
||||||
|
const checkboxList = container.querySelector('.checkbox-dropdown-list');
|
||||||
|
checkboxList.innerHTML = '';
|
||||||
|
|
||||||
|
const selectAllLabel = document.createElement('label');
|
||||||
|
selectAllLabel.className = 'checkbox-item select-all-wrapper';
|
||||||
|
selectAllLabel.innerHTML = `
|
||||||
|
<input type="checkbox" class="select-all" data-target="${containerId}" checked>
|
||||||
|
<span>All</span>
|
||||||
|
`;
|
||||||
|
checkboxList.appendChild(selectAllLabel);
|
||||||
|
|
||||||
|
const separator = document.createElement('div');
|
||||||
|
separator.className = 'dropdown-separator';
|
||||||
|
checkboxList.appendChild(separator);
|
||||||
|
|
||||||
|
items.forEach((item) => {
|
||||||
|
const label = document.createElement('label');
|
||||||
|
label.className = 'checkbox-item';
|
||||||
|
label.innerHTML = `
|
||||||
|
<input type="checkbox"
|
||||||
|
name="${containerId}"
|
||||||
|
value="${item[codeKey]}"
|
||||||
|
data-label="${item[labelKey]}"
|
||||||
|
checked>
|
||||||
|
<span>${item[labelKey]}</span>
|
||||||
|
`;
|
||||||
|
checkboxList.appendChild(label);
|
||||||
|
});
|
||||||
|
|
||||||
|
const selectAllCb = selectAllLabel.querySelector('.select-all');
|
||||||
|
selectAllCb.addEventListener('change', (e) => {
|
||||||
|
const cbs = checkboxList.querySelectorAll(`input[name="${containerId}"]`);
|
||||||
|
cbs.forEach(cb => {
|
||||||
|
if (!cb.disabled) {
|
||||||
|
cb.checked = e.target.checked;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
updateDropdownLabel(containerId);
|
||||||
|
});
|
||||||
|
|
||||||
|
checkboxList.querySelectorAll(`input[name="${containerId}"]`).forEach(cb => {
|
||||||
|
cb.addEventListener('change', () => updateDropdownLabel(containerId));
|
||||||
|
});
|
||||||
|
|
||||||
|
updateDropdownLabel(containerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateDropdownLabel(containerId) {
|
||||||
|
const container = document.getElementById(containerId);
|
||||||
|
if (!container) return;
|
||||||
|
|
||||||
|
const cbs = container.querySelectorAll(`input[name="${containerId}"]`);
|
||||||
|
const checked = Array.from(cbs).filter(cb => cb.checked && !cb.disabled);
|
||||||
|
const labelEl = container.querySelector('.dropdown-label');
|
||||||
|
|
||||||
|
if (checked.length === 0) {
|
||||||
|
labelEl.textContent = 'None selected';
|
||||||
|
} else if (checked.length === cbs.length) {
|
||||||
|
labelEl.textContent = 'All selected';
|
||||||
|
} else {
|
||||||
|
labelEl.textContent = `${checked.length} selected`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function toggleDropdown(containerId) {
|
||||||
|
const container = document.getElementById(containerId);
|
||||||
|
if (!container) return;
|
||||||
|
|
||||||
|
const list = container.querySelector('.checkbox-dropdown-list');
|
||||||
|
const isOpen = list.classList.contains('open');
|
||||||
|
|
||||||
|
document.querySelectorAll('.checkbox-dropdown-list.open').forEach(el => {
|
||||||
|
el.classList.remove('open');
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!isOpen) {
|
||||||
|
list.classList.add('open');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSelectedValues(groupName) {
|
||||||
|
const checkboxes = document.querySelectorAll(`input[name="${groupName}"]:checked`);
|
||||||
|
return Array.from(checkboxes).filter(cb => !cb.disabled).map(cb => cb.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function populateMonthSelect(selectId, months) {
|
||||||
|
const sel = document.getElementById(selectId);
|
||||||
|
sel.innerHTML = '<option value="">All</option>';
|
||||||
|
months.forEach((m) => {
|
||||||
|
const opt = document.createElement('option');
|
||||||
|
opt.value = m;
|
||||||
|
opt.textContent = formatMonthDisplay(m);
|
||||||
|
sel.appendChild(opt);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function populateDropdowns() {
|
||||||
|
showLoading(true);
|
||||||
|
const [speciesList, zonesList, gearList, summaryData] = await Promise.all([
|
||||||
|
fetchData('/api/species'),
|
||||||
|
fetchData('/api/zones'),
|
||||||
|
fetchData('/api/gear'),
|
||||||
|
fetchData('/api/summary'),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (speciesList) {
|
||||||
|
populateDropdown('species', speciesList, 'code', 'label');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (zonesList) {
|
||||||
|
populateDropdown('zone', zonesList, 'code', 'label');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (gearList) {
|
||||||
|
populateDropdown('gear', gearList, 'code', 'label');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (summaryData && summaryData.monthly) {
|
||||||
|
allMonths = summaryData.monthly.map((m) => m.month).sort();
|
||||||
|
|
||||||
|
populateMonthSelect('month_from', allMonths);
|
||||||
|
populateMonthSelect('month_to', allMonths);
|
||||||
|
|
||||||
|
const fromIdx = Math.max(0, allMonths.length - MONTHS_TO_SHOW_DEFAULT);
|
||||||
|
document.getElementById('month_from').value = allMonths[fromIdx];
|
||||||
|
document.getElementById('month_to').value = allMonths[allMonths.length - 1];
|
||||||
|
}
|
||||||
|
|
||||||
|
await updateAvailableFilters();
|
||||||
|
|
||||||
|
showLoading(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updateAvailableFilters() {
|
||||||
|
const species = getSelectedValues('species');
|
||||||
|
const zone = getSelectedValues('zone');
|
||||||
|
const gear = getSelectedValues('gear');
|
||||||
|
const monthFrom = document.getElementById('month_from').value;
|
||||||
|
const monthTo = document.getElementById('month_to').value;
|
||||||
|
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
if (species.length > 0) params.append('species_multi', species.join(','));
|
||||||
|
if (zone.length > 0) params.append('zone_multi', zone.join(','));
|
||||||
|
if (gear.length > 0) params.append('gear_multi', gear.join(','));
|
||||||
|
if (monthFrom) params.append('month_from', monthFrom);
|
||||||
|
if (monthTo) params.append('month_to', monthTo);
|
||||||
|
|
||||||
|
const available = await fetchData(`/api/available-filters?${params.toString()}`);
|
||||||
|
|
||||||
|
if (available) {
|
||||||
|
disableUnavailableOptions('species', available.species);
|
||||||
|
disableUnavailableOptions('zone', available.zones);
|
||||||
|
disableUnavailableOptions('gear', available.gear);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function disableUnavailableOptions(containerId, availableCodes) {
|
||||||
|
const container = document.getElementById(containerId);
|
||||||
|
if (!container) return;
|
||||||
|
|
||||||
|
const checkboxes = container.querySelectorAll(`input[type="checkbox"]`);
|
||||||
|
checkboxes.forEach(cb => {
|
||||||
|
if (cb.classList.contains('select-all')) return;
|
||||||
|
|
||||||
|
const isAvailable = availableCodes.includes(cb.value);
|
||||||
|
cb.disabled = !isAvailable;
|
||||||
|
cb.parentElement.style.opacity = isAvailable ? '1' : '0.4';
|
||||||
|
cb.parentElement.style.cursor = isAvailable ? 'pointer' : 'not-allowed';
|
||||||
|
|
||||||
|
if (!isAvailable && cb.checked) {
|
||||||
|
cb.checked = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
updateDropdownLabel(containerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updateCharts() {
|
||||||
|
showLoading(true);
|
||||||
|
hideEmptyState();
|
||||||
|
|
||||||
|
const species = getSelectedValues('species');
|
||||||
|
const zone = getSelectedValues('zone');
|
||||||
|
const gear = getSelectedValues('gear');
|
||||||
|
const monthFrom = document.getElementById('month_from').value;
|
||||||
|
const monthTo = document.getElementById('month_to').value;
|
||||||
|
|
||||||
|
if (monthFrom && monthTo && monthFrom > monthTo) {
|
||||||
|
alert('From date cannot be greater than To date.');
|
||||||
|
showLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const summaryParams = new URLSearchParams();
|
||||||
|
if (species.length > 0) summaryParams.append('species_multi', species.join(','));
|
||||||
|
if (zone.length > 0) summaryParams.append('zone_multi', zone.join(','));
|
||||||
|
if (gear.length > 0) summaryParams.append('gear_multi', gear.join(','));
|
||||||
|
if (monthFrom) summaryParams.append('month_from', monthFrom);
|
||||||
|
if (monthTo) summaryParams.append('month_to', monthTo);
|
||||||
|
|
||||||
|
const [summary, monthlyBreakdown] = await Promise.all([
|
||||||
|
fetchData(`/api/summary?${summaryParams.toString()}`),
|
||||||
|
fetchData(`/api/summary/monthly-breakdown?${summaryParams.toString()}`),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!summary || !summary.monthly || summary.monthly.length === 0) {
|
||||||
|
showLoading(false);
|
||||||
|
showEmptyState();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const months = summary.monthly.map(m => m.month).sort();
|
||||||
|
|
||||||
|
const showAllSpecies = document.getElementById('show-all-species')?.checked ?? false;
|
||||||
|
|
||||||
|
if (monthlyBreakdown && monthlyBreakdown.length > 0) {
|
||||||
|
renderBarChart(months, monthlyBreakdown, showAllSpecies);
|
||||||
|
} else {
|
||||||
|
renderBarChart([], [], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
const massData = summary.monthly.map((m) => m.total_mass || 0);
|
||||||
|
const valueData = summary.monthly.map((m) => m.total_value || 0);
|
||||||
|
renderLineChart(months, massData, valueData);
|
||||||
|
|
||||||
|
const priceTrend = summary.price_trend ?? [];
|
||||||
|
if (priceTrend.length > 0) {
|
||||||
|
renderPriceChart(priceTrend);
|
||||||
|
} else {
|
||||||
|
chartPrice.setOption({ series: [] });
|
||||||
|
}
|
||||||
|
|
||||||
|
showLoading(false);
|
||||||
|
hideEmptyState();
|
||||||
|
document.getElementById('charts-grid').style.display = 'grid';
|
||||||
|
|
||||||
|
chartLine.resize();
|
||||||
|
chartPrice.resize();
|
||||||
|
chartBar.resize();
|
||||||
|
|
||||||
|
renderLineChart(months, massData, valueData);
|
||||||
|
|
||||||
|
if (priceTrend.length > 0) {
|
||||||
|
renderPriceChart(priceTrend);
|
||||||
|
} else {
|
||||||
|
chartPrice.setOption({ series: [] });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (monthlyBreakdown && monthlyBreakdown.length > 0) {
|
||||||
|
renderBarChart(months, monthlyBreakdown, showAllSpecies);
|
||||||
|
} else {
|
||||||
|
renderBarChart([], [], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
chartLine.resize();
|
||||||
|
chartPrice.resize();
|
||||||
|
chartBar.resize();
|
||||||
|
}, 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
function showLoading(show) {
|
||||||
|
const overlay = document.getElementById('loading-overlay');
|
||||||
|
if (show) {
|
||||||
|
overlay.classList.remove('hidden');
|
||||||
|
} else {
|
||||||
|
overlay.classList.add('hidden');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function showEmptyState() {
|
||||||
|
document.getElementById('charts-grid').style.display = 'none';
|
||||||
|
document.getElementById('empty-state').style.display = 'flex';
|
||||||
|
}
|
||||||
|
|
||||||
|
function hideEmptyState() {
|
||||||
|
document.getElementById('charts-grid').style.display = 'grid';
|
||||||
|
document.getElementById('empty-state').style.display = 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
document.getElementById('apply-filters').addEventListener('click', updateCharts);
|
||||||
|
|
||||||
|
document.getElementById('show-all-species')?.addEventListener('change', updateCharts);
|
||||||
|
|
||||||
|
document.addEventListener('click', (e) => {
|
||||||
|
if (!e.target.closest('.filter-group.multi-select')) {
|
||||||
|
document.querySelectorAll('.checkbox-dropdown-list.open').forEach(el => {
|
||||||
|
el.classList.remove('open');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(() => {
|
||||||
|
function currentSystemTheme() {
|
||||||
|
return window.matchMedia('(prefers-color-scheme: dark)').matches
|
||||||
|
? 'macchiato'
|
||||||
|
: 'light';
|
||||||
|
}
|
||||||
|
const btn = document.getElementById('theme-toggle');
|
||||||
|
if (btn) {
|
||||||
|
btn.addEventListener('click', () => {
|
||||||
|
const current =
|
||||||
|
document.documentElement.getAttribute('data-theme') || 'light';
|
||||||
|
const next = current === 'light' ? 'macchiato' : 'light';
|
||||||
|
document.documentElement.setAttribute('data-theme', next);
|
||||||
|
localStorage.setItem('poc-theme', next);
|
||||||
|
|
||||||
|
initCharts();
|
||||||
|
updateCharts();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
window
|
||||||
|
.matchMedia('(prefers-color-scheme: dark)')
|
||||||
|
.addEventListener('change', () => {
|
||||||
|
if (!localStorage.getItem('poc-theme')) {
|
||||||
|
document.documentElement.setAttribute('data-theme', currentSystemTheme());
|
||||||
|
initCharts();
|
||||||
|
updateCharts();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', () => {
|
||||||
|
initCharts();
|
||||||
|
updateNavbarUpdateDate();
|
||||||
|
|
||||||
|
['species', 'zone', 'gear'].forEach(id => {
|
||||||
|
const container = document.getElementById(id);
|
||||||
|
if (container) {
|
||||||
|
const label = container.querySelector('.dropdown-label');
|
||||||
|
if (label) {
|
||||||
|
label.addEventListener('click', (e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
toggleDropdown(id);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const list = container.querySelector('.checkbox-dropdown-list');
|
||||||
|
if (list) {
|
||||||
|
list.addEventListener('change', () => {
|
||||||
|
updateAvailableFilters();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
document.getElementById('month_from').addEventListener('change', updateAvailableFilters);
|
||||||
|
document.getElementById('month_to').addEventListener('change', updateAvailableFilters);
|
||||||
|
|
||||||
|
populateDropdowns().then(updateCharts);
|
||||||
|
});
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, viewport-fit=cover" />
|
||||||
|
<title>Fisheries Dashboard — FLÓ</title>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
(function () {
|
||||||
|
var stored = localStorage.getItem('poc-theme');
|
||||||
|
var theme = stored || (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'macchiato' : 'light');
|
||||||
|
document.documentElement.setAttribute('data-theme', theme);
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||||
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||||
|
<link href="https://fonts.googleapis.com/css2?family=Noto+Sans:wght@300;400;500;700&display=swap" rel="stylesheet">
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/echarts@5.4.3/dist/echarts.min.js"></script>
|
||||||
|
|
||||||
|
<link rel="stylesheet" href="/static/style.css">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<nav>
|
||||||
|
<a href="https://fló.fo" class="back-link">← FLÓ.FO</a>
|
||||||
|
<a href="/" class="nav-brand">
|
||||||
|
FLÓ <span class="brand-sep">/</span> <span class="brand-sub">HAGFISH</span>
|
||||||
|
</a>
|
||||||
|
<div class="update-date-display" id="update-date-display"></div>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<main class="page-body">
|
||||||
|
<div class="sidebar">
|
||||||
|
<div class="filters-panel">
|
||||||
|
<div class="filters">
|
||||||
|
<div class="filter-group multi-select" id="species">
|
||||||
|
<label>Species</label>
|
||||||
|
<div class="dropdown-trigger dropdown-label">All</div>
|
||||||
|
<div class="checkbox-dropdown-list"></div>
|
||||||
|
</div>
|
||||||
|
<div class="filter-group multi-select" id="zone">
|
||||||
|
<label>Zone</label>
|
||||||
|
<div class="dropdown-trigger dropdown-label">All</div>
|
||||||
|
<div class="checkbox-dropdown-list"></div>
|
||||||
|
</div>
|
||||||
|
<div class="filter-group multi-select" id="gear">
|
||||||
|
<label>Gear</label>
|
||||||
|
<div class="dropdown-trigger dropdown-label">All</div>
|
||||||
|
<div class="checkbox-dropdown-list"></div>
|
||||||
|
</div>
|
||||||
|
<div class="filter-group">
|
||||||
|
<label for="month_from">From</label>
|
||||||
|
<select id="month_from"><option value="">All</option></select>
|
||||||
|
</div>
|
||||||
|
<div class="filter-group">
|
||||||
|
<label for="month_to">To</label>
|
||||||
|
<select id="month_to"><option value="">All</option></select>
|
||||||
|
</div>
|
||||||
|
<div class="filter-group">
|
||||||
|
<label class="toggle-label">
|
||||||
|
<input type="checkbox" id="show-all-species">
|
||||||
|
<span>Show "Other species" bucket</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<button class="apply-btn" id="apply-filters">Apply</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="dashboard-wrapper">
|
||||||
|
<div id="loading-overlay" class="loading-overlay">
|
||||||
|
<span>Loading fisheries data…</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="dashboard">
|
||||||
|
<div id="empty-state" class="empty-state" style="display:none;">
|
||||||
|
<p>No data available for the selected filters.</p>
|
||||||
|
<p>Try adjusting your filters or running an ingestion.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="charts-grid" class="charts-grid" style="display:none;">
|
||||||
|
<div class="chart-container full-width">
|
||||||
|
<div class="chart-title">Total Landings (Mass & Value)</div>
|
||||||
|
<div id="chart-line" style="width:100%; height:420px;"></div>
|
||||||
|
</div>
|
||||||
|
<div class="chart-container">
|
||||||
|
<div class="chart-title">Price per kg Trend</div>
|
||||||
|
<div id="chart-price" style="width:100%; height:380px;"></div>
|
||||||
|
</div>
|
||||||
|
<div class="chart-container">
|
||||||
|
<div class="chart-title">Monthly Value by Top Species</div>
|
||||||
|
<div id="chart-bar" style="width:100%; height:380px;"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<button class="theme-toggle" id="theme-toggle" aria-label="Toggle theme"></button>
|
||||||
|
|
||||||
|
<footer>
|
||||||
|
<span>© <script>document.write(new Date().getFullYear())</script> FLÓ.FO | ALL RIGHTS RESERVED</span>
|
||||||
|
<a href="https://www.facebook.com/people/FL%C3%93/61593006613799/" target="_blank" rel="noopener" class="footer-icon" aria-label="Facebook">
|
||||||
|
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor"><path d="M24 12.073c0-6.627-5.373-12-12-12s-12 5.373-12 12c0 5.99 4.388 10.954 10.125 11.854v-8.385H7.078v-3.47h3.047V9.43c0-3.007 1.792-4.669 4.533-4.669 1.312 0 2.686.235 2.686.235v2.953H15.83c-1.491 0-1.956.925-1.956 1.874v2.25h3.328l-.532 3.47h-2.796v8.385C19.612 23.027 24 18.062 24 12.073z"/></svg>
|
||||||
|
</a>
|
||||||
|
<a href="https://git.flo.fo/FLO" target="_blank" rel="noopener" class="footer-icon" aria-label="Gitea">
|
||||||
|
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor"><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg>
|
||||||
|
</a>
|
||||||
|
<a href="https://www.linkedin.com/in/bartal-l%C3%A6arsson-63793895/" target="_blank" rel="noopener" class="footer-icon" aria-label="LinkedIn">
|
||||||
|
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor"><path d="M20.447 20.452h-3.554v-5.569c0-1.328-.027-3.037-1.852-3.037-1.853 0-2.136 1.445-2.136 2.939v5.667H9.351V9h3.414v1.561h.046c.477-.9 1.637-1.85 3.37-1.85 3.601 0 4.267 2.37 4.267 5.455v6.286zM5.337 7.433a2.062 2.062 0 01-2.063-2.065 2.064 2.064 0 112.063 2.065zm1.782 13.019H3.555V9h3.564v11.452zM22.225 0H1.771C.792 0 0 .774 0 1.729v20.542C0 23.227.792 24 1.771 24h20.451C23.2 24 24 23.227 24 22.271V1.729C24 .774 23.2 0 22.222 0h.003z"/></svg>
|
||||||
|
</a>
|
||||||
|
</footer>
|
||||||
|
|
||||||
|
<script src="/static/app.js"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,528 @@
|
|||||||
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||||
|
|
||||||
|
:root,
|
||||||
|
[data-theme="light"] {
|
||||||
|
--bg: #f4f1ec;
|
||||||
|
--bg-panel: #fafaf5;
|
||||||
|
--nav-bg: rgba(255, 253, 250, 0.92);
|
||||||
|
--ink: #1a1d22;
|
||||||
|
--ink-muted: #595f68;
|
||||||
|
--ink-dim: #9299a2;
|
||||||
|
--accent: #2c5a7a;
|
||||||
|
--border: rgba(26, 29, 34, 0.1);
|
||||||
|
--danger: #a03030;
|
||||||
|
--input-bg: #f0ede7;
|
||||||
|
--surface-hover: rgba(26, 29, 34, 0.04);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="macchiato"] {
|
||||||
|
--bg: #181926;
|
||||||
|
--bg-panel: #1e2030;
|
||||||
|
--nav-bg: rgba(36, 39, 58, 0.65);
|
||||||
|
--ink: #cad3f5;
|
||||||
|
--ink-muted: #a5adcb;
|
||||||
|
--ink-dim: #8087a2;
|
||||||
|
--accent: #8aadf4;
|
||||||
|
--border: rgba(165, 173, 203, 0.15);
|
||||||
|
--danger: #ed8796;
|
||||||
|
--input-bg: #1e2030;
|
||||||
|
--surface-hover: rgba(138, 173, 244, 0.12);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="macchiato"] .nav-brand {
|
||||||
|
color: #cad3f5;
|
||||||
|
text-shadow: 0 0 8px rgba(138, 173, 244, 0.25);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="macchiato"] .apply-btn {
|
||||||
|
box-shadow: 0 0 10px rgba(138, 173, 244, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="macchiato"] .theme-toggle {
|
||||||
|
background: #24273a;
|
||||||
|
border-color: rgba(138, 173, 244, 0.4);
|
||||||
|
box-shadow:
|
||||||
|
0 0 10px rgba(138, 173, 244, 0.4),
|
||||||
|
0 0 25px rgba(138, 173, 244, 0.2);
|
||||||
|
animation: neon-breathe-toggle 4s ease-in-out infinite;
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes neon-breathe-toggle {
|
||||||
|
0%, 100% {
|
||||||
|
box-shadow:
|
||||||
|
0 0 10px rgba(138, 173, 244, 0.5),
|
||||||
|
0 0 25px rgba(138, 173, 244, 0.3),
|
||||||
|
0 0 50px rgba(116, 150, 230, 0.2);
|
||||||
|
}
|
||||||
|
50% {
|
||||||
|
box-shadow:
|
||||||
|
0 0 15px rgba(138, 173, 244, 0.6),
|
||||||
|
0 0 35px rgba(138, 173, 244, 0.4),
|
||||||
|
0 0 70px rgba(116, 150, 230, 0.25);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
html { scroll-behavior: smooth; }
|
||||||
|
|
||||||
|
body {
|
||||||
|
min-height: 100vh;
|
||||||
|
min-height: 100dvh;
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--ink);
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-size: 0.92rem;
|
||||||
|
font-weight: 300;
|
||||||
|
line-height: 1.65;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
transition: background 0.3s ease, color 0.3s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
a {
|
||||||
|
color: var(--accent);
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 1px solid transparent;
|
||||||
|
transition: all 0.2s ease;
|
||||||
|
}
|
||||||
|
a:hover { border-bottom-color: var(--accent); }
|
||||||
|
|
||||||
|
/* ---- Nav ---- */
|
||||||
|
nav {
|
||||||
|
position: fixed; top: 0; left: 0;
|
||||||
|
width: 100%; z-index: 20;
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr auto 1fr;
|
||||||
|
align-items: center;
|
||||||
|
gap: 1rem;
|
||||||
|
padding: 1.25rem 2.5rem;
|
||||||
|
background: var(--nav-bg);
|
||||||
|
backdrop-filter: blur(12px) saturate(120%);
|
||||||
|
-webkit-backdrop-filter: blur(12px) saturate(120%);
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
transition: background 0.3s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.back-link {
|
||||||
|
grid-column: 1;
|
||||||
|
justify-self: start;
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 0.7rem;
|
||||||
|
letter-spacing: 0.15em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
white-space: nowrap;
|
||||||
|
color: var(--ink-muted);
|
||||||
|
text-decoration: none;
|
||||||
|
padding: 0.35rem 0.6rem;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 2px;
|
||||||
|
transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease;
|
||||||
|
}
|
||||||
|
.back-link:hover {
|
||||||
|
color: var(--ink);
|
||||||
|
border-color: var(--ink-muted);
|
||||||
|
background: var(--bg-panel);
|
||||||
|
border-bottom-color: var(--ink-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nav-brand {
|
||||||
|
grid-column: 2;
|
||||||
|
justify-self: center;
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: baseline;
|
||||||
|
gap: 0.35rem;
|
||||||
|
white-space: nowrap;
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-weight: 700;
|
||||||
|
font-size: 1.1rem;
|
||||||
|
letter-spacing: -0.02em;
|
||||||
|
color: var(--ink);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.nav-brand:hover { color: var(--accent); border-bottom: none; }
|
||||||
|
|
||||||
|
.update-date-display {
|
||||||
|
grid-column: 3;
|
||||||
|
justify-self: end;
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
letter-spacing: 0.08em;
|
||||||
|
color: var(--ink-muted);
|
||||||
|
white-space: nowrap;
|
||||||
|
padding: 0.35rem 0.8rem;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 2px;
|
||||||
|
background: var(--input-bg);
|
||||||
|
}
|
||||||
|
|
||||||
|
.brand-sep { font-weight: 300; color: var(--ink-dim); }
|
||||||
|
.brand-sub {
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 0.78rem;
|
||||||
|
letter-spacing: 0.18em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
color: var(--ink-muted);
|
||||||
|
align-self: center;
|
||||||
|
padding-top: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Main ---- */
|
||||||
|
.page-body {
|
||||||
|
flex: 1;
|
||||||
|
display: flex;
|
||||||
|
padding: calc(65px + 2.5rem) 2rem 2rem;
|
||||||
|
gap: 2rem;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.sidebar {
|
||||||
|
width: 320px;
|
||||||
|
flex-shrink: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.filters-panel {
|
||||||
|
background: var(--bg-panel);
|
||||||
|
backdrop-filter: blur(14px);
|
||||||
|
-webkit-backdrop-filter: blur(14px);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 3px;
|
||||||
|
padding: 1.5rem;
|
||||||
|
box-shadow: 0 8px 30px rgba(26, 29, 34, 0.06);
|
||||||
|
animation: panel-in 0.7s cubic-bezier(0.16, 1, 0.3, 1) forwards;
|
||||||
|
opacity: 0;
|
||||||
|
transform: translateX(-10px);
|
||||||
|
transition: background 0.3s ease, border-color 0.3s ease;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes panel-in {
|
||||||
|
0% { opacity: 0; transform: translateX(-10px); }
|
||||||
|
100% { opacity: 1; transform: translateX(0); }
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-wrapper {
|
||||||
|
flex: 1;
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard {
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
background: var(--bg-panel);
|
||||||
|
backdrop-filter: blur(14px);
|
||||||
|
-webkit-backdrop-filter: blur(14px);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 3px;
|
||||||
|
padding: 2rem;
|
||||||
|
box-shadow: 0 8px 30px rgba(26, 29, 34, 0.06);
|
||||||
|
animation: panel-in 0.7s cubic-bezier(0.16, 1, 0.3, 1) forwards;
|
||||||
|
opacity: 0;
|
||||||
|
transform: translateY(10px);
|
||||||
|
transition: background 0.3s ease, border-color 0.3s ease;
|
||||||
|
position: relative;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Loading Overlay ---- */
|
||||||
|
.loading-overlay {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
background: var(--bg-panel);
|
||||||
|
backdrop-filter: blur(8px);
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
z-index: 10;
|
||||||
|
border-radius: 3px;
|
||||||
|
color: var(--ink-muted);
|
||||||
|
font-size: 0.88rem;
|
||||||
|
letter-spacing: 0.05em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.loading-overlay.hidden {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Empty State ---- */
|
||||||
|
.empty-state {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 0.5rem;
|
||||||
|
padding: 3rem 1rem;
|
||||||
|
color: var(--ink-dim);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
text-align: center;
|
||||||
|
line-height: 1.6;
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Filters ---- */
|
||||||
|
.filters {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 1rem;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.filter-group {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.4rem;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
.filter-group label {
|
||||||
|
font-size: 0.65rem;
|
||||||
|
font-weight: 500;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.15em;
|
||||||
|
color: var(--ink-dim);
|
||||||
|
}
|
||||||
|
|
||||||
|
.filter-group select {
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
background: var(--input-bg);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 2px;
|
||||||
|
color: var(--ink);
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-size: 0.82rem;
|
||||||
|
outline: none;
|
||||||
|
transition: border-color 0.2s;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.filter-group select:focus { border-color: var(--accent); }
|
||||||
|
|
||||||
|
/* ---- Multi-Select Dropdown ---- */
|
||||||
|
.filter-group.multi-select {
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dropdown-trigger {
|
||||||
|
width: 100%;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
background: var(--input-bg);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 2px;
|
||||||
|
color: var(--ink);
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-size: 0.82rem;
|
||||||
|
cursor: pointer;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
transition: border-color 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dropdown-trigger:hover {
|
||||||
|
border-color: var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dropdown-trigger::after {
|
||||||
|
content: '▼';
|
||||||
|
font-size: 0.6rem;
|
||||||
|
margin-left: 0.5rem;
|
||||||
|
opacity: 0.6;
|
||||||
|
}
|
||||||
|
|
||||||
|
.checkbox-dropdown-list {
|
||||||
|
position: absolute;
|
||||||
|
top: 100%;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
|
max-height: 200px;
|
||||||
|
overflow-y: auto;
|
||||||
|
background: var(--bg-panel);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-top: none;
|
||||||
|
border-radius: 0 0 2px 2px;
|
||||||
|
z-index: 100;
|
||||||
|
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
||||||
|
display: none;
|
||||||
|
padding: 0.5rem 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.checkbox-dropdown-list.open {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dropdown-separator {
|
||||||
|
border-top: 1px solid var(--border);
|
||||||
|
margin: 0.25rem 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.checkbox-item {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.5rem;
|
||||||
|
padding: 0.25rem 0.75rem;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: background 0.1s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.checkbox-item:hover {
|
||||||
|
background: var(--surface-hover);
|
||||||
|
}
|
||||||
|
|
||||||
|
.checkbox-item input[type="checkbox"] {
|
||||||
|
cursor: pointer;
|
||||||
|
accent-color: var(--accent);
|
||||||
|
margin-right: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.select-all-wrapper {
|
||||||
|
font-weight: 600;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
margin-bottom: 0.25rem;
|
||||||
|
padding-bottom: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.apply-btn {
|
||||||
|
width: 100%;
|
||||||
|
padding: 0.5rem 2rem;
|
||||||
|
background: var(--accent);
|
||||||
|
color: var(--bg-panel);
|
||||||
|
border: none;
|
||||||
|
border-radius: 2px;
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-weight: 700;
|
||||||
|
font-size: 0.82rem;
|
||||||
|
letter-spacing: 0.08em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: opacity 0.2s, box-shadow 0.2s;
|
||||||
|
}
|
||||||
|
.apply-btn:hover {
|
||||||
|
opacity: 0.85;
|
||||||
|
box-shadow: 0 0 8px rgba(138, 173, 244, 0.4);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Charts ---- */
|
||||||
|
.charts-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, 1fr);
|
||||||
|
gap: 1.5rem;
|
||||||
|
flex: 1;
|
||||||
|
min-height: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chart-container {
|
||||||
|
background: var(--input-bg);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 2px;
|
||||||
|
padding: 1rem;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chart-container.full-width {
|
||||||
|
grid-column: span 2;
|
||||||
|
height: 450px;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chart-container:nth-child(2),
|
||||||
|
.chart-container:nth-child(3) {
|
||||||
|
flex: 1;
|
||||||
|
min-height: 350px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chart-title {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
font-weight: 600;
|
||||||
|
margin-bottom: 0.8rem;
|
||||||
|
color: var(--ink);
|
||||||
|
letter-spacing: 0.02em;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Theme Toggle ---- */
|
||||||
|
.theme-toggle {
|
||||||
|
position: fixed;
|
||||||
|
bottom: 1.5rem;
|
||||||
|
right: 1.5rem;
|
||||||
|
width: 28px;
|
||||||
|
height: 28px;
|
||||||
|
border-radius: 50%;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
cursor: pointer;
|
||||||
|
padding: 0;
|
||||||
|
z-index: 30;
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.15);
|
||||||
|
}
|
||||||
|
.theme-toggle:hover { transform: scale(1.08); }
|
||||||
|
[data-theme="light"] .theme-toggle {
|
||||||
|
background: linear-gradient(135deg, #24273a 0%, #8aadf4 100%);
|
||||||
|
border-color: rgba(26, 29, 34, 0.2);
|
||||||
|
}
|
||||||
|
|
||||||
|
footer {
|
||||||
|
position: relative;
|
||||||
|
z-index: 10;
|
||||||
|
padding: 0.9rem 2rem;
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
align-items: center;
|
||||||
|
gap: 1.5rem;
|
||||||
|
font-family: 'Noto Sans', sans-serif;
|
||||||
|
font-weight: 300;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
letter-spacing: 0.05em;
|
||||||
|
color: var(--ink-muted);
|
||||||
|
border-top: 1px solid var(--border);
|
||||||
|
background: var(--nav-bg);
|
||||||
|
backdrop-filter: blur(12px);
|
||||||
|
-webkit-backdrop-filter: blur(12px);
|
||||||
|
transition: background 0.3s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Responsive ---- */
|
||||||
|
@media (max-width: 820px) {
|
||||||
|
.sidebar {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
.page-body {
|
||||||
|
padding: calc(55px + 1.25rem) 1rem 3rem;
|
||||||
|
}
|
||||||
|
.dashboard {
|
||||||
|
padding: 1.5rem;
|
||||||
|
}
|
||||||
|
.update-date-display {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 460px) {
|
||||||
|
.nav-brand .brand-sub { display: none; }
|
||||||
|
.nav-brand .brand-sep { display: none; }
|
||||||
|
.dashboard { padding: 1rem; }
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
.toggle-label {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.5rem;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
cursor: pointer;
|
||||||
|
color: var(--ink-muted);
|
||||||
|
padding: 0.35rem 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.toggle-label input[type="checkbox"] {
|
||||||
|
accent-color: var(--accent);
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.toggle-label span {
|
||||||
|
line-height: 1.4;
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Hagfish — Monthly Data Ingestion
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=%h/hagfisk
|
||||||
|
ExecStart=%h/hagfisk/hagfisk ingest
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Run Hagfish ingestion monthly
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-28 03:00:00
|
||||||
|
Persistent=true
|
||||||
|
Unit=hagfisk-ingest.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Hagfish — Faroese Fisheries Dashboard
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=%h/hagfisk
|
||||||
|
ExecStart=%h/hagfisk/hagfisk serve
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
Reference in New Issue
Block a user