Compare commits

..
62 Commits
Author SHA1 Message Date
bartal-lsn b9a5902460 bold 2026-08-27 10:40:22 +01:00
bartal-lsn a1a93fddfd argh 2026-08-27 10:39:46 +01:00
bartal-lsn ae8a48780d stupid vol 3 2026-08-27 10:38:36 +01:00
bartal-lsn 24115272cd stupid vol 2 2026-08-27 10:37:30 +01:00
bartal-lsn f2bfe5c1d6 stupid 2026-08-27 10:36:36 +01:00
bartal-lsn 6ebf123368 update readme 2026-08-27 10:35:53 +01:00
bartal-lsn dfa9911ded readme gif 2026-08-27 10:31:57 +01:00
bartal-lsn 55c190999a readme 2026-08-27 10:30:28 +01:00
bartal-lsn a88efaacd8 test 2026-08-27 00:28:57 +01:00
bartal-lsn 60398645d7 test 2026-08-27 00:27:03 +01:00
bartal-lsn 011920e2f9 test 2026-08-27 00:21:42 +01:00
bartal-lsn 3090d5ed5a test 2026-08-27 00:18:57 +01:00
bartal-lsn ab48ccffe4 test 2026-08-27 00:17:44 +01:00
bartal-lsn 1cd305e4c3 test 2026-08-27 00:11:34 +01:00
bartal-lsn 25dd518eaa test 2026-08-27 00:09:55 +01:00
bartal-lsn ecb2afe709 test 2026-08-27 00:09:18 +01:00
bartal-lsn 014c1d7eec test 2026-08-27 00:07:49 +01:00
bartal-lsn b8950393f9 er 2026-08-27 00:05:03 +01:00
bartal-lsn f4f2f2c76f test 2026-08-27 00:03:31 +01:00
bartal-lsn afe25bc3a3 try webm 2026-08-27 00:01:07 +01:00
bartal-lsn a421751d41 add readme and demo gif 2026-08-26 23:51:13 +01:00
bartal-lsn 1c281c4dc8 frontend done 2026-08-26 00:09:11 +01:00
bartal-lsn f9cfb99916 add data validation 2026-08-25 23:33:54 +01:00
bartal-lsn 3220d1d9a4 add correct date format 2026-08-25 22:04:37 +01:00
bartal-lsn 3ca28677ec add nicer theme, and next update in navbar 2026-08-25 21:58:46 +01:00
bartal-lsn 225aa9b1c6 move filters to left box 2026-08-25 21:25:50 +01:00
bartal-lsn e4e09df834 fix div.dahsboard height 2026-08-25 21:20:02 +01:00
bartal-lsn 665ad28364 add footer html and css 2026-08-25 21:02:09 +01:00
bartal-lsn 8f27c3658c change dropdown names on page load 2026-08-25 18:41:30 +01:00
bartal-lsn af2092d8f8 change font to noto sans 2026-08-25 15:48:17 +01:00
bartal-lsn 17102a120b fix dropdown issues and apply button 2026-08-25 15:43:14 +01:00
bartal-lsn 02f143b9b9 transparency issues fixed in dropdowns 2026-08-25 15:32:15 +01:00
bartal-lsn f85ddb0c66 add justfile remove taskfile 2026-08-25 15:14:35 +01:00
bartal-lsn 7cb28861e5 css themes fixed 2026-08-23 21:02:38 +01:00
bartal-lsn e95a5b1a9f moving closer to something useful 2026-08-22 23:41:18 +01:00
bartal-lsn a7cd06d40d fix clippy errs 2026-08-22 20:54:43 +01:00
bartal-lsn 30a4476643 more frontend tinkering 2026-08-21 11:32:28 +01:00
bartal-lsn 0dcb7d1414 fix fronted 2026-08-21 09:04:21 +01:00
bartal-lsn 1af25a20fe damn frontend 2026-08-20 23:21:41 +01:00
bartal-lsn 8630a61abd fixes 2026-08-20 22:42:55 +01:00
bartal-lsn 6bedac526f not rm rf whatever 2026-08-19 10:46:49 +01:00
bartal-lsn c0cf0d8495 add systemd files that are in prod 2026-08-19 10:42:49 +01:00
bartal-lsn 8fb8d507a4 deployment files 2026-08-19 10:39:38 +01:00
bartal-lsn 20786a9e14 update todo 2026-08-18 22:28:11 +01:00
bartal-lsn 42a697aa57 phase 5 qa and review pass 2026-08-18 22:21:45 +01:00
bartal-lsn bd7459e455 correct version 2026-08-18 22:05:08 +01:00
bartal-lsn fed7d1af0e fix test errs 2026-08-18 21:35:11 +01:00
bartal-lsn 2879fecf70 add structured logging 2026-08-18 21:28:48 +01:00
bartal-lsn 4f3995da17 phase 5, fix critical issues 2026-08-17 22:02:28 +01:00
bartal-lsn 4ef3ca706a phase 5 complete, sending to QA and code review 2026-08-17 21:20:35 +01:00
bartal-lsn 06886a2286 starting phase 5; clap cli 2026-08-17 18:02:47 +01:00
bartal-lsn 338e6c830a phase 4 todo check 2026-08-17 17:43:59 +01:00
bartal-lsn 53e553e155 phase 4 finished 2026-08-17 17:41:37 +01:00
bartal-lsn b072e2c3dc phase 4 first review and QA 2026-08-17 17:32:24 +01:00
bartal-lsn 1a6840d16c phase 4 ready for review 2026-08-17 16:57:38 +01:00
bartal-lsn 5e80204e34 add db gitignore 2026-08-17 14:38:02 +01:00
bartal-lsn d52bcf2d64 phase 3 complete 2026-08-17 14:08:20 +01:00
bartal-lsn f639b19c13 qa and review for phase 2 2026-08-17 10:18:58 +01:00
bartal-lsn 7a64f00014 phase 2 ready for qa review 2026-08-16 23:53:30 +01:00
bartal-lsn bd63b7e788 starting on duckdb database work 2026-08-16 23:42:55 +01:00
bartal-lsn 5db5c847b2 phase 1 complete 2026-08-16 23:05:29 +01:00
bartal-lsn 5ec60a71c3 phase 1 complete 2026-08-16 22:22:59 +01:00
22 changed files with 8579 additions and 451 deletions
+2
View File
@@ -1 +1,3 @@
/target
*.db*
*.log*
Generated
+3094
View File
File diff suppressed because it is too large Load Diff
+13 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "hagfish"
version = "0.1.0"
name = "hagfisk"
version = "0.0.5"
edition = "2024"
description = "Faroese fisheries data pipeline and dashboard"
authors = ["Bartal Læarsson"]
@@ -12,9 +12,19 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "2"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
tracing-appender = "0.2"
anyhow = "1"
duckdb = { version = "1.10505", features = ["bundled"] }
axum = "0.8"
tower-http = { version = "0.6", features = ["cors", "trace"] }
rust-embed = "8.12"
mime_guess = "2"
tokio-util = { version = "0.7", features = ["io"] }
tempfile = "3"
clap = { version = "4", features = ["derive"] }
[dev-dependencies]
mockito = "1"
tokio-test = "0.4"
reqwest = "0.12"
+93
View File
@@ -0,0 +1,93 @@
# Hagfisk
Faroese fisheries data pipeline and dashboard. Pulls monthly landing statistics from Hagstova Føroya's PX-Web API, stuffs them into a local DuckDB, and serves a filtered ECharts dashboard — all from a single Rust binary.
Named after the hagfish: a deep-sea creature famous for producing copious amounts of slime and being generally unpleasant. The name stuck anyway. The fish has no backbone. Neither does this README.
## What It Does
![Watch demo](./gif/hagfish.gif)
---
- Fetches fisheries data (mass + value by species, gear, zone, month) from `statbank.hagstova.fo`
- Stores everything in a local DuckDB file
- Serves a REST API with filtered queries, summaries, and Parquet export
- Renders an interactive dashboard with line, price-trend, and stacked bar charts
- Runs monthly ingestion automatically via systemd timer (28th of each month, 03:00)
## Tech Stack
| Thing | Choice | Why |
|---|---|---|
| Language | Rust 2024 | Fast, safe, single static binary. No runtime, no VM, no nonsense. |
| Web framework | Axum | Async, typed, plays well with Tokio. Doesn't fight you. |
| Database | DuckDB | Embedded OLAP. No server, no config, just a file. Perfect for analytics on a single machine. |
| Charts | ECharts | Powerful, flexible, no build step. Vanilla JS, no framework tax. |
| Static assets | rust-embed | Compiled into the binary. One artifact to deploy, nothing to forget. |
| Logging | tracing + tracing-subscriber | Structured logs to file and journal. |
| Deployment | Bare metal + systemd | No containers. No compose files. No port mappings. Just a binary and a `.service` unit. |
## Getting Started
```bash
cargo build --release
./target/release/hagfisk ingest --full
./target/release/hagfisk serve
```
Dashboard lives at `http://localhost:8090`.
## Configuration
`config.json` next to the binary:
```json
{
"duckdb_path": "hagfish.db",
"bind_address": "127.0.0.1:8090",
"data_source_url": "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px",
"log_file_path": "hagfish.log",
"allowed_origins": []
}
```
## CLI
```
hagfisk serve Start the web server
hagfisk ingest [--full] Fetch data (incremental or full backfill)
hagfisk export -o FILE Export landings to Parquet
```
## Deployment
The `justfile` handles everything, but you'll need to [install](https://just.systems/man/en/installation.html) it on your machine.
```
just deploy # Build, rsync, restore SELinux contexts, restart
just deploy-reingest # Same + full re-ingest
just logs-app # Tail application logs
just ps-app # Check service status
```
Systemd units live in `systemd/` — drop them in `~/.config/systemd/user/` and enable the timer for monthly ingestion.
## API
| Endpoint | What |
|---|---|
| `GET /healthz` | Alive check |
| `GET /api/species` | Species codes + Faroese names |
| `GET /api/zones` | Economic zone codes |
| `GET /api/gear` | Fishing gear codes |
| `GET /api/landings` | Filtered landing records (month range, species, gear, zone, measure) |
| `GET /api/summary` | Monthly aggregates, top 10 species, price/kg trend |
| `GET /api/summary/monthly-breakdown` | Per-month per-species breakdown |
| `GET /api/available-filters` | Cascading filter options based on active selections |
| `GET /api/export.parquet` | Full dataset as Parquet download |
Full PX-Web API reference in `docs/API.md`.
---
**Like the hagfish itself: not glamorous, but it works.**
+7
View File
@@ -0,0 +1,7 @@
{
"duckdb_path": "hagfish.db",
"bind_address": "127.0.0.1:8090",
"data_source_url": "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px",
"log_file_path": "hagfish.log",
"allowed_origins": []
}
+249
View File
@@ -0,0 +1,249 @@
# API Documentation
This document describes the PX-Web API used by hagfish to fetch Faroese fisheries statistics from the official Statbank.
## EndpointBase URL: https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px
- **GET**: Returns metadata (table structure, dimension codes, labels)
- **POST**: Returns data in JSON-stat2 format
## Endpoint Reference
### Health Check
- **GET** `/healthz` - Returns 200 OK if service is running
### Species Lookup
- **GET** `/api/species` - Returns list of all species codes and Faroese names
### Zones Lookup
- **GET** `/api/zones` - Returns list of all economic zone codes and labels
### Gear Lookup
- **GET** `/api/gear` - Returns list of all fishing gear codes and labels
### Landings Data
- **GET** `/api/landings` - Returns filtered landing records
**Query Parameters:**
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `month` | string | No | Single month filter (legacy, e.g., `2024M01`) |
| `month_from` | string | No | Start of date range (inclusive, e.g., `2024M01`) |
| `month_to` | string | No | End of date range (inclusive, e.g., `2024M12`) |
| `species` | string | No | Filter by species code (e.g., `COD`) |
| `gear` | string | No | Filter by gear code (e.g., `TR1`) |
| `zone` | string | No | Filter by zone code (e.g., `FO`) |
| `measure` | string | No | Filter by measure type (`MASS` or `VALUE`) |
| `limit` | integer | No | Max results (default: 10000, max: 10000) |
**Example:**
bash curl "http://localhost:8090/api/landings?month_from=2024M01&month_to=2024M06&species=COD&limit=1000"
### Summary Aggregates
- **GET** `/api/summary` - Returns aggregated statistics
**Query Parameters:**
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `species` | string | No | Filter aggregations by species code |
**Response Fields:**
- `monthly`: Array of `{month, total_mass, total_value}`
- `top_species`: Array of top 10 species by value
- `price_trend`: Array of `{month, price_per_kg}`
### Parquet Export
- **GET** `/api/export.parquet` - Downloads full dataset as Parquet file
**Note:** Triggers temp file creation in system temp directory with automatic cleanup after 300 seconds.
---
## Metadata Request (GET)
### Requestbash
curl -s "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
### Response Structurejson
{
"title": "AVR01010 Avreiðingar í nøgd og virði...",
"variables": [
{
"code": "measure",
"text": "mát",
"role": null,
"values": ["MASS", "VALUE"],
"valueTexts": ["Nøgd", "Virði"]
}
]
}
### Fields
| Field | Type | Description |
|-------|------|-------------|
| `title` | string | Table name in Faroese |
| `variables[].code` | string | Dimension identifier (used in queries) |
| `variables[].text` | string | Human-readable label in Faroese |
| `variables[].role` | null/string | Classification (always `null` on this endpoint) |
| `variables[].values` | string[] | Valid dimension codes |
| `variables[].valueTexts` | string[] | Display labels (parallel with `values`) |
### Verified Dimension Codes
| Variable Code | Faroese Label | Values |
|---------------|---------------|--------|
| `measure` | mát | `MASS`, `VALUE` |
| `Species (ASFIS2022)` | Fiskaslag (ASFIS2022) | `TOTAL`, `148XXXXXXX00000`, ... (72 total) |
| `Fishing Gear (ISSCFG2016)` | Reiðskapur (ISSCFG2016) | `TOTAL`, ... (12 total) |
| `Economic Zone (GEONOM2023)` | Búskapar øki (GEONOM2023) | `TOTAL`, ... (10 total) |
| `Processing (EUMOFAPresentation)` | Virking (EUMOFAPresentation) | `TOTAL`, ... (6 total) |
| `Preservation (EUMOFAPreservation)` | Viðgerð (EUMOFAPreservation) | `TOTAL`, ... (9 total) |
| `Shipsize` | Skipastødd | `TOTAL`, ... (11 total) |
| `month` | mánaður | `2015M01` through `2026M05` (137 months) |
**Note**: `role` is always `null`. Time dimension identification must use `code == "month"`.
---
## Data Request (POST)
### Request Body Formatjson
{
"query": [
{
"code": "dimension_code",
"selection": {
"filter": "item",
"values": ["value1", "value2"]
}
}
],
"response": {
"format": "json-stat2"
}
}
### Filter Types
| Filter | Values | Description |
|--------|--------|-------------|
| `item` | explicit codes | Select specific values |
| `all` | `["*"]` | Wildcard — select all values |
| `top` | `["TOP_N"]` | Top N values by magnitude |
### Example Querybash
curl -s -X POST
-H "Content-Type: application/json"
-d '{
"query": [
{"code": "month", "selection": {"filter": "item", "values": ["2024M01", "2024M02"]}},
{"code": "Species (ASFIS2022)", "selection": {"filter": "all", "values": [""]}},
{"code": "Fishing Gear (ISSCFG2016)", "selection": {"filter": "all", "values": [""]}},
{"code": "Economic Zone (GEONOM2023)", "selection": {"filter": "all", "values": ["*"]}},
{"code": "Processing (EUMOFAPresentation)", "selection": {"filter": "item", "values": ["TOTAL"]}},
{"code": "Preservation (EUMOFAPreservation)", "selection": {"filter": "item", "values": ["TOTAL"]}},
{"code": "Shipsize", "selection": {"filter": "item", "values": ["TOTAL"]}},
{"code": "measure", "selection": {"filter": "item", "values": ["MASS", "VALUE"]}}
],
"response": {"format": "json-stat2"}
}'
"https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
### Cell Limit
- Maximum cells per query: ~8,000,000
- Recommended maximum: 1,000,000 for reliability
- To fetch full dataset, paginate by month or use smaller dimension selections
---
## Response Format (JSON-stat2)
### Structurejson
{
"class": "dataset",
"label": "...",
"id": ["measure", "Species (ASFIS2022)", ..., "month"],
"size": [2, 72, 12, 10, 1, 1, 1, 2],
"dimension": {
"measure": {
"label": "mát",
"category": {
"index": {"MASS": 0, "VALUE": 1},
"label": {"MASS": "Nøgd", "VALUE": "Virði"}
}
}
},
"value": [73653738, 1234567, ...]
}
### Fields
| Field | Description |
|-------|-------------|
| `id` | Array of dimension codes in order (defines cube layout) |
| `size` | Cardinality per dimension (parallel with `id`) |
| `dimension.<code>.category.index` | Maps value code → numeric position in dimension |
| `dimension.<code>.category.label` | Maps value code → display text |
| `value` | Flattened array of measurements (row-major order) |
### Row-Major Index Decoding
Values are stored in row-major order: the last dimension (`month`) varies fastest.
Given `size = [2, 72, 12, 10, 1, 1, 1, 2]`:
- Flat index `0` → `[0,0,0,0,0,0,0,0]` = measure=MASS, species=TOTAL, ..., month=2024M01
- Flat index `1` → `[0,0,0,0,0,0,0,1]` = measure=MASS, species=TOTAL, ..., month=2024M02
- Flat index `2` → `[0,0,0,0,0,0,1,0]` = measure=MASS, species=SPECIES_1, ..., month=2024M01
Decoding algorithm (reverse modulo):rust
fn decode(flat_index: usize, sizes: &[usize]) -> Vec<usize> {
let mut indices = Vec::new();
let mut remaining = flat_index;
for &size in sizes.iter().rev() {
indices.push(remaining % size);
remaining /= size;
}
indices.reverse();
indices
}
### Sentinel Values
| Value | Meaning |
|-------|---------|
| `-1.0` | Missing/no data (treated as `NULL`) |
| `null` | Missing/no data (already nullable in JSON) |
---
## Known Constraints
1. **Cell limit**: ~8 million cells per query
2. **Rate limits**: Unknown — assume reasonable backoff for large downloads
3. **Language**: API responds in requested language (`fo` or `en`)
4. **Time zone**: No timezone specified — treat timestamps as local
5. **Updates**: Data updated irregularly — metadata timestamp in response header
---
## Integration Checklist
- [x] Metadata endpoint reachable via GET
- [x] POST queries return valid JSON-stat2
- [x] Dimension codes match `variables[].code` from metadata
- [x] Row-major index decoding verified
- [x] Sentinel value coercion (`-1.0` → `None`) implemented
- [x] Unicode (Faroese characters) handled correctly
- [ ] Incremental ingestion logic tested
- [ ] Parquet export tested
- [ ] Error handling for network timeouts implemented
Summary of Changes
File Change Reason
src/api.rs Added MAX_LIMIT constant and hard cap Prevent unlimited query results
src/api.rs Replaced CorsLayer::permissive() with explicit origins Security hardening
src/api.rs Updated tempfile::Builder with prefix Ensure cleanup function finds files
src/api.rs Added test_get_landings_range_filter test Cover new range-filtering logic
src/types.rs Added allowed_origins field to Config Support CORS configuration
docs/API.md Added endpoint reference table Document new parameters
+54 -37
View File
@@ -1,4 +1,4 @@
# HAGFISH Project todo
# HAGFISH Project TODO
`GET statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px — metadata`
POST same URL — JSON-stat2 query
@@ -59,55 +59,72 @@ hagfish/
### Phase 1: Types & Ingestion
- [ ] 1.1 Define types in types.rs: MetadataResponse, VariableMeta, DataResponse, DataRow, Query, Selection, QueryItem, Config — all with serde derives
- [ ] 1.2 Implement ingest.rs::fetch_metadata(url) — GET request, parse JSON, return HashMap<(variable_code, value_code), faroese_label>
- [ ] 1.3 Implement ingest.rs::build_query(months: &[String]) — construct POST body with all species/gear/zones set to "*", processing/preservation/shipsize set to TOTAL, measure set to both MASS and VALUE
- [ ] 1.4 Implement ingest.rs::fetch_data(url, query) — POST request, parse JSON-stat2 response, return Vec<DataRow>
- [ ] 1.5 Implement ingest.rs::parse_row(row, lookup_maps) — decode key[] positions into labeled Landing struct. Handle "-" → None.
- [ ] 1.6 Write unit tests: mock JSON-stat2 response, verify key-to-label mapping, verify "-" handling, verify Faroese Unicode characters in species names (ð, á, í, ý, ø, ó)
- [x] 1.1 Define types in types.rs: MetadataResponse, VariableMeta, DataResponse, DataRow, Query, Selection, QueryItem, Config — all with serde derives
- [x] 1.2 Implement ingest.rs::fetch_metadata(url) — GET request, parse JSON, return HashMap<(variable_code, value_code), faroese_label>
- [x] 1.3 Implement ingest.rs::build_query(months: &[String]) — construct POST body with all species/gear/zones set to "*", processing/preservation/shipsize set to TOTAL, measure set to both MASS and VALUE
- [x] 1.4 Implement ingest.rs::fetch_data(url, query) — POST request, parse JSON-stat2 response, return Vec<DataRow>
- [x] 1.5 Implement ingest.rs::parse_row(row, lookup_maps) — decode key[] positions into labeled Landing struct. Handle "-" → None.
- [x] 1.6 Write unit tests: mock JSON-stat2 response, verify key-to-label mapping, verify "-" handling, verify Faroese Unicode characters in species names (ð, á, í, ý, ø, ó)
### Phase 2: DuckDB Storage
- [ ] 2.1 Add duckdb crate dependency (bundled feature)
- [ ] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code.
- [ ] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE
- [ ] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response
- [ ] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion
- [ ] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month
- [ ] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling
- [x] 2.1 Add duckdb crate dependency (bundled feature)
- [x] 2.2 Implement db.rs::init(path) — create tables: landings fact table + 6 lookup tables. Add indexes on month, species_code.
- [x] 2.3 Implement db.rs::upsert_landings(rows) — batch insert with delete+insert per month or INSERT OR REPLACE
- [x] 2.4 Implement db.rs::update_lookups(metadata) — populate lookup tables from metadata response
- [x] 2.5 Implement db.rs::get_last_month() — query max month from landings table for incremental ingestion
- [x] 2.6 Implement db.rs::export_parquet(path) — COPY landings TO 'path' (FORMAT PARQUET) partitioned by month
- [x] 2.7 Write integration tests: init in-memory DB, insert sample rows, query back, verify NULL handling
### Phase 3: API (Axum)
- [ ] 3.1 Set up Axum router in main.rs with Tokio runtime. AppState holds DuckDB connection wrapped in Mutex.
- [ ] 3.2 Implement GET /api/species — query species lookup table, return JSON array
- [ ] 3.3 Implement GET /api/landings — parse query params, build DuckDB SQL with WHERE clauses. Support: months, species, gear, zone, measure filters.
- [ ] 3.4 Implement GET /api/summary — aggregate query: total mass + value by month, top 10 species by value, price/kg trend
- [ ] 3.5 Implement GET /api/export.parquet — generate and stream Parquet via DuckDB COPY
- [ ] 3.6 Implement GET /healthz
- [ ] 3.7 Serve static files via rust-embed
- [ ] 3.8 Write API tests
- [x] 3.1 Set up Axum router in main.rs with Tokio runtime. AppState holds DuckDB connection wrapped in Mutex.
- [x] 3.2 Implement GET /api/species — query species lookup table, return JSON array
- [x] 3.3 Implement GET /api/landings — parse query params, build DuckDB SQL with WHERE clauses. Support: months, species, gear, zone, measure filters.
- [x] 3.4 Implement GET /api/summary — aggregate query: total mass + value by month, top 10 species by value, price/kg trend
- [x] 3.5 Implement GET /api/export.parquet — generate and stream Parquet via DuckDB COPY
- [x] 3.6 Implement GET /healthz
- [x] 3.7 Serve static files via rust-embed
- [x] 3.8 Write API tests
#### Production Hardening (Deferred to Post-Launch)
- [ ] 🔒 DB operation timeouts — Add 30s `tokio::time::timeout()` wrapper around all `spawn_blocking` DB calls
- [ ] 🔒 CORS hardening — Replace `CorsLayer::permissive()` with explicit allowed origins before production deployment
- [ ] 🔒 Rate limiting — Optional: add `tower_governor` middleware to prevent abuse on public deployments
- [ ] ⚠️ Startup config validation — Verify `data_source_url` is reachable, `duckdb_path` is writable before accepting connections
### Phase 4: Frontend
- [ ] 4.1 index.html — dropdown filters (species, zone, gear, month range) and 3 chart containers
- [ ] 4.2 app.js — fetch species list on load, populate dropdowns, fetch /api/landings, render charts
- [ ] 4.3 ECharts line chart: x=month, y=mass/value toggle
- [ ] 4.4 ECharts stacked bar: x=month, y=value by species (top 10 + "other")
- [ ] 4.5 ECharts donut: species distribution for selected month
- [ ] 4.6 Loading states, error handling, empty state
- [ ] 4.7 Responsive layout, plain CSS
- [x] 4.1 index.html — dropdown filters (species, zone, gear, month range) and 3 chart containers
- [x] 4.2 app.js — fetch species list on load, populate dropdowns, fetch /api/landings, render charts
- [x] 4.3 ECharts line chart: x=month, y=mass/value toggle
- [x] 4.4 ECharts stacked bar: x=month, y=value by species (top 10 + "other")
- [x] 4.5 ECharts donut: species distribution for selected month
- [x] 4.6 Loading states, error handling, empty state
- [x] 4.7 Responsive layout, plain CSS
### Phase 5: CLI & Scheduling
- [ ] 5.1 Add clap derive subcommands: hagfish ingest [--full] and hagfish serve
- [ ] 5.2 Implement incremental logic: read get_last_month(), compute remaining months from metadata, fetch in batches if >12 months
- [ ] 5.3 Log ingestion runs with slog (rows inserted, duration, errors)
- [ ] 5.4 Add hagfish export --out /path/to/parquet subcommand
- [ ] 5.5 Load config.json on startup (DuckDB path, bind address, data source URL, log file path)
- [x] 5.1 Add clap derive subcommands: hagfish ingest [--full] and hagfish serve
- [x] 5.2 Implement incremental logic: read get_last_month(), compute remaining months from metadata, fetch in batches if >12 months
- [x] 5.3 Log ingestion runs with tracing (rows inserted, duration, errors)
- [x] 5.4 Add hagfish export --out /path/to/parquet subcommand
- [x] 5.5 Load config.json on startup (DuckDB path, bind address, data source URL, log file path)
### Phase 6: Bare Metal Deployment
- [ ] 6.1 Write systemd service unit file (hagfish.service) — ExecStart=/usr/local/bin/hagfish serve, restart policy
- [ ] 6.2 Write systemd timer (hagfish-ingest.timer + hagfish-ingest.service) — monthly, runs hagfish ingest
- [ ] 6.3 Taskfile: build (release, static), deploy (rsync binary + config + units, ssh reload)
- [x] 6.1 Write systemd service unit file (hagfish.service) — ExecStart=/usr/local/bin/hagfish serve, restart policy
- [x] 6.2 Write systemd timer (hagfish-ingest.timer + hagfish-ingest.service) — monthly, runs hagfish ingest
- [x] 6.3 Taskfile: build (release, static), deploy (rsync binary + config + units, ssh reload)
- [ ] 6.4 README with ELI5 Technology Choices section (why DuckDB, why Rust, why embedded static assets)
### Phase 7: post deploy fixes
---
## Current Status
**Bug Fixes**: Next up
**Production Hardening**: Deferred ⏸️
+99
View File
@@ -0,0 +1,99 @@
[[ FO ]]
- Lýsing
Sum maðurin segði; tað frálíkt at vísa á, at ein dugur at heinta dátur úr einum API og vísa tað á einari heimasíðu har brúkarar kunna filtrera visualiseringina.
Her nýta vit fiskahagtøl sum verða dagførd einaferð um mánaðin. So kunna vit líkasum síggja hvat hendir har, ha? Eg kalli hesa applikatión 'hagfisk'. Altso hagtøl + fisk. Hagfisk. Hagfisk, havi eg funni útav, er eisini ein heilt ótrúligani ólekkur og slímutur fiskur.
[INSERT PHOTO].
Men ja, eg fór við tí allíkavæl.
Anyways, her er ikki so nógv at siga: taka dátur frá hagstovu-API, skriva til ein lokalan DuckDB OLTP-dátugrunn á servara, servera API til web dátur, front end til at visualisera dáturnar.
- Leiðbeining
Far inn á síðuna, hygg og filtrera.
- Tøkni
Rust - axum, serde, tokio, tower-http, reqwest, rust-embed
DuckDB
html
js
css
echarts
- Keldu kota
https://git.flo.fo/FLO/hagfisk
- Leinkja
https://hagfisk.poc.fló.fo
[[ DA ]]
## Beskrivelse
Som den gamle mand sagde: Det er jo meget fint at vise, at man kan hente data fra et API og visualisere dem på en hjemmeside, hvor brugerne kan filtrere visningen.
Her bruger vi fiskeristatistik, som bliver opdateret én gang om måneden. Så kan vi ligesom se, hvad der sker, ikke? Jeg kalder denne applikation 'hagfisk'. Altså statistik (FO = hagtøl) + fisk. Hagfisk. Jeg har fundet ud af, at hagfisk også er en helt utrolig modbydelig og slimet fisk.
[INDSÆT FOTO].
Men ja, jeg gik med navnet alligevel.
Anyways, der er ikke så meget mere at sige: Hent data fra Hagstova-API'et, gem dem i en lokal DuckDB-database på serveren, eksponér dem via et API til webfrontend'en, og visualisér dataene.
## Vejledning
Gå ind på siden, kig dig omkring og filtrér.
## Teknologi
Rust – axum, serde, tokio, tower-http, reqwest, rust-embed
DuckDB
HTML
JS
CSS
ECharts
## Kildekode
https://git.flo.fo/FLO/hagfisk
## Link
https://hagfisk.poc.fló.fo
[[ EN ]]
## Description
As the old man said: It’s pretty great to show that you can pull data from an API and visualise it on a website where users can filter the view.
Here we’re using fisheries statistics, updated once a month. So we can kind of see what’s going on, right? I’m calling this application ‘hagfisk’. Basically, statistics + fish. Hagfisk. I’ve since discovered that hagfish is also an incredibly gross and slimy fish.
[INSERT PHOTO].
But yeah, I went with the name anyway.
Anyways, there’s not much more to say: fetch data from the Hagstova API, store it in a local DuckDB database on the server, expose it through an API to the web frontend, and visualise the data.
## Guide
Go to the page, have a look around, and filter.
## Technology
Rust – axum, serde, tokio, tower-http, reqwest, rust-embed
DuckDB
HTML
JS
CSS
ECharts
## Source Code
https://git.flo.fo/FLO/hagfisk
## Link
https://hagfisk.poc.fló.fo
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.8 MiB

+61
View File
@@ -0,0 +1,61 @@
host := "bl@flo-homeserver"
hagfisk_dir := "~/hagfisk"
# Build binary locally
build:
echo "Building hagfisk binary..."
cargo build --release
# Stop app service (must stop before DB operations)
stop-app:
ssh {{host}} "systemctl --user stop hagfisk.service"
# Start app service
start-app:
ssh {{host}} "systemctl --user start hagfisk.service"
# Restart app service
restart-app:
ssh {{host}} "systemctl --user restart hagfisk.service"
# Deploy and restart (safe for regular updates)
deploy: build
ssh -q {{host}} "rm -f {{hagfisk_dir}}/hagfisk"
echo "Syncing binary to remote..."
rsync -avz --progress ./target/release/hagfisk {{host}}:{{hagfisk_dir}}/hagfisk
echo "Restoring SELinux contexts..."
ssh -t -q {{host}} "sudo restorecon -RF {{hagfisk_dir}}"
ssh -q {{host}} "systemctl --user daemon-reload"
ssh -q {{host}} "systemctl --user restart hagfisk.service"
# Deploy with full re-ingest (stops service first)
deploy-reingest: build stop-app
echo "Syncing binary to remote..."
rsync -avz --progress ./target/release/hagfisk {{host}}:{{hagfisk_dir}}/hagfisk
echo "Restoring SELinux contexts..."
ssh -t -q {{host}} "sudo restorecon -RF {{hagfisk_dir}}"
ssh -q {{host}} "systemctl --user daemon-reload"
echo "Running full re-ingest..."
ssh {{host}} "cd {{hagfisk_dir}} && ./hagfisk ingest --full"
echo "Starting service..."
ssh {{host}} "systemctl --user start hagfisk.service"
# Manual full ingest (assumes service is stopped)
ingest-full:
ssh {{host}} "cd {{hagfisk_dir}} && ./hagfisk ingest --full"
# Stream logs from remote app
logs-app:
ssh {{host}} "journalctl --user -u hagfisk -f"
# Show status of app
ps-app:
ssh {{host}} "systemctl --user status hagfisk.service"
# Stream logs from remote ingest (manual run)
logs-ingest:
ssh {{host}} "journalctl --user -u hagfisk-ingest -f"
# Show status of ingest
ps-ingest:
ssh {{host}} "systemctl --user status hagfisk-ingest.service"
+1493
View File
File diff suppressed because it is too large Load Diff
+25
View File
@@ -0,0 +1,25 @@
use clap::{Parser, Subcommand};
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "hagfish", version, about)]
pub struct Cli {
#[arg(short, long, value_name = "FILE")]
pub config: Option<PathBuf>,
#[command(subcommand)]
pub command: Command,
}
#[derive(Subcommand)]
pub enum Command {
Ingest {
#[arg(long)]
full: bool,
},
Serve,
Export {
#[arg(short, long, value_name = "FILE")]
out: PathBuf,
},
}
+596
View File
@@ -0,0 +1,596 @@
use crate::types::{Landing, LookupMap};
use duckdb::{Connection, params};
use std::collections::HashSet;
use thiserror::Error;
use tracing::info;
const LOOKUP_TABLES: &[(&str, &str)] = &[
("Species (ASFIS2022)", "species"),
("Fishing Gear (ISSCFG2016)", "gear"),
("Economic Zone (GEONOM2023)", "zone"),
("Processing (EUMOFAPresentation)", "processing"),
("Preservation (EUMOFAPreservation)", "preservation"),
("Shipsize", "shipsize"),
];
#[derive(Debug, Error)]
pub enum DbError {
#[error("DuckDB error: {0}")]
Duckdb(#[from] duckdb::Error),
#[error("invalid export path: {0}")]
InvalidPath(String),
}
pub type Result<T> = std::result::Result<T, DbError>;
pub fn init(path: &str) -> Result<Connection> {
let conn = Connection::open(path)?;
init_schema(&conn)?;
info!("Initialized DuckDB at {}", path);
Ok(conn)
}
pub fn init_schema(conn: &Connection) -> Result<()> {
conn.execute_batch(
"
CREATE TABLE IF NOT EXISTS species (
code TEXT PRIMARY KEY,
label TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS gear (
code TEXT PRIMARY KEY,
label TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS zone (
code TEXT PRIMARY KEY,
label TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS processing (
code TEXT PRIMARY KEY,
label TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS preservation (
code TEXT PRIMARY KEY,
label TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS shipsize (
code TEXT PRIMARY KEY,
label TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS landings (
month TEXT NOT NULL,
species_code TEXT NOT NULL,
species_label TEXT,
gear_code TEXT NOT NULL,
zone_code TEXT NOT NULL,
processing_code TEXT NOT NULL,
preservation_code TEXT NOT NULL,
shipsize_code TEXT NOT NULL,
measure_code TEXT NOT NULL,
value DOUBLE,
PRIMARY KEY (month, species_code, gear_code, zone_code,
processing_code, preservation_code, shipsize_code,
measure_code)
);
CREATE INDEX IF NOT EXISTS idx_landings_month
ON landings(month);
CREATE INDEX IF NOT EXISTS idx_landings_species
ON landings(species_code);
",
)?;
Ok(())
}
pub fn update_lookups(conn: &Connection, lookup_maps: &LookupMap) -> Result<()> {
for (dim_code, table_name) in LOOKUP_TABLES {
if let Some(codes) = lookup_maps.get(*dim_code) {
let sql = format!(
"INSERT INTO {} (code, label) VALUES (?, ?) \
ON CONFLICT(code) DO UPDATE SET label = excluded.label",
table_name
);
for (code, label) in codes {
conn.execute(&sql, params![code, label])?;
}
info!(
"Updated {} entries in '{}' lookup table",
codes.len(),
table_name
);
}
}
Ok(())
}
pub fn upsert_landings(conn: &Connection, rows: &[Landing]) -> Result<usize> {
if rows.is_empty() {
return Ok(0);
}
let months: HashSet<&str> = rows.iter().map(|r| r.month.as_str()).collect();
let tx = conn.unchecked_transaction()?;
for month in &months {
tx.execute("DELETE FROM landings WHERE month = ?", params![month])?;
}
{
let mut app = tx.appender("landings")?;
for row in rows {
app.append_row(params![
row.month,
row.species_code,
row.species_label,
row.gear_code,
row.zone_code,
row.processing_code,
row.preservation_code,
row.shipsize_code,
row.measure_code,
row.value,
])?;
}
app.flush()?;
}
tx.commit()?;
info!(
"Upserted {} rows across {} month(s)",
rows.len(),
months.len()
);
Ok(rows.len())
}
pub fn get_last_month(conn: &Connection) -> Result<Option<String>> {
let result = conn.query_row("SELECT MAX(month) FROM landings", [], |row| {
let val: Option<String> = row.get(0)?;
Ok(val)
})?;
Ok(result)
}
pub fn export_parquet(conn: &Connection, path: &str) -> Result<()> {
if path.contains('\'') || path.contains('\0') {
return Err(DbError::InvalidPath(path.to_string()));
}
if !path
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '/' | '-' | '_' | ' '))
{
return Err(DbError::InvalidPath(format!(
"Path contains disallowed characters: {path}"
)));
}
conn.execute(
&format!(
"COPY (SELECT * FROM landings) TO '{}' (FORMAT PARQUET)",
path
),
[],
)?;
info!("Exported landings to Parquet at {}", path);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::Landing;
use std::collections::HashMap;
fn test_conn() -> Connection {
let conn = Connection::open_in_memory().unwrap();
init_schema(&conn).unwrap();
conn
}
fn sample_landings() -> Vec<Landing> {
vec![
Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(1234.5),
},
Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "VALUE".to_string(),
value: None,
},
Landing {
month: "2024M02".to_string(),
species_code: "HER".to_string(),
species_label: "Sild".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(5678.9),
},
]
}
#[test]
fn test_init_creates_all_tables() {
let conn = test_conn();
for table in &[
"species",
"gear",
"zone",
"processing",
"preservation",
"shipsize",
"landings",
] {
let count: i64 = conn
.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |row| {
row.get(0)
})
.unwrap_or_else(|_| panic!("Failed to query {}", table));
assert_eq!(count, 0, "Table {} should be empty", table);
}
}
#[test]
fn test_init_is_idempotent() {
let conn = Connection::open_in_memory().unwrap();
init_schema(&conn).unwrap();
init_schema(&conn).unwrap();
}
#[test]
fn test_upsert_insert_and_idempotency() {
let conn = test_conn();
let rows = sample_landings();
let inserted = upsert_landings(&conn, &rows).unwrap();
assert_eq!(inserted, 3);
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 3);
let inserted = upsert_landings(&conn, &rows).unwrap();
assert_eq!(inserted, 3);
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 3);
}
#[test]
fn test_upsert_replaces_month_data() {
let conn = test_conn();
upsert_landings(&conn, &sample_landings()).unwrap();
let modified = vec![Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(9999.9),
}];
upsert_landings(&conn, &modified).unwrap();
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 2);
let val: f64 = conn
.query_row(
"SELECT value FROM landings
WHERE month = '2024M01' AND measure_code = 'MASS'",
[],
|row| row.get(0),
)
.unwrap();
assert!((val - 9999.9).abs() < f64::EPSILON);
}
#[test]
fn test_upsert_empty_input() {
let conn = test_conn();
let result = upsert_landings(&conn, &[]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), 0);
}
#[test]
fn test_upsert_multi_month_atomic() {
let conn = test_conn();
upsert_landings(&conn, &sample_landings()).unwrap();
let rows = vec![
Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(111.0),
},
Landing {
month: "2024M02".to_string(),
species_code: "HER".to_string(),
species_label: "Sild".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(222.0),
},
];
upsert_landings(&conn, &rows).unwrap();
let jan_val: f64 = conn
.query_row(
"SELECT value FROM landings WHERE month = '2024M01' AND measure_code = 'MASS'",
[],
|row| row.get(0),
)
.unwrap();
assert!((jan_val - 111.0).abs() < f64::EPSILON);
let feb_val: f64 = conn
.query_row(
"SELECT value FROM landings WHERE month = '2024M02' AND measure_code = 'MASS'",
[],
|row| row.get(0),
)
.unwrap();
assert!((feb_val - 222.0).abs() < f64::EPSILON);
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 2);
}
#[test]
fn test_null_preserved_on_insert() {
let conn = test_conn();
let rows = vec![Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "VALUE".to_string(),
value: None,
}];
upsert_landings(&conn, &rows).unwrap();
let val: Option<f64> = conn
.query_row(
"SELECT value FROM landings WHERE measure_code = 'VALUE'",
[],
|row| row.get(0),
)
.unwrap();
assert!(val.is_none());
}
#[test]
fn test_null_preserved_after_upsert() {
let conn = test_conn();
upsert_landings(&conn, &sample_landings()).unwrap();
let val: Option<f64> = conn
.query_row(
"SELECT value FROM landings
WHERE month = '2024M01' AND measure_code = 'VALUE'",
[],
|row| row.get(0),
)
.unwrap();
assert!(val.is_none());
}
#[test]
fn test_get_last_month_empty_table() {
let conn = test_conn();
let result = get_last_month(&conn).unwrap();
assert!(result.is_none());
}
#[test]
fn test_get_last_month_populated() {
let conn = test_conn();
upsert_landings(&conn, &sample_landings()).unwrap();
let last = get_last_month(&conn).unwrap();
assert_eq!(last.as_deref(), Some("2024M02"));
}
#[test]
fn test_get_last_month_single_month() {
let conn = test_conn();
let rows = vec![Landing {
month: "2015M03".to_string(),
species_code: "COD".to_string(),
species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(100.0),
}];
upsert_landings(&conn, &rows).unwrap();
let last = get_last_month(&conn).unwrap();
assert_eq!(last.as_deref(), Some("2015M03"));
}
#[test]
fn test_update_lookups_populates_tables() {
let conn = test_conn();
let mut maps = LookupMap::new();
let mut species = HashMap::new();
species.insert("COD".to_string(), "Toskur".to_string());
species.insert("HER".to_string(), "Sild".to_string());
maps.insert("Species (ASFIS2022)".to_string(), species);
let mut gear = HashMap::new();
gear.insert("TOTAL".to_string(), "Tilsamans".to_string());
maps.insert("Fishing Gear (ISSCFG2016)".to_string(), gear);
update_lookups(&conn, &maps).unwrap();
let label: String = conn
.query_row("SELECT label FROM species WHERE code = 'COD'", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(label, "Toskur");
let label: String = conn
.query_row("SELECT label FROM gear WHERE code = 'TOTAL'", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(label, "Tilsamans");
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 2);
}
#[test]
fn test_update_lookups_upsert_existing() {
let conn = test_conn();
let mut maps = LookupMap::new();
let mut species = HashMap::new();
species.insert("COD".to_string(), "Old label".to_string());
maps.insert("Species (ASFIS2022)".to_string(), species);
update_lookups(&conn, &maps).unwrap();
let mut species2 = HashMap::new();
species2.insert("COD".to_string(), "Toskur".to_string());
let mut maps2 = LookupMap::new();
maps2.insert("Species (ASFIS2022)".to_string(), species2);
update_lookups(&conn, &maps2).unwrap();
let label: String = conn
.query_row("SELECT label FROM species WHERE code = 'COD'", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(label, "Toskur");
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM species", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 1);
}
#[test]
fn test_update_lookups_skips_missing_dims() {
let conn = test_conn();
let mut maps = LookupMap::new();
let mut species = HashMap::new();
species.insert("COD".to_string(), "Toskur".to_string());
maps.insert("Species (ASFIS2022)".to_string(), species);
update_lookups(&conn, &maps).unwrap();
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM gear", [], |row| row.get(0))
.unwrap();
assert_eq!(count, 0);
}
#[test]
fn test_export_parquet() {
let conn = test_conn();
upsert_landings(&conn, &sample_landings()).unwrap();
let tmp = tempfile::NamedTempFile::new().unwrap();
let path = tmp.path().to_str().unwrap().to_string() + ".parquet";
export_parquet(&conn, &path).unwrap();
let count: i64 = conn
.query_row(
&format!("SELECT COUNT(*) FROM read_parquet('{}')", path),
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(count, 3);
}
#[test]
fn test_faroese_labels_round_trip() {
let conn = test_conn();
let rows = vec![Landing {
month: "2024M01".to_string(),
species_code: "148XXXXXXX00000".to_string(),
species_label: "Hýsa".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(100.0),
}];
upsert_landings(&conn, &rows).unwrap();
let label: String = conn
.query_row("SELECT species_label FROM landings LIMIT 1", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(label, "Hýsa");
assert!(label.contains('ý'));
assert!(!label.contains('\u{FFFD}'));
}
}
+643 -355
View File
File diff suppressed because it is too large Load Diff
+380 -2
View File
@@ -1,3 +1,381 @@
fn main() {
println!("Hello, world!");
mod api;
mod cli;
mod db;
mod ingest;
mod types;
use std::path::Path;
use std::sync::Arc;
use std::time::Instant;
use axum::serve;
use clap::Parser;
use tokio::net::TcpListener;
use tracing_subscriber::EnvFilter;
struct LoggerGuard {
_guard: Option<tracing_appender::non_blocking::WorkerGuard>,
}
fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json");
match log_file_path {
Some(path) => {
let file = std::fs::File::create(path).expect("Failed to create log file");
let (non_blocking, guard) = tracing_appender::non_blocking(file);
let subscriber = tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(non_blocking);
if is_json {
subscriber.json().init();
} else {
subscriber.init();
}
LoggerGuard {
_guard: Some(guard),
}
}
None => {
let subscriber = tracing_subscriber::fmt().with_env_filter(filter);
if is_json {
subscriber.json().init();
} else {
subscriber.init();
}
LoggerGuard { _guard: None }
}
}
}
fn cleanup_stale_parquet_files() {
let tmp_dir = std::env::temp_dir();
match std::fs::read_dir(&tmp_dir) {
Ok(entries) => {
for entry in entries.filter_map(|e| e.ok()) {
if let Some(name) = entry.file_name().to_str()
&& name.starts_with("hagfish-")
&& name.ends_with(".parquet")
&& let Ok(metadata) = entry.metadata()
&& let Ok(modified) = metadata.modified()
{
let age = std::time::SystemTime::now().duration_since(modified);
if age.map(|d| d.as_secs() > 3600).unwrap_or(false) {
let _ = std::fs::remove_file(entry.path());
}
}
}
}
Err(e) => tracing::warn!(error = %e, "Failed to list temp dir"),
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let cli = cli::Cli::parse();
let config = match &cli.config {
Some(path) => types::Config::load(path)?,
None => {
let default_path = std::path::Path::new("config.json");
if default_path.exists() {
types::Config::load(default_path)?
} else {
eprintln!("WARN: No config file found, using defaults");
types::Config::default()
}
}
};
let _logger_guard = init_logging(config.log_file_path.as_deref());
tracing::info!(
duckdb_path = %config.duckdb_path,
bind_address = %config.bind_address,
"configuration loaded"
);
match cli.command {
cli::Command::Serve => run_serve(&config).await,
cli::Command::Ingest { full } => run_ingest(&config, full).await,
cli::Command::Export { out } => run_export(&config, &out),
}
}
async fn run_serve(config: &types::Config) -> anyhow::Result<()> {
cleanup_stale_parquet_files();
let conn = db::init(&config.duckdb_path)?;
let state = api::AppState {
conn: Arc::new(tokio::sync::Mutex::new(conn)),
config: Arc::new(config.clone()),
};
let app = api::build_router(state);
let listener = TcpListener::bind(&config.bind_address).await?;
tracing::info!(address = %config.bind_address, "hagfish server started");
serve(listener, app).await?;
Ok(())
}
async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
let run_start = Instant::now();
tracing::info!(
mode = if full { "full" } else { "incremental" },
"starting ingestion run"
);
let conn = db::init(&config.duckdb_path)?;
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(60))
.build()?;
let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url)
.await
.inspect_err(|e| {
tracing::error!(error = %e, stage = "fetch_metadata", "ingestion failed");
})?;
db::update_lookups(&conn, &lookup_maps)?;
let species_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_SPECIES);
let gear_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_GEAR);
let zone_codes = ingest::extract_non_total_values(&metadata, ingest::DIM_ZONE);
tracing::info!(
species_count = species_codes.len(),
gear_count = gear_codes.len(),
zone_count = zone_codes.len(),
"Extracted non-TOTAL dimension values for leaf-only query"
);
let all_months = ingest::extract_available_months(&metadata);
if full {
conn.execute("DELETE FROM landings", [])?;
tracing::info!("Cleared landings table for full re-ingest");
}
let months_to_fetch: Vec<String> = if full {
all_months
} else {
let last = db::get_last_month(&conn)?;
match last {
Some(last_month) => all_months
.into_iter()
.filter(|m| m.as_str() > last_month.as_str())
.collect(),
None => all_months,
}
};
if months_to_fetch.is_empty() {
let elapsed = run_start.elapsed();
tracing::info!(
mode = if full { "full" } else { "incremental" },
months_requested = 0,
total_rows = 0,
batches = 0,
errors = 0,
elapsed_ms = elapsed.as_millis() as u64,
"ingestion complete — no new months"
);
return Ok(());
}
const BATCH_SIZE: usize = 12;
let batches = ingest::chunk_months(&months_to_fetch, BATCH_SIZE);
tracing::info!(
mode = if full { "full" } else { "incremental" },
months_requested = months_to_fetch.len(),
batches = batches.len(),
"beginning batched ingestion"
);
let mut total_rows = 0;
let mut errors = 0;
for (i, batch) in batches.iter().enumerate() {
let batch_start = Instant::now();
tracing::info!(
batch = i + 1,
total_batches = batches.len(),
months_in_batch = batch.len(),
months = %batch.join(", "),
"fetching batch"
);
let query = ingest::build_query(batch, &species_codes, &gear_codes, &zone_codes)?;
let data = match ingest::fetch_data(&client, &config.data_source_url, &query).await {
Ok(d) => d,
Err(e) => {
tracing::error!(
batch = i + 1,
error = %e,
stage = "fetch_data",
"batch failed, continuing to next batch"
);
errors += 1;
continue;
}
};
let mut landings = Vec::with_capacity(data.dataset.value.len());
let mut parse_errors = 0;
for j in 0..data.dataset.value.len() {
match ingest::parse_row(j, &data.dataset, &lookup_maps) {
Ok(row) => {
if row.species_code == "TOTAL"
|| row.gear_code == "TOTAL"
|| row.zone_code == "TOTAL"
{
continue;
}
landings.push(ingest::data_row_to_landing(row));
}
Err(e) => {
tracing::warn!(
batch = i + 1,
row_index = j,
error = %e,
"failed to parse row"
);
parse_errors += 1;
errors += 1;
}
}
}
let count = match db::upsert_landings(&conn, &landings) {
Ok(c) => c,
Err(e) => {
tracing::error!(
batch = i + 1,
error = %e,
stage = "upsert_landings",
"batch upsert failed"
);
errors += 1;
continue;
}
};
total_rows += count;
let batch_elapsed = batch_start.elapsed();
tracing::info!(
batch = i + 1,
total_batches = batches.len(),
rows_inserted = count,
parse_errors = parse_errors,
elapsed_ms = batch_elapsed.as_millis() as u64,
"batch complete"
);
if i < batches.len() - 1 {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
let elapsed = run_start.elapsed();
tracing::info!(
mode = if full { "full" } else { "incremental" },
months_requested = months_to_fetch.len(),
batches = batches.len(),
total_rows = total_rows,
errors = errors,
elapsed_ms = elapsed.as_millis() as u64,
"ingestion run complete"
);
Ok(())
}
fn run_export(config: &types::Config, out: &Path) -> anyhow::Result<()> {
let run_start = Instant::now();
let conn = db::init(&config.duckdb_path)?;
let path_str = out
.to_str()
.ok_or_else(|| anyhow::anyhow!("Output path contains invalid UTF-8"))?;
db::export_parquet(&conn, path_str)?;
let elapsed = run_start.elapsed();
tracing::info!(
path = path_str,
elapsed_ms = elapsed.as_millis() as u64,
"export complete"
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::Config;
#[test]
fn test_config_default_values() {
let config = Config::default();
assert_eq!(config.duckdb_path, "hagfish.db");
assert_eq!(config.bind_address, "127.0.0.1:8090");
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
}
#[test]
fn test_run_export_writes_valid_parquet() {
let tmp_dir = tempfile::tempdir().unwrap();
let db_path = tmp_dir.path().join("test.db");
let parquet_path = tmp_dir.path().join("export.parquet");
let conn = db::init(db_path.to_str().unwrap()).unwrap();
let rows = vec![crate::types::Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "Toskur".to_string(),
gear_code: "TR1".to_string(),
zone_code: "FO".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(1000.0),
}];
db::upsert_landings(&conn, &rows).unwrap();
drop(conn);
let config = Config {
duckdb_path: db_path.to_str().unwrap().to_string(),
..Default::default()
};
let result = run_export(&config, &parquet_path);
assert!(result.is_ok());
let metadata = std::fs::metadata(&parquet_path).unwrap();
assert!(metadata.len() > 0);
let file_bytes = std::fs::read(&parquet_path).unwrap();
assert_eq!(&file_bytes[..4], b"PAR1");
}
}
+234 -54
View File
@@ -1,18 +1,19 @@
//! Type definitions for hagfish data structures.
//!
//! These types represent the PX-Web API contract and internal data models.
//! All public types derive Serialize/Deserialize for JSON (de)serialization.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::Path;
/// Configuration loaded from config.json on startup.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Config {
pub duckdb_path: String,
pub bind_address: String,
pub data_source_url: String,
pub log_file_path: Option<String>,
#[serde(default = "default_allowed_origins")]
pub allowed_origins: Vec<String>,
}
fn default_allowed_origins() -> Vec<String> {
vec![]
}
impl Default for Config {
@@ -20,92 +21,128 @@ impl Default for Config {
Self {
duckdb_path: "hagfish.db".to_string(),
bind_address: "127.0.0.1:8090".to_string(),
data_source_url: "https://statbank.hagstova.fo/data/api/table/fisknv_md".to_string(),
data_source_url: "https://statbank.hagstova.fo/api/v1/fo/H2/VV/VV01/fisknv_md.px"
.to_string(),
log_file_path: Some("hagfish.log".to_string()),
allowed_origins: default_allowed_origins(),
}
}
}
/// Metadata response from PX-Web API GET request.
/// Contains variable definitions and value codes with labels.
impl Config {
pub fn load(path: &Path) -> anyhow::Result<Self> {
let contents = std::fs::read_to_string(path)
.map_err(|e| anyhow::anyhow!("Failed to read config file {}: {e}", path.display()))?;
let config: Config = serde_json::from_str(&contents)
.map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {e}", path.display()))?;
tracing::info!("Loaded config from {}", path.display());
Ok(config)
}
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct MetadataResponse {
#[serde(default)]
pub title: Option<String>,
pub variables: Vec<VariableMeta>,
pub values: HashMap<String, Vec<ValueMeta>>,
}
/// Variable metadata describing a dimension in the PX-Web table.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AvailableFiltersDto {
pub species: Vec<String>,
pub zones: Vec<String>,
pub gear: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct VariableMeta {
pub id: String,
pub text: String,
pub role: String,
/// Position in the key array (0-indexed)
#[serde(rename = "keyPosition", skip_serializing_if = "Option::is_none")]
pub key_position: Option<usize>,
}
/// Value metadata: code → display label mapping for a variable.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ValueMeta {
pub code: String,
#[serde(default)]
pub text: String,
#[serde(default)]
pub role: Option<String>,
#[serde(default)]
pub values: Vec<String>,
#[serde(default, rename = "valueTexts")]
pub value_texts: Vec<String>,
#[serde(flatten)]
pub extra: HashMap<String, serde_json::Value>,
}
impl VariableMeta {
pub fn lookup_map(&self) -> HashMap<String, String> {
self.values
.iter()
.zip(self.value_texts.iter())
.map(|(code, label)| (code.clone(), label.clone()))
.collect()
}
}
/// Query body sent to PX-Web API POST endpoint.
#[derive(Debug, Clone, Serialize)]
pub struct Query {
pub query: Vec<QueryItem>,
pub language: String,
pub response: QueryResponse,
}
/// Single query item representing a dimension selection.
#[derive(Debug, Clone, Serialize)]
pub struct QueryItem {
pub id: String,
pub code: String,
pub selection: Selection,
}
#[derive(Debug, Clone, Serialize)]
pub struct Selection {
pub filter: String,
pub values: Vec<String>,
}
/// Selection helper for building queries programmatically.
#[derive(Debug, Clone)]
pub struct Selection {
/// Dimension name (matches variable.id from metadata)
pub dimension: String,
/// Codes to include, or "*" for all
pub codes: Vec<String>,
#[derive(Debug, Clone, Serialize)]
pub struct QueryResponse {
pub format: String,
}
impl Default for QueryResponse {
fn default() -> Self {
Self {
format: "json-stat2".to_string(),
}
}
}
/// Raw data response from PX-Web API POST request.
/// JSON-stat2 format with metadata and data sections.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct DataResponse {
#[serde(flatten)]
pub dataset: Dataset,
}
/// Dataset wrapper containing dimensions and actual values.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Dataset {
pub dimension: DimInfo,
#[serde(default)]
pub id: Vec<String>,
#[serde(default)]
pub size: Vec<usize>,
#[serde(flatten)]
pub extra_fields: HashMap<String, serde_json::Value>,
pub dimension: HashMap<String, Dimension>,
pub value: Vec<Option<f64>>,
}
/// Dimension metadata describing key layout.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct DimInfo {
#[serde(rename = "key")]
pub keys: Vec<String>,
pub struct Dimension {
#[serde(default)]
pub label: String,
pub category: CategoryInfo,
}
/// Category info containing value lists for each dimension.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct CategoryInfo {
pub label: HashMap<String, Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub index: Option<HashMap<String, Vec<usize>>>,
pub index: HashMap<String, usize>,
#[serde(default)]
pub label: HashMap<String, String>,
}
/// Decoded row of landing data with labeled dimensions.
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct DataRow {
pub month: String,
@@ -126,7 +163,6 @@ pub struct DataRow {
pub value: Option<f64>,
}
/// Structured representation of a single landing record.
#[derive(Debug, Clone)]
pub struct Landing {
pub month: String,
@@ -141,7 +177,86 @@ pub struct Landing {
pub value: Option<f64>,
}
/// Error types for ingestion module.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpeciesDto {
pub code: String,
pub label: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LookupDto {
pub code: String,
pub label: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LandingDto {
pub month: String,
pub species_code: String,
pub species_label: String,
pub gear_code: String,
pub zone_code: String,
pub processing_code: String,
pub preservation_code: String,
pub shipsize_code: String,
pub measure_code: String,
pub value: Option<f64>,
}
impl From<Landing> for LandingDto {
fn from(l: Landing) -> Self {
Self {
month: l.month,
species_code: l.species_code,
species_label: l.species_label,
gear_code: l.gear_code,
zone_code: l.zone_code,
processing_code: l.processing_code,
preservation_code: l.preservation_code,
shipsize_code: l.shipsize_code,
measure_code: l.measure_code,
value: l.value,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MonthlyAggregate {
pub month: String,
pub total_mass: Option<f64>,
pub total_value: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TopSpecies {
pub species_code: String,
pub species_label: String,
pub total_value: Option<f64>,
pub total_mass: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PriceTrend {
pub month: String,
pub price_per_kg: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SummaryDto {
pub monthly: Vec<MonthlyAggregate>,
pub top_species: Vec<TopSpecies>,
pub price_trend: Vec<PriceTrend>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MonthlySpeciesBreakdown {
pub month: String,
pub species_code: String,
pub species_label: String,
pub total_value: Option<f64>,
pub total_mass: Option<f64>,
}
#[derive(Debug, thiserror::Error)]
pub enum IngestError {
#[error("HTTP request failed: {0}")]
@@ -150,6 +265,9 @@ pub enum IngestError {
#[error("JSON parsing failed: {0}")]
JsonError(#[from] serde_json::Error),
#[error("API returned status {status}: {body}")]
ApiStatus { status: u16, body: String },
#[error("Missing dimension in response: {0}")]
MissingDimension(String),
@@ -158,14 +276,76 @@ pub enum IngestError {
#[error("API returned empty data set")]
EmptyDataset,
#[error("Unicode decode error: {0}")]
UnicodeError(String),
}
/// Lookup map for decoding key arrays into labeled values.
/// Keyed by dimension name, contains code → label mappings.
pub type LookupMap = HashMap<String, HashMap<String, String>>;
/// Result alias using custom error type.
pub type Result<T> = std::result::Result<T, IngestError>;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_config_load_missing_file_hard_error() {
let result = Config::load(Path::new("/nonexistent/path/config.json"));
assert!(result.is_err());
let err_msg = result.unwrap_err().to_string();
assert!(err_msg.contains("Failed to read config file"));
}
#[test]
fn test_config_load_invalid_json_error() {
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(tmp.path(), "not valid json {{{").unwrap();
let result = Config::load(tmp.path());
assert!(result.is_err());
let err_msg = result.unwrap_err().to_string();
assert!(err_msg.contains("Failed to parse config file"));
}
#[test]
fn test_config_load_valid_file() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let content = r#"{
"duckdb_path": "test.db",
"bind_address": "127.0.0.1:9090",
"data_source_url": "http://example.com",
"log_file_path": null,
"allowed_origins": ["https://example.com"]
}"#;
std::fs::write(tmp.path(), content).unwrap();
let config = Config::load(tmp.path()).unwrap();
assert_eq!(config.duckdb_path, "test.db");
assert_eq!(config.bind_address, "127.0.0.1:9090");
assert_eq!(config.data_source_url, "http://example.com");
assert!(config.log_file_path.is_none());
assert_eq!(config.allowed_origins, vec!["https://example.com"]);
}
#[test]
fn test_config_load_defaults_allowed_origins() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let content = r#"{
"duckdb_path": "test.db",
"bind_address": "127.0.0.1:9090",
"data_source_url": "http://example.com"
}"#;
std::fs::write(tmp.path(), content).unwrap();
let config = Config::load(tmp.path()).unwrap();
assert!(config.allowed_origins.is_empty());
}
#[test]
fn test_config_default_values() {
let config = Config::default();
assert_eq!(config.duckdb_path, "hagfish.db");
assert_eq!(config.bind_address, "127.0.0.1:8090");
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
assert!(config.log_file_path.is_some());
assert!(config.allowed_origins.is_empty());
}
}
+857
View File
@@ -0,0 +1,857 @@
let chartLine, chartPrice, chartBar;
let allMonths = [];
const MONTHS_TO_SHOW_DEFAULT = 6;
function formatMonthDisplay(monthCode) {
const monthNames = [
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
];
const year = monthCode.substring(0, 4);
const monthIdx = parseInt(monthCode.substring(5, 7)) - 1;
return `${monthNames[monthIdx]} ${year}`;
}
function formatDKK(value) {
return new Intl.NumberFormat('da-DK', {
minimumFractionDigits: 0,
maximumFractionDigits: 0,
}).format(value) + ' DKK';
}
function formatDKKShort(value) {
if (value >= 1e9) return (value / 1e9).toFixed(1).replace('.', ',') + ' mia.';
if (value >= 1e6) return (value / 1e6).toFixed(1).replace('.', ',') + ' mio.';
if (value >= 1e3) return (value / 1e3).toFixed(0) + ' tus.';
return value.toString();
}
function formatPricePerKg(value) {
return new Intl.NumberFormat('da-DK', {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
}).format(value) + ' DKK/kg';
}
function getNextUpdateDate() {
const now = new Date();
const currentMonth = now.getMonth();
const currentYear = now.getFullYear();
const today = now.getDate();
let updateMonth = currentMonth;
let updateYear = currentYear;
if (today > 28) {
updateMonth++;
if (updateMonth > 11) {
updateMonth = 0;
updateYear++;
}
}
const updateDate = new Date(updateYear, updateMonth, 28);
const yyyy = updateDate.getFullYear();
const mm = String(updateDate.getMonth() + 1).padStart(2, '0');
const dd = String(updateDate.getDate()).padStart(2, '0');
return `${yyyy}-${mm}-${dd}`;
}
function updateNavbarUpdateDate() {
const container = document.getElementById('update-date-display');
if (container) {
container.textContent = `Next update: ${getNextUpdateDate()}`;
}
}
function getThemeColors() {
const theme = document.documentElement.getAttribute('data-theme');
const isDark = theme === 'macchiato';
if (isDark) {
return {
isDark: true,
textColor: '#cdd6f4',
subTextColor: '#bac2de',
gridColor: 'rgba(180, 185, 200, 0.12)',
accent: '#89b4fa',
secondary: '#94e2d5',
palette: [
'#89b4fa',
'#fab387',
'#a6e3a1',
'#f9e2af',
'#f5c2e7',
'#cba6f7',
'#f38ba8',
'#eba0ac',
'#94e2d5',
'#89dceb',
],
tooltipBg: 'rgba(30, 32, 48, 0.95)',
tooltipBorder: 'rgba(180, 185, 200, 0.2)',
bgTransparent: 'transparent',
};
}
return {
isDark: false,
textColor: '#45475a',
subTextColor: '#7f849c',
gridColor: 'rgba(73, 77, 113, 0.08)',
accent: '#89b4fa',
secondary: '#94e2d5',
palette: [
'#89b4fa',
'#fab387',
'#a6e3a1',
'#f9e2af',
'#f5c2e7',
'#cba6f7',
'#f38ba8',
'#eba0ac',
'#94e2d5',
'#89dceb',
],
tooltipBg: 'rgba(255, 255, 255, 0.98)',
tooltipBorder: 'rgba(73, 77, 113, 0.15)',
bgTransparent: 'transparent',
};
}
function initCharts() {
if (chartLine) chartLine.dispose();
if (chartPrice) chartPrice.dispose();
if (chartBar) chartBar.dispose();
chartLine = echarts.init(document.getElementById('chart-line'));
chartPrice = echarts.init(document.getElementById('chart-price'));
chartBar = echarts.init(document.getElementById('chart-bar'));
chartLine.setOption({ series: [] });
chartPrice.setOption({ series: [] });
chartBar.setOption({ series: [] });
setTimeout(() => {
chartLine.resize();
chartPrice.resize();
chartBar.resize();
}, 0);
window.addEventListener('resize', () => {
chartLine.resize();
chartPrice.resize();
chartBar.resize();
});
}
function renderLineChart(months, massData, valueData) {
const c = getThemeColors();
const formattedMonths = months.map(m => formatMonthDisplay(m));
const step = months.length > 24 ? 6 : (months.length > 12 ? 3 : 1);
const xLabels = formattedMonths.map((label, idx) =>
idx % step === 0 ? label : ''
);
chartLine.setOption({
backgroundColor: c.bgTransparent,
textStyle: { fontFamily: 'Noto Sans', color: c.textColor },
tooltip: {
trigger: 'axis',
axisPointer: {
type: 'cross',
lineStyle: { color: c.gridColor, type: 'dashed' },
crossStyle: { color: c.gridColor },
},
backgroundColor: c.tooltipBg,
borderColor: c.tooltipBorder,
borderWidth: 1,
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
},
legend: {
data: ['Mass (kg)', 'Value (DKK)'],
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
top: 0,
left: 'center',
orient: 'horizontal',
},
grid: {
left: '8%',
right: '10%',
bottom: 50,
containLabel: true,
top: '18%',
},
xAxis: {
type: 'category',
data: xLabels,
axisLine: { lineStyle: { color: c.gridColor } },
axisLabel: {
color: c.subTextColor,
interval: 0,
rotate: 0,
fontSize: 10,
margin: 8,
overflow: 'break',
width: 70,
},
boundaryGap: false,
},
yAxis: [
{
type: 'value',
name: 'Mass (kg)',
position: 'left',
splitLine: { lineStyle: { color: c.gridColor } },
axisLabel: {
color: c.subTextColor,
formatter: (val) => (val >= 1e6 ? (val / 1e6).toFixed(1) + 'M' : val.toLocaleString()),
margin: 8,
},
axisLine: { show: true, lineStyle: { color: c.gridColor } },
nameTextStyle: { color: c.accent, padding: [0, 0, 0, 0] },
},
{
type: 'value',
name: 'Value (DKK)',
position: 'right',
splitLine: { show: false },
axisLabel: {
color: c.subTextColor,
formatter: (val) => (val >= 1e6 ? (val / 1e6).toFixed(1) + 'M' : val.toLocaleString()),
margin: 8,
},
axisLine: { show: true, lineStyle: { color: c.gridColor } },
nameTextStyle: { color: c.secondary, padding: [0, 10, 0, 0] },
},
],
series: [
{
name: 'Mass (kg)',
type: 'line',
yAxisIndex: 0,
smooth: true,
symbol: 'none',
lineStyle: { width: 2, opacity: 0.9 },
itemStyle: { color: c.accent },
areaStyle: {
color: {
type: 'linear',
x: 0, y: 0, x2: 0, y2: 1,
colorStops: [
{ offset: 0, color: c.isDark ? 'rgba(137, 180, 250, 0.22)' : 'rgba(137, 180, 250, 0.18)' },
{ offset: 1, color: 'rgba(137, 180, 250, 0)' },
],
},
},
data: massData,
},
{
name: 'Value (DKK)',
type: 'line',
yAxisIndex: 1,
smooth: true,
symbol: 'none',
lineStyle: { width: 2, opacity: 0.9 },
itemStyle: { color: c.secondary },
areaStyle: {
color: {
type: 'linear',
x: 0, y: 0, x2: 0, y2: 1,
colorStops: [
{ offset: 0, color: c.isDark ? 'rgba(148, 226, 213, 0.22)' : 'rgba(148, 226, 213, 0.18)' },
{ offset: 1, color: 'rgba(148, 226, 213, 0)' },
],
},
},
data: valueData,
},
],
});
}
function renderPriceChart(priceTrend) {
const c = getThemeColors();
chartPrice.setOption({
backgroundColor: c.bgTransparent,
textStyle: { fontFamily: 'Noto Sans', color: c.textColor },
tooltip: {
trigger: 'axis',
axisPointer: { type: 'cross' },
backgroundColor: c.tooltipBg,
borderColor: c.tooltipBorder,
borderWidth: 1,
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
formatter: (params) => {
const p = params[0];
return `<strong>${p.axisValue}</strong><br/>${formatPricePerKg(p.value)}`;
},
},
grid: {
left: '8%',
right: '10%',
bottom: 50,
containLabel: true,
top: '18%',
},
xAxis: {
type: 'category',
data: priceTrend.map(d => formatMonthDisplay(d.month)),
axisLine: { lineStyle: { color: c.gridColor } },
axisLabel: {
color: c.subTextColor,
interval: priceTrend.length > 24 ? 6 : (priceTrend.length > 12 ? 3 : 1),
fontSize: 10,
rotate: 0,
margin: 8,
},
},
yAxis: {
type: 'value',
name: 'Price per kg (DKK)',
splitLine: { lineStyle: { color: c.gridColor, type: 'dashed' } },
axisLabel: {
color: c.subTextColor,
formatter: (val) => formatPricePerKg(val),
margin: 8,
},
axisLine: { show: true, lineStyle: { color: c.gridColor } },
nameTextStyle: { color: c.accent, padding: [0, 0, 0, 0] },
},
series: [
{
name: 'Price per kg',
type: 'line',
smooth: true,
symbol: 'none',
lineStyle: { width: 3 },
itemStyle: { color: c.secondary },
areaStyle: {
color: {
type: 'linear',
x: 0, y: 0, x2: 0, y2: 1,
colorStops: [
{ offset: 0, color: c.isDark ? 'rgba(148, 226, 213, 0.3)' : 'rgba(148, 226, 213, 0.2)' },
{ offset: 1, color: 'rgba(148, 226, 213, 0)' },
],
},
},
data: priceTrend.map(d => d.price_per_kg),
},
],
});
}
function renderBarChart(months, breakdown, showOthers) {
const c = getThemeColors();
if (months.length === 0 || breakdown.length === 0) {
chartBar.setOption({ series: [] });
return;
}
const formattedMonths = months.map(m => formatMonthDisplay(m));
const step = months.length > 36 ? 12 : (months.length > 18 ? 6 : (months.length > 12 ? 3 : 1));
const xLabels = formattedMonths.map((label, idx) =>
idx % step === 0 ? label : ''
);
const speciesMonthlyMap = {};
const speciesLabels = {};
const speciesTotals = {};
breakdown.forEach(row => {
if (!speciesMonthlyMap[row.species_code]) {
speciesMonthlyMap[row.species_code] = {};
speciesLabels[row.species_code] = row.species_label || row.species_code;
speciesTotals[row.species_code] = 0;
}
speciesMonthlyMap[row.species_code][row.month] = row.total_value || 0;
speciesTotals[row.species_code] += row.total_value || 0;
});
const rankedSpecies = Object.entries(speciesTotals)
.sort((a, b) => b[1] - a[1])
.slice(0, 5)
.map(([code]) => code);
const otherValues = months.map(m => {
return Object.keys(speciesMonthlyMap)
.filter(code => !rankedSpecies.includes(code))
.reduce((sum, code) => sum + (speciesMonthlyMap[code][m] || 0), 0);
});
let seriesData = rankedSpecies.map(code => ({
name: speciesLabels[code],
type: 'bar',
stack: 'total',
emphasis: { focus: 'series' },
data: months.map(m => speciesMonthlyMap[code][m] || 0),
}));
if (showOthers) {
seriesData.push({
name: 'Other species',
type: 'bar',
stack: 'total',
emphasis: { focus: 'series' },
data: otherValues,
});
}
const chartWidth = chartBar.getWidth();
const barMaxWidth = Math.max(15, Math.floor((chartWidth * 0.7) / (months.length * 1.5)));
chartBar.setOption({
backgroundColor: c.bgTransparent,
textStyle: { fontFamily: 'Noto Sans', color: c.textColor },
tooltip: {
trigger: 'axis',
axisPointer: { type: 'shadow' },
backgroundColor: c.tooltipBg,
borderColor: c.tooltipBorder,
borderWidth: 1,
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
formatter: (params) => {
const month = params[0]?.axisValue || '';
let html = `<strong>${month}</strong><br/>`;
[...params].reverse().forEach(p => {
const val = p.value || 0;
if (val > 0) {
html += `${p.marker} ${p.seriesName}: ${formatDKK(val)}<br/>`;
}
});
return html;
},
},
legend: {
textStyle: { color: c.textColor, fontFamily: 'Noto Sans' },
top: 0,
left: 'center',
orient: 'horizontal',
type: 'scroll',
maxItems: 8,
padding: [0, 0, 0, 0],
itemGap: 10,
},
grid: {
left: '8%',
right: '5%',
bottom: 65,
containLabel: true,
top: '25%',
},
xAxis: {
type: 'category',
data: xLabels,
axisLine: { lineStyle: { color: c.gridColor } },
axisLabel: {
color: c.subTextColor,
interval: 0,
fontSize: 9,
rotate: 0,
margin: 8,
overflow: 'break',
width: 60,
},
axisTick: { show: false },
},
yAxis: {
type: 'value',
splitLine: { lineStyle: { color: c.gridColor, type: 'dashed' } },
axisLabel: {
color: c.subTextColor,
formatter: (val) => formatDKKShort(val),
margin: 8,
},
nameTextStyle: { padding: [0, 0, 0, 10] },
},
series: seriesData.map((s, idx) => ({
...s,
barMaxWidth: barMaxWidth,
itemStyle: {
color: c.palette[idx % c.palette.length],
},
label: {
show: false,
},
emphasis: {
itemStyle: {
shadowBlur: 0,
},
},
})),
});
}
async function fetchData(endpoint) {
try {
const res = await fetch(endpoint);
if (!res.ok) throw new Error(`API Error: ${res.status}`);
return await res.json();
} catch (err) {
console.error(err);
return null;
}
}
function populateDropdown(containerId, items, codeKey, labelKey) {
const container = document.getElementById(containerId);
if (!container) return;
const checkboxList = container.querySelector('.checkbox-dropdown-list');
checkboxList.innerHTML = '';
const selectAllLabel = document.createElement('label');
selectAllLabel.className = 'checkbox-item select-all-wrapper';
selectAllLabel.innerHTML = `
<input type="checkbox" class="select-all" data-target="${containerId}" checked>
<span>All</span>
`;
checkboxList.appendChild(selectAllLabel);
const separator = document.createElement('div');
separator.className = 'dropdown-separator';
checkboxList.appendChild(separator);
items.forEach((item) => {
const label = document.createElement('label');
label.className = 'checkbox-item';
label.innerHTML = `
<input type="checkbox"
name="${containerId}"
value="${item[codeKey]}"
data-label="${item[labelKey]}"
checked>
<span>${item[labelKey]}</span>
`;
checkboxList.appendChild(label);
});
const selectAllCb = selectAllLabel.querySelector('.select-all');
selectAllCb.addEventListener('change', (e) => {
const cbs = checkboxList.querySelectorAll(`input[name="${containerId}"]`);
cbs.forEach(cb => {
if (!cb.disabled) {
cb.checked = e.target.checked;
}
});
updateDropdownLabel(containerId);
});
checkboxList.querySelectorAll(`input[name="${containerId}"]`).forEach(cb => {
cb.addEventListener('change', () => updateDropdownLabel(containerId));
});
updateDropdownLabel(containerId);
}
function updateDropdownLabel(containerId) {
const container = document.getElementById(containerId);
if (!container) return;
const cbs = container.querySelectorAll(`input[name="${containerId}"]`);
const checked = Array.from(cbs).filter(cb => cb.checked && !cb.disabled);
const labelEl = container.querySelector('.dropdown-label');
if (checked.length === 0) {
labelEl.textContent = 'None selected';
} else if (checked.length === cbs.length) {
labelEl.textContent = 'All selected';
} else {
labelEl.textContent = `${checked.length} selected`;
}
}
function toggleDropdown(containerId) {
const container = document.getElementById(containerId);
if (!container) return;
const list = container.querySelector('.checkbox-dropdown-list');
const isOpen = list.classList.contains('open');
document.querySelectorAll('.checkbox-dropdown-list.open').forEach(el => {
el.classList.remove('open');
});
if (!isOpen) {
list.classList.add('open');
}
}
function getSelectedValues(groupName) {
const checkboxes = document.querySelectorAll(`input[name="${groupName}"]:checked`);
return Array.from(checkboxes).filter(cb => !cb.disabled).map(cb => cb.value);
}
function populateMonthSelect(selectId, months) {
const sel = document.getElementById(selectId);
sel.innerHTML = '<option value="">All</option>';
months.forEach((m) => {
const opt = document.createElement('option');
opt.value = m;
opt.textContent = formatMonthDisplay(m);
sel.appendChild(opt);
});
}
async function populateDropdowns() {
showLoading(true);
const [speciesList, zonesList, gearList, summaryData] = await Promise.all([
fetchData('/api/species'),
fetchData('/api/zones'),
fetchData('/api/gear'),
fetchData('/api/summary'),
]);
if (speciesList) {
populateDropdown('species', speciesList, 'code', 'label');
}
if (zonesList) {
populateDropdown('zone', zonesList, 'code', 'label');
}
if (gearList) {
populateDropdown('gear', gearList, 'code', 'label');
}
if (summaryData && summaryData.monthly) {
allMonths = summaryData.monthly.map((m) => m.month).sort();
populateMonthSelect('month_from', allMonths);
populateMonthSelect('month_to', allMonths);
const fromIdx = Math.max(0, allMonths.length - MONTHS_TO_SHOW_DEFAULT);
document.getElementById('month_from').value = allMonths[fromIdx];
document.getElementById('month_to').value = allMonths[allMonths.length - 1];
}
await updateAvailableFilters();
showLoading(false);
}
async function updateAvailableFilters() {
const species = getSelectedValues('species');
const zone = getSelectedValues('zone');
const gear = getSelectedValues('gear');
const monthFrom = document.getElementById('month_from').value;
const monthTo = document.getElementById('month_to').value;
const params = new URLSearchParams();
if (species.length > 0) params.append('species_multi', species.join(','));
if (zone.length > 0) params.append('zone_multi', zone.join(','));
if (gear.length > 0) params.append('gear_multi', gear.join(','));
if (monthFrom) params.append('month_from', monthFrom);
if (monthTo) params.append('month_to', monthTo);
const available = await fetchData(`/api/available-filters?${params.toString()}`);
if (available) {
disableUnavailableOptions('species', available.species);
disableUnavailableOptions('zone', available.zones);
disableUnavailableOptions('gear', available.gear);
}
}
function disableUnavailableOptions(containerId, availableCodes) {
const container = document.getElementById(containerId);
if (!container) return;
const checkboxes = container.querySelectorAll(`input[type="checkbox"]`);
checkboxes.forEach(cb => {
if (cb.classList.contains('select-all')) return;
const isAvailable = availableCodes.includes(cb.value);
cb.disabled = !isAvailable;
cb.parentElement.style.opacity = isAvailable ? '1' : '0.4';
cb.parentElement.style.cursor = isAvailable ? 'pointer' : 'not-allowed';
if (!isAvailable && cb.checked) {
cb.checked = false;
}
});
updateDropdownLabel(containerId);
}
async function updateCharts() {
showLoading(true);
hideEmptyState();
const species = getSelectedValues('species');
const zone = getSelectedValues('zone');
const gear = getSelectedValues('gear');
const monthFrom = document.getElementById('month_from').value;
const monthTo = document.getElementById('month_to').value;
if (monthFrom && monthTo && monthFrom > monthTo) {
alert('From date cannot be greater than To date.');
showLoading(false);
return;
}
const summaryParams = new URLSearchParams();
if (species.length > 0) summaryParams.append('species_multi', species.join(','));
if (zone.length > 0) summaryParams.append('zone_multi', zone.join(','));
if (gear.length > 0) summaryParams.append('gear_multi', gear.join(','));
if (monthFrom) summaryParams.append('month_from', monthFrom);
if (monthTo) summaryParams.append('month_to', monthTo);
const [summary, monthlyBreakdown] = await Promise.all([
fetchData(`/api/summary?${summaryParams.toString()}`),
fetchData(`/api/summary/monthly-breakdown?${summaryParams.toString()}`),
]);
if (!summary || !summary.monthly || summary.monthly.length === 0) {
showLoading(false);
showEmptyState();
return;
}
const months = summary.monthly.map(m => m.month).sort();
const showAllSpecies = document.getElementById('show-all-species')?.checked ?? false;
if (monthlyBreakdown && monthlyBreakdown.length > 0) {
renderBarChart(months, monthlyBreakdown, showAllSpecies);
} else {
renderBarChart([], [], false);
}
const massData = summary.monthly.map((m) => m.total_mass || 0);
const valueData = summary.monthly.map((m) => m.total_value || 0);
renderLineChart(months, massData, valueData);
const priceTrend = summary.price_trend ?? [];
if (priceTrend.length > 0) {
renderPriceChart(priceTrend);
} else {
chartPrice.setOption({ series: [] });
}
showLoading(false);
hideEmptyState();
document.getElementById('charts-grid').style.display = 'grid';
chartLine.resize();
chartPrice.resize();
chartBar.resize();
renderLineChart(months, massData, valueData);
if (priceTrend.length > 0) {
renderPriceChart(priceTrend);
} else {
chartPrice.setOption({ series: [] });
}
if (monthlyBreakdown && monthlyBreakdown.length > 0) {
renderBarChart(months, monthlyBreakdown, showAllSpecies);
} else {
renderBarChart([], [], false);
}
setTimeout(() => {
chartLine.resize();
chartPrice.resize();
chartBar.resize();
}, 50);
}
function showLoading(show) {
const overlay = document.getElementById('loading-overlay');
if (show) {
overlay.classList.remove('hidden');
} else {
overlay.classList.add('hidden');
}
}
function showEmptyState() {
document.getElementById('charts-grid').style.display = 'none';
document.getElementById('empty-state').style.display = 'flex';
}
function hideEmptyState() {
document.getElementById('charts-grid').style.display = 'grid';
document.getElementById('empty-state').style.display = 'none';
}
document.getElementById('apply-filters').addEventListener('click', updateCharts);
document.getElementById('show-all-species')?.addEventListener('change', updateCharts);
document.addEventListener('click', (e) => {
if (!e.target.closest('.filter-group.multi-select')) {
document.querySelectorAll('.checkbox-dropdown-list.open').forEach(el => {
el.classList.remove('open');
});
}
});
(() => {
function currentSystemTheme() {
return window.matchMedia('(prefers-color-scheme: dark)').matches
? 'macchiato'
: 'light';
}
const btn = document.getElementById('theme-toggle');
if (btn) {
btn.addEventListener('click', () => {
const current =
document.documentElement.getAttribute('data-theme') || 'light';
const next = current === 'light' ? 'macchiato' : 'light';
document.documentElement.setAttribute('data-theme', next);
localStorage.setItem('poc-theme', next);
initCharts();
updateCharts();
});
}
window
.matchMedia('(prefers-color-scheme: dark)')
.addEventListener('change', () => {
if (!localStorage.getItem('poc-theme')) {
document.documentElement.setAttribute('data-theme', currentSystemTheme());
initCharts();
updateCharts();
}
});
})();
document.addEventListener('DOMContentLoaded', () => {
initCharts();
updateNavbarUpdateDate();
['species', 'zone', 'gear'].forEach(id => {
const container = document.getElementById(id);
if (container) {
const label = container.querySelector('.dropdown-label');
if (label) {
label.addEventListener('click', (e) => {
e.stopPropagation();
toggleDropdown(id);
});
}
const list = container.querySelector('.checkbox-dropdown-list');
if (list) {
list.addEventListener('change', () => {
updateAvailableFilters();
});
}
}
});
document.getElementById('month_from').addEventListener('change', updateAvailableFilters);
document.getElementById('month_to').addEventListener('change', updateAvailableFilters);
populateDropdowns().then(updateCharts);
});
+117
View File
@@ -0,0 +1,117 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, viewport-fit=cover" />
<title>Fisheries Dashboard — FLÓ</title>
<script>
(function () {
var stored = localStorage.getItem('poc-theme');
var theme = stored || (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'macchiato' : 'light');
document.documentElement.setAttribute('data-theme', theme);
})();
</script>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Noto+Sans:wght@300;400;500;700&display=swap" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/echarts@5.4.3/dist/echarts.min.js"></script>
<link rel="stylesheet" href="/static/style.css">
</head>
<body>
<nav>
<a href="https://fló.fo" class="back-link">← FLÓ.FO</a>
<a href="/" class="nav-brand">
FLÓ <span class="brand-sep">/</span> <span class="brand-sub">HAGFISH</span>
</a>
<div class="update-date-display" id="update-date-display"></div>
</nav>
<main class="page-body">
<div class="sidebar">
<div class="filters-panel">
<div class="filters">
<div class="filter-group multi-select" id="species">
<label>Species</label>
<div class="dropdown-trigger dropdown-label">All</div>
<div class="checkbox-dropdown-list"></div>
</div>
<div class="filter-group multi-select" id="zone">
<label>Zone</label>
<div class="dropdown-trigger dropdown-label">All</div>
<div class="checkbox-dropdown-list"></div>
</div>
<div class="filter-group multi-select" id="gear">
<label>Gear</label>
<div class="dropdown-trigger dropdown-label">All</div>
<div class="checkbox-dropdown-list"></div>
</div>
<div class="filter-group">
<label for="month_from">From</label>
<select id="month_from"><option value="">All</option></select>
</div>
<div class="filter-group">
<label for="month_to">To</label>
<select id="month_to"><option value="">All</option></select>
</div>
<div class="filter-group">
<label class="toggle-label">
<input type="checkbox" id="show-all-species">
<span>Show "Other species" bucket</span>
</label>
</div>
<button class="apply-btn" id="apply-filters">Apply</button>
</div>
</div>
</div>
<div class="dashboard-wrapper">
<div id="loading-overlay" class="loading-overlay">
<span>Loading fisheries data…</span>
</div>
<div class="dashboard">
<div id="empty-state" class="empty-state" style="display:none;">
<p>No data available for the selected filters.</p>
<p>Try adjusting your filters or running an ingestion.</p>
</div>
<div id="charts-grid" class="charts-grid" style="display:none;">
<div class="chart-container full-width">
<div class="chart-title">Total Landings (Mass &amp; Value)</div>
<div id="chart-line" style="width:100%; height:420px;"></div>
</div>
<div class="chart-container">
<div class="chart-title">Price per kg Trend</div>
<div id="chart-price" style="width:100%; height:380px;"></div>
</div>
<div class="chart-container">
<div class="chart-title">Monthly Value by Top Species</div>
<div id="chart-bar" style="width:100%; height:380px;"></div>
</div>
</div>
</div>
</div>
</main>
<button class="theme-toggle" id="theme-toggle" aria-label="Toggle theme"></button>
<footer>
<span>&copy; <script>document.write(new Date().getFullYear())</script> FLÓ.FO | ALL RIGHTS RESERVED</span>
<a href="https://www.facebook.com/people/FL%C3%93/61593006613799/" target="_blank" rel="noopener" class="footer-icon" aria-label="Facebook">
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor"><path d="M24 12.073c0-6.627-5.373-12-12-12s-12 5.373-12 12c0 5.99 4.388 10.954 10.125 11.854v-8.385H7.078v-3.47h3.047V9.43c0-3.007 1.792-4.669 4.533-4.669 1.312 0 2.686.235 2.686.235v2.953H15.83c-1.491 0-1.956.925-1.956 1.874v2.25h3.328l-.532 3.47h-2.796v8.385C19.612 23.027 24 18.062 24 12.073z"/></svg>
</a>
<a href="https://git.flo.fo/FLO" target="_blank" rel="noopener" class="footer-icon" aria-label="Gitea">
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor"><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg>
</a>
<a href="https://www.linkedin.com/in/bartal-l%C3%A6arsson-63793895/" target="_blank" rel="noopener" class="footer-icon" aria-label="LinkedIn">
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor"><path d="M20.447 20.452h-3.554v-5.569c0-1.328-.027-3.037-1.852-3.037-1.853 0-2.136 1.445-2.136 2.939v5.667H9.351V9h3.414v1.561h.046c.477-.9 1.637-1.85 3.37-1.85 3.601 0 4.267 2.37 4.267 5.455v6.286zM5.337 7.433a2.062 2.062 0 01-2.063-2.065 2.064 2.064 0 112.063 2.065zm1.782 13.019H3.555V9h3.564v11.452zM22.225 0H1.771C.792 0 0 .774 0 1.729v20.542C0 23.227.792 24 1.771 24h20.451C23.2 24 24 23.227 24 22.271V1.729C24 .774 23.2 0 22.222 0h.003z"/></svg>
</a>
</footer>
<script src="/static/app.js"></script>
</body>
</html>
+528
View File
@@ -0,0 +1,528 @@
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root,
[data-theme="light"] {
--bg: #f4f1ec;
--bg-panel: #fafaf5;
--nav-bg: rgba(255, 253, 250, 0.92);
--ink: #1a1d22;
--ink-muted: #595f68;
--ink-dim: #9299a2;
--accent: #2c5a7a;
--border: rgba(26, 29, 34, 0.1);
--danger: #a03030;
--input-bg: #f0ede7;
--surface-hover: rgba(26, 29, 34, 0.04);
}
[data-theme="macchiato"] {
--bg: #181926;
--bg-panel: #1e2030;
--nav-bg: rgba(36, 39, 58, 0.65);
--ink: #cad3f5;
--ink-muted: #a5adcb;
--ink-dim: #8087a2;
--accent: #8aadf4;
--border: rgba(165, 173, 203, 0.15);
--danger: #ed8796;
--input-bg: #1e2030;
--surface-hover: rgba(138, 173, 244, 0.12);
}
[data-theme="macchiato"] .nav-brand {
color: #cad3f5;
text-shadow: 0 0 8px rgba(138, 173, 244, 0.25);
}
[data-theme="macchiato"] .apply-btn {
box-shadow: 0 0 10px rgba(138, 173, 244, 0.3);
}
[data-theme="macchiato"] .theme-toggle {
background: #24273a;
border-color: rgba(138, 173, 244, 0.4);
box-shadow:
0 0 10px rgba(138, 173, 244, 0.4),
0 0 25px rgba(138, 173, 244, 0.2);
animation: neon-breathe-toggle 4s ease-in-out infinite;
}
@keyframes neon-breathe-toggle {
0%, 100% {
box-shadow:
0 0 10px rgba(138, 173, 244, 0.5),
0 0 25px rgba(138, 173, 244, 0.3),
0 0 50px rgba(116, 150, 230, 0.2);
}
50% {
box-shadow:
0 0 15px rgba(138, 173, 244, 0.6),
0 0 35px rgba(138, 173, 244, 0.4),
0 0 70px rgba(116, 150, 230, 0.25);
}
}
html { scroll-behavior: smooth; }
body {
min-height: 100vh;
min-height: 100dvh;
background: var(--bg);
color: var(--ink);
font-family: 'Noto Sans', sans-serif;
font-size: 0.92rem;
font-weight: 300;
line-height: 1.65;
display: flex;
flex-direction: column;
transition: background 0.3s ease, color 0.3s ease;
}
a {
color: var(--accent);
text-decoration: none;
border-bottom: 1px solid transparent;
transition: all 0.2s ease;
}
a:hover { border-bottom-color: var(--accent); }
/* ---- Nav ---- */
nav {
position: fixed; top: 0; left: 0;
width: 100%; z-index: 20;
display: grid;
grid-template-columns: 1fr auto 1fr;
align-items: center;
gap: 1rem;
padding: 1.25rem 2.5rem;
background: var(--nav-bg);
backdrop-filter: blur(12px) saturate(120%);
-webkit-backdrop-filter: blur(12px) saturate(120%);
border-bottom: 1px solid var(--border);
transition: background 0.3s ease;
}
.back-link {
grid-column: 1;
justify-self: start;
font-family: 'Noto Sans', sans-serif;
font-weight: 400;
font-size: 0.7rem;
letter-spacing: 0.15em;
text-transform: uppercase;
white-space: nowrap;
color: var(--ink-muted);
text-decoration: none;
padding: 0.35rem 0.6rem;
border: 1px solid var(--border);
border-radius: 2px;
transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease;
}
.back-link:hover {
color: var(--ink);
border-color: var(--ink-muted);
background: var(--bg-panel);
border-bottom-color: var(--ink-muted);
}
.nav-brand {
grid-column: 2;
justify-self: center;
display: inline-flex;
align-items: baseline;
gap: 0.35rem;
white-space: nowrap;
font-family: 'Noto Sans', sans-serif;
font-weight: 700;
font-size: 1.1rem;
letter-spacing: -0.02em;
color: var(--ink);
text-decoration: none;
}
.nav-brand:hover { color: var(--accent); border-bottom: none; }
.update-date-display {
grid-column: 3;
justify-self: end;
font-family: 'Noto Sans', sans-serif;
font-weight: 400;
font-size: 0.72rem;
letter-spacing: 0.08em;
color: var(--ink-muted);
white-space: nowrap;
padding: 0.35rem 0.8rem;
border: 1px solid var(--border);
border-radius: 2px;
background: var(--input-bg);
}
.brand-sep { font-weight: 300; color: var(--ink-dim); }
.brand-sub {
font-weight: 400;
font-size: 0.78rem;
letter-spacing: 0.18em;
text-transform: uppercase;
color: var(--ink-muted);
align-self: center;
padding-top: 2px;
}
/* ---- Main ---- */
.page-body {
flex: 1;
display: flex;
padding: calc(65px + 2.5rem) 2rem 2rem;
gap: 2rem;
overflow: hidden;
}
.sidebar {
width: 320px;
flex-shrink: 0;
display: flex;
flex-direction: column;
height: 100%;
}
.filters-panel {
background: var(--bg-panel);
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
border: 1px solid var(--border);
border-radius: 3px;
padding: 1.5rem;
box-shadow: 0 8px 30px rgba(26, 29, 34, 0.06);
animation: panel-in 0.7s cubic-bezier(0.16, 1, 0.3, 1) forwards;
opacity: 0;
transform: translateX(-10px);
transition: background 0.3s ease, border-color 0.3s ease;
height: 100%;
}
@keyframes panel-in {
0% { opacity: 0; transform: translateX(-10px); }
100% { opacity: 1; transform: translateX(0); }
}
.dashboard-wrapper {
flex: 1;
display: flex;
min-width: 0;
height: 100%;
}
.dashboard {
width: 100%;
height: 100%;
background: var(--bg-panel);
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
border: 1px solid var(--border);
border-radius: 3px;
padding: 2rem;
box-shadow: 0 8px 30px rgba(26, 29, 34, 0.06);
animation: panel-in 0.7s cubic-bezier(0.16, 1, 0.3, 1) forwards;
opacity: 0;
transform: translateY(10px);
transition: background 0.3s ease, border-color 0.3s ease;
position: relative;
display: flex;
flex-direction: column;
}
/* ---- Loading Overlay ---- */
.loading-overlay {
position: absolute;
inset: 0;
background: var(--bg-panel);
backdrop-filter: blur(8px);
display: flex;
align-items: center;
justify-content: center;
z-index: 10;
border-radius: 3px;
color: var(--ink-muted);
font-size: 0.88rem;
letter-spacing: 0.05em;
}
.loading-overlay.hidden {
display: none;
}
/* ---- Empty State ---- */
.empty-state {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 0.5rem;
padding: 3rem 1rem;
color: var(--ink-dim);
font-size: 0.85rem;
text-align: center;
line-height: 1.6;
flex: 1;
}
/* ---- Filters ---- */
.filters {
display: flex;
flex-direction: column;
gap: 1rem;
height: 100%;
}
.filter-group {
display: flex;
flex-direction: column;
gap: 0.4rem;
position: relative;
}
.filter-group label {
font-size: 0.65rem;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.15em;
color: var(--ink-dim);
}
.filter-group select {
padding: 0.5rem 0.75rem;
background: var(--input-bg);
border: 1px solid var(--border);
border-radius: 2px;
color: var(--ink);
font-family: 'Noto Sans', sans-serif;
font-size: 0.82rem;
outline: none;
transition: border-color 0.2s;
cursor: pointer;
}
.filter-group select:focus { border-color: var(--accent); }
/* ---- Multi-Select Dropdown ---- */
.filter-group.multi-select {
position: relative;
}
.dropdown-trigger {
width: 100%;
padding: 0.5rem 0.75rem;
background: var(--input-bg);
border: 1px solid var(--border);
border-radius: 2px;
color: var(--ink);
font-family: 'Noto Sans', sans-serif;
font-size: 0.82rem;
cursor: pointer;
display: flex;
justify-content: space-between;
align-items: center;
transition: border-color 0.2s;
}
.dropdown-trigger:hover {
border-color: var(--accent);
}
.dropdown-trigger::after {
content: '▼';
font-size: 0.6rem;
margin-left: 0.5rem;
opacity: 0.6;
}
.checkbox-dropdown-list {
position: absolute;
top: 100%;
left: 0;
right: 0;
max-height: 200px;
overflow-y: auto;
background: var(--bg-panel);
border: 1px solid var(--border);
border-top: none;
border-radius: 0 0 2px 2px;
z-index: 100;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
display: none;
padding: 0.5rem 0;
}
.checkbox-dropdown-list.open {
display: block;
}
.dropdown-separator {
border-top: 1px solid var(--border);
margin: 0.25rem 0;
}
.checkbox-item {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.25rem 0.75rem;
font-size: 0.75rem;
cursor: pointer;
transition: background 0.1s;
}
.checkbox-item:hover {
background: var(--surface-hover);
}
.checkbox-item input[type="checkbox"] {
cursor: pointer;
accent-color: var(--accent);
margin-right: 0.5rem;
}
.select-all-wrapper {
font-weight: 600;
border-bottom: 1px solid var(--border);
margin-bottom: 0.25rem;
padding-bottom: 0.25rem;
}
.apply-btn {
width: 100%;
padding: 0.5rem 2rem;
background: var(--accent);
color: var(--bg-panel);
border: none;
border-radius: 2px;
font-family: 'Noto Sans', sans-serif;
font-weight: 700;
font-size: 0.82rem;
letter-spacing: 0.08em;
text-transform: uppercase;
cursor: pointer;
transition: opacity 0.2s, box-shadow 0.2s;
}
.apply-btn:hover {
opacity: 0.85;
box-shadow: 0 0 8px rgba(138, 173, 244, 0.4);
}
/* ---- Charts ---- */
.charts-grid {
display: grid;
grid-template-columns: repeat(2, 1fr);
gap: 1.5rem;
flex: 1;
min-height: 0;
}
.chart-container {
background: var(--input-bg);
border: 1px solid var(--border);
border-radius: 2px;
padding: 1rem;
position: relative;
}
.chart-container.full-width {
grid-column: span 2;
height: 450px;
flex-shrink: 0;
}
.chart-container:nth-child(2),
.chart-container:nth-child(3) {
flex: 1;
min-height: 350px;
}
.chart-title {
font-size: 0.75rem;
font-weight: 600;
margin-bottom: 0.8rem;
color: var(--ink);
letter-spacing: 0.02em;
}
/* ---- Theme Toggle ---- */
.theme-toggle {
position: fixed;
bottom: 1.5rem;
right: 1.5rem;
width: 28px;
height: 28px;
border-radius: 50%;
border: 1px solid var(--border);
cursor: pointer;
padding: 0;
z-index: 30;
transition: all 0.3s ease;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.15);
}
.theme-toggle:hover { transform: scale(1.08); }
[data-theme="light"] .theme-toggle {
background: linear-gradient(135deg, #24273a 0%, #8aadf4 100%);
border-color: rgba(26, 29, 34, 0.2);
}
footer {
position: relative;
z-index: 10;
padding: 0.9rem 2rem;
display: flex;
justify-content: center;
align-items: center;
gap: 1.5rem;
font-family: 'Noto Sans', sans-serif;
font-weight: 300;
font-size: 0.72rem;
letter-spacing: 0.05em;
color: var(--ink-muted);
border-top: 1px solid var(--border);
background: var(--nav-bg);
backdrop-filter: blur(12px);
-webkit-backdrop-filter: blur(12px);
transition: background 0.3s ease;
}
/* ---- Responsive ---- */
@media (max-width: 820px) {
.sidebar {
display: none;
}
.page-body {
padding: calc(55px + 1.25rem) 1rem 3rem;
}
.dashboard {
padding: 1.5rem;
}
.update-date-display {
display: none;
}
}
@media (max-width: 460px) {
.nav-brand .brand-sub { display: none; }
.nav-brand .brand-sep { display: none; }
.dashboard { padding: 1rem; }
}
.toggle-label {
display: flex;
align-items: center;
gap: 0.5rem;
font-size: 0.75rem;
cursor: pointer;
color: var(--ink-muted);
padding: 0.35rem 0;
}
.toggle-label input[type="checkbox"] {
accent-color: var(--accent);
cursor: pointer;
}
.toggle-label span {
line-height: 1.4;
}
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Hagfish — Monthly Data Ingestion
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=%h/hagfisk
ExecStart=%h/hagfisk/hagfisk ingest
StandardOutput=journal
StandardError=journal
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Run Hagfish ingestion monthly
[Timer]
OnCalendar=*-*-28 03:00:00
Persistent=true
Unit=hagfisk-ingest.service
[Install]
WantedBy=timers.target
+13
View File
@@ -0,0 +1,13 @@
[Unit]
Description=Hagfish — Faroese Fisheries Dashboard
After=network-online.target
[Service]
Type=simple
WorkingDirectory=%h/hagfisk
ExecStart=%h/hagfisk/hagfisk serve
Restart=on-failure
RestartSec=5
[Install]
WantedBy=default.target