phase 5 qa and review pass

This commit is contained in:
2026-08-18 22:21:45 +01:00
parent bd7459e455
commit 42a697aa57
4 changed files with 295 additions and 168 deletions
+142 -142
View File
@@ -1,3 +1,4 @@
// src/api.rs
use crate::db; use crate::db;
use crate::types::{ use crate::types::{
Config, LandingDto, LookupDto, MonthlyAggregate, PriceTrend, SpeciesDto, SummaryDto, TopSpecies, Config, LandingDto, LookupDto, MonthlyAggregate, PriceTrend, SpeciesDto, SummaryDto, TopSpecies,
@@ -31,7 +32,7 @@ struct StaticAssets;
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Deserialize)]
pub struct LandingsQuery { pub struct LandingsQuery {
pub month: Option<String>, // Kept for backward compatibility or single-month quick lookups pub month: Option<String>,
pub month_from: Option<String>, pub month_from: Option<String>,
pub month_to: Option<String>, pub month_to: Option<String>,
pub species: Option<String>, pub species: Option<String>,
@@ -164,7 +165,6 @@ async fn get_landings(
Query(params): Query<LandingsQuery>, Query(params): Query<LandingsQuery>,
) -> ApiResult<Json<Vec<LandingDto>>> { ) -> ApiResult<Json<Vec<LandingDto>>> {
let conn = state.conn.clone(); let conn = state.conn.clone();
// Apply hard cap to prevent abuse
let capped_limit = std::cmp::min(params.limit, MAX_LIMIT); let capped_limit = std::cmp::min(params.limit, MAX_LIMIT);
let rows = tokio::task::spawn_blocking(move || -> db::Result<Vec<LandingDto>> { let rows = tokio::task::spawn_blocking(move || -> db::Result<Vec<LandingDto>> {
@@ -178,14 +178,12 @@ async fn get_landings(
let mut args: Vec<Box<dyn duckdb::ToSql>> = Vec::new(); let mut args: Vec<Box<dyn duckdb::ToSql>> = Vec::new();
let mut idx = 1; let mut idx = 1;
// Single month filter (legacy)
if let Some(ref month) = params.month { if let Some(ref month) = params.month {
sql.push_str(&format!(" AND month = ${idx}")); sql.push_str(&format!(" AND month = ${idx}"));
args.push(Box::new(month.clone())); args.push(Box::new(month.clone()));
idx += 1; idx += 1;
} }
// Range filter (new)
if let Some(ref month_from) = params.month_from { if let Some(ref month_from) = params.month_from {
sql.push_str(&format!(" AND month >= ${idx}")); sql.push_str(&format!(" AND month >= ${idx}"));
args.push(Box::new(month_from.clone())); args.push(Box::new(month_from.clone()));
@@ -466,14 +464,12 @@ async fn export_parquet(State(state): State<AppState>) -> ApiResult<Response> {
async fn static_handler(uri: axum::http::Uri) -> Response { async fn static_handler(uri: axum::http::Uri) -> Response {
let path = uri.path().trim_start_matches('/'); let path = uri.path().trim_start_matches('/');
// Strip "static/" prefix if present
let asset_path = if let Some(stripped) = path.strip_prefix("static/") { let asset_path = if let Some(stripped) = path.strip_prefix("static/") {
stripped stripped
} else { } else {
path path
}; };
// If empty path (e.g., visiting "/"), serve index.html directly
let asset_path = if asset_path.is_empty() { let asset_path = if asset_path.is_empty() {
"index.html" "index.html"
} else { } else {
@@ -493,7 +489,12 @@ async fn static_handler(uri: axum::http::Uri) -> Response {
.status(StatusCode::OK) .status(StatusCode::OK)
.header(header::CONTENT_TYPE, mime) .header(header::CONTENT_TYPE, mime)
.body(Body::from(file.data.into_owned())) .body(Body::from(file.data.into_owned()))
.unwrap_or_else(|_| {
Response::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from("Internal server error"))
.unwrap() .unwrap()
})
} }
None => Response::builder() None => Response::builder()
.status(StatusCode::NOT_FOUND) .status(StatusCode::NOT_FOUND)
@@ -655,37 +656,44 @@ mod tests {
} }
} }
#[tokio::test] fn empty_test_state() -> AppState {
async fn test_healthz_returns_ok() { let conn = Connection::open_in_memory().unwrap();
let state = test_state(); db::init_schema(&conn).unwrap();
AppState {
conn: Arc::new(Mutex::new(conn)),
config: Arc::new(Config {
allowed_origins: vec![],
..Default::default()
}),
}
}
async fn spawn_test_server(state: AppState) -> String {
let app = build_router(state); let app = build_router(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap(); let addr = listener.local_addr().unwrap();
tokio::spawn(async move { tokio::spawn(async move {
axum::serve(listener, app).await.unwrap(); axum::serve(listener, app).await.unwrap();
}); });
format!("http://{addr}")
}
let resp = reqwest::get(format!("http://{addr}/healthz")) #[tokio::test]
.await async fn test_healthz_returns_ok() {
.unwrap(); let state = test_state();
let base = spawn_test_server(state).await;
let resp = reqwest::get(format!("{base}/healthz")).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
} }
#[tokio::test] #[tokio::test]
async fn test_get_species_returns_all() { async fn test_get_species_returns_all() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/species")).await.unwrap();
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/species"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
let body: Vec<SpeciesDto> = resp.json().await.unwrap(); let body: Vec<SpeciesDto> = resp.json().await.unwrap();
@@ -696,17 +704,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_landings_no_filter() { async fn test_get_landings_no_filter() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/landings")).await.unwrap();
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/landings"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
let body: Vec<LandingDto> = resp.json().await.unwrap(); let body: Vec<LandingDto> = resp.json().await.unwrap();
@@ -717,15 +717,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_landings_filter_by_species() { async fn test_get_landings_filter_by_species() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/landings?species=COD"))
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/landings?species=COD"))
.await .await
.unwrap(); .unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
@@ -738,15 +732,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_landings_filter_by_measure() { async fn test_get_landings_filter_by_measure() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/landings?measure=MASS"))
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/landings?measure=MASS"))
.await .await
.unwrap(); .unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
@@ -759,17 +747,10 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_landings_range_filter() { async fn test_get_landings_range_filter() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
// Test single month range (same from and to)
let resp = reqwest::get(format!( let resp = reqwest::get(format!(
"http://{addr}/api/landings?month_from=2024M01&month_to=2024M01" "{base}/api/landings?month_from=2024M01&month_to=2024M01"
)) ))
.await .await
.unwrap(); .unwrap();
@@ -779,32 +760,23 @@ mod tests {
assert!(body.iter().all(|l| l.month == "2024M01")); assert!(body.iter().all(|l| l.month == "2024M01"));
assert_eq!(body.len(), 4); assert_eq!(body.len(), 4);
// Test multi-month range
let resp = reqwest::get(format!( let resp = reqwest::get(format!(
"http://{addr}/api/landings?month_from=2024M01&month_to=2024M02" "{base}/api/landings?month_from=2024M01&month_to=2024M02"
)) ))
.await .await
.unwrap(); .unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
let body: Vec<LandingDto> = resp.json().await.unwrap(); let body: Vec<LandingDto> = resp.json().await.unwrap();
assert!(body.len() > 0); assert!(!body.is_empty());
} }
#[tokio::test] #[tokio::test]
async fn test_get_summary_monthly_aggregates() { async fn test_get_summary_monthly_aggregates() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/summary")).await.unwrap();
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/summary"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
let body: SummaryDto = resp.json().await.unwrap(); let body: SummaryDto = resp.json().await.unwrap();
@@ -823,17 +795,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_summary_top_species() { async fn test_get_summary_top_species() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/summary")).await.unwrap();
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/summary"))
.await
.unwrap();
let body: SummaryDto = resp.json().await.unwrap(); let body: SummaryDto = resp.json().await.unwrap();
@@ -845,17 +809,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_summary_price_trend() { async fn test_get_summary_price_trend() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/summary")).await.unwrap();
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/summary"))
.await
.unwrap();
let body: SummaryDto = resp.json().await.unwrap(); let body: SummaryDto = resp.json().await.unwrap();
@@ -877,18 +833,41 @@ mod tests {
assert!((feb.price_per_kg.unwrap() - 5.0).abs() < f64::EPSILON); assert!((feb.price_per_kg.unwrap() - 5.0).abs() < f64::EPSILON);
} }
#[tokio::test]
async fn test_get_summary_with_species_filter() {
let state = test_state();
let base = spawn_test_server(state).await;
let resp = reqwest::get(format!("{base}/api/summary?species=COD"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body: SummaryDto = resp.json().await.unwrap();
assert_eq!(body.monthly.len(), 2);
let jan = body.monthly.iter().find(|m| m.month == "2024M01").unwrap();
assert!((jan.total_mass.unwrap() - 1000.0).abs() < f64::EPSILON);
assert!((jan.total_value.unwrap() - 5000.0).abs() < f64::EPSILON);
assert!(!body.top_species.is_empty());
for ts in &body.top_species {
assert_eq!(ts.species_code, "COD");
}
assert_eq!(body.price_trend.len(), 2);
for pt in &body.price_trend {
assert!(pt.price_per_kg.is_some());
}
}
#[tokio::test] #[tokio::test]
async fn test_get_landings_faroese_label_preserved() { async fn test_get_landings_faroese_label_preserved() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/landings?species=COD"))
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/landings?species=COD"))
.await .await
.unwrap(); .unwrap();
@@ -901,15 +880,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_landings_empty_result() { async fn test_get_landings_empty_result() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/landings?species=NONEXISTENT"))
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/landings?species=NONEXISTENT"))
.await .await
.unwrap(); .unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
@@ -921,15 +894,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_get_landings_limit_applied() { async fn test_get_landings_limit_applied() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/landings?limit=2"))
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/landings?limit=2"))
.await .await
.unwrap(); .unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
@@ -941,15 +908,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_export_parquet() { async fn test_export_parquet() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let resp = reqwest::get(format!("{base}/api/export.parquet"))
axum::serve(listener, app).await.unwrap();
});
let resp = reqwest::get(format!("http://{addr}/api/export.parquet"))
.await .await
.unwrap(); .unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
@@ -963,18 +924,27 @@ mod tests {
assert_eq!(&bytes[..4], b"PAR1"); assert_eq!(&bytes[..4], b"PAR1");
} }
#[tokio::test]
async fn test_export_parquet_empty_db() {
let state = empty_test_state();
let base = spawn_test_server(state).await;
let resp = reqwest::get(format!("{base}/api/export.parquet"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let bytes = resp.bytes().await.unwrap();
assert_eq!(&bytes[..4], b"PAR1");
assert!(bytes.len() > 4);
}
#[tokio::test] #[tokio::test]
async fn test_concurrent_export_requests() { async fn test_concurrent_export_requests() {
let state = test_state(); let state = test_state();
let app = build_router(state); let base = spawn_test_server(state).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { let url = format!("{base}/api/export.parquet");
axum::serve(listener, app).await.unwrap();
});
let url = format!("http://{addr}/api/export.parquet");
let (resp1, resp2) = tokio::join!(reqwest::get(&url), reqwest::get(&url)); let (resp1, resp2) = tokio::join!(reqwest::get(&url), reqwest::get(&url));
let resp1 = resp1.unwrap(); let resp1 = resp1.unwrap();
@@ -992,27 +962,57 @@ mod tests {
assert!(!bytes2.is_empty()); assert!(!bytes2.is_empty());
} }
#[tokio::test]
async fn test_static_handler_falls_back_to_index_html() {
let state = test_state();
let base = spawn_test_server(state).await;
let resp = reqwest::get(format!("{base}/nonexistent-path-xyz"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = resp.text().await.unwrap();
assert!(body.contains("<html"));
assert!(body.contains("<title>Fisheries Dashboard"));
}
#[tokio::test] #[tokio::test]
async fn test_sql_injection_safe() { async fn test_sql_injection_safe() {
let state = test_state(); let state = test_state();
let state_clone = state.clone(); let base = spawn_test_server(state.clone()).await;
let app = build_router(state.clone());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let malicious_params = "?species='; DROP TABLE landings; --"; let malicious_params = "?species='; DROP TABLE landings; --";
let url = format!("http://{addr}/api/landings{malicious_params}"); let url = format!("{base}/api/landings{malicious_params}");
let resp = reqwest::get(&url).await.unwrap(); let resp = reqwest::get(&url).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK); assert_eq!(resp.status(), StatusCode::OK);
let count: i64 = tokio::task::spawn_blocking(move || { let count: i64 = tokio::task::spawn_blocking(move || {
let conn = state_clone.conn.blocking_lock(); let conn = state.conn.blocking_lock();
conn.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
.unwrap()
})
.await
.unwrap();
assert_eq!(count, 6);
}
#[tokio::test]
async fn test_sql_injection_all_params_safe() {
let state = test_state();
let base = spawn_test_server(state.clone()).await;
let malicious_params =
"?month_from=2024M01'; DROP TABLE landings;--&month_to=2024M12&species=COD";
let url = format!("{base}/api/landings{malicious_params}");
let resp = reqwest::get(&url).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let count: i64 = tokio::task::spawn_blocking(move || {
let conn = state.conn.blocking_lock();
conn.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0)) conn.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
.unwrap() .unwrap()
}) })
+17 -17
View File
@@ -15,7 +15,7 @@ const DIM_MEASURE: &str = "measure";
const SENTINEL_VALUE: f64 = -1.0; const SENTINEL_VALUE: f64 = -1.0;
fn is_sentinel(v: f64) -> bool { fn is_sentinel(v: f64) -> bool {
v.total_cmp(&SENTINEL_VALUE) == std::cmp::Ordering::Equal v == SENTINEL_VALUE
} }
pub async fn fetch_metadata(client: &Client, url: &str) -> Result<(LookupMap, MetadataResponse)> { pub async fn fetch_metadata(client: &Client, url: &str) -> Result<(LookupMap, MetadataResponse)> {
@@ -322,17 +322,17 @@ pub fn parse_row(row_index: usize, dataset: &Dataset, lookup_maps: &LookupMap) -
}) })
} }
pub fn data_row_to_landing(row: &DataRow) -> Landing { pub fn data_row_to_landing(row: DataRow) -> Landing {
Landing { Landing {
month: row.month.clone(), month: row.month,
species_code: row.species_code.clone(), species_code: row.species_code,
species_label: row.species_label.clone(), species_label: row.species_label,
gear_code: row.gear_code.clone(), gear_code: row.gear_code,
zone_code: row.zone_code.clone(), zone_code: row.zone_code,
processing_code: row.processing_code.clone(), processing_code: row.processing_code,
preservation_code: row.preservation_code.clone(), preservation_code: row.preservation_code,
shipsize_code: row.shipsize_code.clone(), shipsize_code: row.shipsize_code,
measure_code: row.measure_code.clone(), measure_code: row.measure_code,
value: row.value, value: row.value,
} }
} }
@@ -608,13 +608,13 @@ mod tests {
let lookup_maps = mock_lookup_maps(); let lookup_maps = mock_lookup_maps();
let row = parse_row(0, &dataset.dataset, &lookup_maps).expect("parse failed"); let row = parse_row(0, &dataset.dataset, &lookup_maps).expect("parse failed");
let landing = data_row_to_landing(&row); let landing = data_row_to_landing(row);
assert_eq!(landing.month, row.month); assert_eq!(landing.month, "2015M01");
assert_eq!(landing.species_code, row.species_code); assert_eq!(landing.species_code, "148XXXXXXX00000");
assert_eq!(landing.species_label, row.species_label); assert_eq!(landing.species_label, "Sild");
assert_eq!(landing.measure_code, row.measure_code); assert_eq!(landing.measure_code, "MASS");
assert_eq!(landing.value, row.value); assert_eq!(landing.value, Some(1234.5));
} }
#[test] #[test]
+67 -8
View File
@@ -1,3 +1,4 @@
// src/main.rs
mod api; mod api;
mod cli; mod cli;
mod db; mod db;
@@ -19,7 +20,6 @@ struct LoggerGuard {
fn init_logging(log_file_path: Option<&str>) -> LoggerGuard { fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json"); let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json");
match log_file_path { match log_file_path {
@@ -28,7 +28,7 @@ fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
let (non_blocking, guard) = tracing_appender::non_blocking(file); let (non_blocking, guard) = tracing_appender::non_blocking(file);
let subscriber = tracing_subscriber::fmt() let subscriber = tracing_subscriber::fmt()
.with_env_filter(filter.clone()) .with_env_filter(filter)
.with_writer(non_blocking); .with_writer(non_blocking);
if is_json { if is_json {
@@ -42,7 +42,7 @@ fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
} }
} }
None => { None => {
let subscriber = tracing_subscriber::fmt().with_env_filter(filter.clone()); let subscriber = tracing_subscriber::fmt().with_env_filter(filter);
if is_json { if is_json {
subscriber.json().init(); subscriber.json().init();
@@ -89,15 +89,20 @@ async fn main() -> anyhow::Result<()> {
if default_path.exists() { if default_path.exists() {
types::Config::load(default_path)? types::Config::load(default_path)?
} else { } else {
tracing::warn!("No config file found, using defaults"); eprintln!("WARN: No config file found, using defaults");
types::Config::default() types::Config::default()
} }
} }
}; };
// Initialize logging AFTER config is loaded so we have log_file_path
let _logger_guard = init_logging(config.log_file_path.as_deref()); let _logger_guard = init_logging(config.log_file_path.as_deref());
tracing::info!(
duckdb_path = %config.duckdb_path,
bind_address = %config.bind_address,
"configuration loaded"
);
match cli.command { match cli.command {
cli::Command::Serve => run_serve(&config).await, cli::Command::Serve => run_serve(&config).await,
cli::Command::Ingest { full } => run_ingest(&config, full).await, cli::Command::Ingest { full } => run_ingest(&config, full).await,
@@ -134,7 +139,9 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
); );
let conn = db::init(&config.duckdb_path)?; let conn = db::init(&config.duckdb_path)?;
let client = reqwest::Client::new(); let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(60))
.build()?;
let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url) let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url)
.await .await
@@ -219,7 +226,7 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
for j in 0..data.dataset.value.len() { for j in 0..data.dataset.value.len() {
match ingest::parse_row(j, &data.dataset, &lookup_maps) { match ingest::parse_row(j, &data.dataset, &lookup_maps) {
Ok(row) => { Ok(row) => {
landings.push(ingest::data_row_to_landing(&row)); landings.push(ingest::data_row_to_landing(row));
} }
Err(e) => { Err(e) => {
tracing::warn!( tracing::warn!(
@@ -260,7 +267,6 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
"batch complete" "batch complete"
); );
// Backoff between batches to avoid rate limiting
if i < batches.len() - 1 { if i < batches.len() - 1 {
tokio::time::sleep(std::time::Duration::from_secs(1)).await; tokio::time::sleep(std::time::Duration::from_secs(1)).await;
} }
@@ -300,3 +306,56 @@ fn run_export(config: &types::Config, out: &Path) -> anyhow::Result<()> {
Ok(()) Ok(())
} }
#[cfg(test)]
mod tests {
use super::*;
use crate::types::Config;
#[test]
fn test_config_default_values() {
let config = Config::default();
assert_eq!(config.duckdb_path, "hagfish.db");
assert_eq!(config.bind_address, "127.0.0.1:8090");
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
}
#[test]
fn test_run_export_writes_valid_parquet() {
let tmp_dir = tempfile::tempdir().unwrap();
let db_path = tmp_dir.path().join("test.db");
let parquet_path = tmp_dir.path().join("export.parquet");
let conn = db::init(db_path.to_str().unwrap()).unwrap();
let rows = vec![crate::types::Landing {
month: "2024M01".to_string(),
species_code: "COD".to_string(),
species_label: "Toskur".to_string(),
gear_code: "TOTAL".to_string(),
zone_code: "TOTAL".to_string(),
processing_code: "TOTAL".to_string(),
preservation_code: "TOTAL".to_string(),
shipsize_code: "TOTAL".to_string(),
measure_code: "MASS".to_string(),
value: Some(1000.0),
}];
db::upsert_landings(&conn, &rows).unwrap();
drop(conn);
let config = Config {
duckdb_path: db_path.to_str().unwrap().to_string(),
..Default::default()
};
let result = run_export(&config, &parquet_path);
assert!(result.is_ok());
let metadata = std::fs::metadata(&parquet_path).unwrap();
assert!(metadata.len() > 0);
let file_bytes = std::fs::read(&parquet_path).unwrap();
assert_eq!(&file_bytes[..4], b"PAR1");
}
}
+68
View File
@@ -265,3 +265,71 @@ pub enum IngestError {
pub type LookupMap = HashMap<String, HashMap<String, String>>; pub type LookupMap = HashMap<String, HashMap<String, String>>;
pub type Result<T> = std::result::Result<T, IngestError>; pub type Result<T> = std::result::Result<T, IngestError>;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_config_load_missing_file_hard_error() {
let result = Config::load(Path::new("/nonexistent/path/config.json"));
assert!(result.is_err());
let err_msg = result.unwrap_err().to_string();
assert!(err_msg.contains("Failed to read config file"));
}
#[test]
fn test_config_load_invalid_json_error() {
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(tmp.path(), "not valid json {{{").unwrap();
let result = Config::load(tmp.path());
assert!(result.is_err());
let err_msg = result.unwrap_err().to_string();
assert!(err_msg.contains("Failed to parse config file"));
}
#[test]
fn test_config_load_valid_file() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let content = r#"{
"duckdb_path": "test.db",
"bind_address": "127.0.0.1:9090",
"data_source_url": "http://example.com",
"log_file_path": null,
"allowed_origins": ["https://example.com"]
}"#;
std::fs::write(tmp.path(), content).unwrap();
let config = Config::load(tmp.path()).unwrap();
assert_eq!(config.duckdb_path, "test.db");
assert_eq!(config.bind_address, "127.0.0.1:9090");
assert_eq!(config.data_source_url, "http://example.com");
assert!(config.log_file_path.is_none());
assert_eq!(config.allowed_origins, vec!["https://example.com"]);
}
#[test]
fn test_config_load_defaults_allowed_origins() {
let tmp = tempfile::NamedTempFile::new().unwrap();
let content = r#"{
"duckdb_path": "test.db",
"bind_address": "127.0.0.1:9090",
"data_source_url": "http://example.com"
}"#;
std::fs::write(tmp.path(), content).unwrap();
let config = Config::load(tmp.path()).unwrap();
assert!(config.allowed_origins.is_empty());
}
#[test]
fn test_config_default_values() {
let config = Config::default();
assert_eq!(config.duckdb_path, "hagfish.db");
assert_eq!(config.bind_address, "127.0.0.1:8090");
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
assert!(config.log_file_path.is_some());
assert!(config.allowed_origins.is_empty());
}
}