phase 5 qa and review pass
This commit is contained in:
+143
-143
@@ -1,3 +1,4 @@
|
|||||||
|
// src/api.rs
|
||||||
use crate::db;
|
use crate::db;
|
||||||
use crate::types::{
|
use crate::types::{
|
||||||
Config, LandingDto, LookupDto, MonthlyAggregate, PriceTrend, SpeciesDto, SummaryDto, TopSpecies,
|
Config, LandingDto, LookupDto, MonthlyAggregate, PriceTrend, SpeciesDto, SummaryDto, TopSpecies,
|
||||||
@@ -31,7 +32,7 @@ struct StaticAssets;
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct LandingsQuery {
|
pub struct LandingsQuery {
|
||||||
pub month: Option<String>, // Kept for backward compatibility or single-month quick lookups
|
pub month: Option<String>,
|
||||||
pub month_from: Option<String>,
|
pub month_from: Option<String>,
|
||||||
pub month_to: Option<String>,
|
pub month_to: Option<String>,
|
||||||
pub species: Option<String>,
|
pub species: Option<String>,
|
||||||
@@ -164,7 +165,6 @@ async fn get_landings(
|
|||||||
Query(params): Query<LandingsQuery>,
|
Query(params): Query<LandingsQuery>,
|
||||||
) -> ApiResult<Json<Vec<LandingDto>>> {
|
) -> ApiResult<Json<Vec<LandingDto>>> {
|
||||||
let conn = state.conn.clone();
|
let conn = state.conn.clone();
|
||||||
// Apply hard cap to prevent abuse
|
|
||||||
let capped_limit = std::cmp::min(params.limit, MAX_LIMIT);
|
let capped_limit = std::cmp::min(params.limit, MAX_LIMIT);
|
||||||
|
|
||||||
let rows = tokio::task::spawn_blocking(move || -> db::Result<Vec<LandingDto>> {
|
let rows = tokio::task::spawn_blocking(move || -> db::Result<Vec<LandingDto>> {
|
||||||
@@ -178,14 +178,12 @@ async fn get_landings(
|
|||||||
let mut args: Vec<Box<dyn duckdb::ToSql>> = Vec::new();
|
let mut args: Vec<Box<dyn duckdb::ToSql>> = Vec::new();
|
||||||
let mut idx = 1;
|
let mut idx = 1;
|
||||||
|
|
||||||
// Single month filter (legacy)
|
|
||||||
if let Some(ref month) = params.month {
|
if let Some(ref month) = params.month {
|
||||||
sql.push_str(&format!(" AND month = ${idx}"));
|
sql.push_str(&format!(" AND month = ${idx}"));
|
||||||
args.push(Box::new(month.clone()));
|
args.push(Box::new(month.clone()));
|
||||||
idx += 1;
|
idx += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Range filter (new)
|
|
||||||
if let Some(ref month_from) = params.month_from {
|
if let Some(ref month_from) = params.month_from {
|
||||||
sql.push_str(&format!(" AND month >= ${idx}"));
|
sql.push_str(&format!(" AND month >= ${idx}"));
|
||||||
args.push(Box::new(month_from.clone()));
|
args.push(Box::new(month_from.clone()));
|
||||||
@@ -466,14 +464,12 @@ async fn export_parquet(State(state): State<AppState>) -> ApiResult<Response> {
|
|||||||
async fn static_handler(uri: axum::http::Uri) -> Response {
|
async fn static_handler(uri: axum::http::Uri) -> Response {
|
||||||
let path = uri.path().trim_start_matches('/');
|
let path = uri.path().trim_start_matches('/');
|
||||||
|
|
||||||
// Strip "static/" prefix if present
|
|
||||||
let asset_path = if let Some(stripped) = path.strip_prefix("static/") {
|
let asset_path = if let Some(stripped) = path.strip_prefix("static/") {
|
||||||
stripped
|
stripped
|
||||||
} else {
|
} else {
|
||||||
path
|
path
|
||||||
};
|
};
|
||||||
|
|
||||||
// If empty path (e.g., visiting "/"), serve index.html directly
|
|
||||||
let asset_path = if asset_path.is_empty() {
|
let asset_path = if asset_path.is_empty() {
|
||||||
"index.html"
|
"index.html"
|
||||||
} else {
|
} else {
|
||||||
@@ -493,7 +489,12 @@ async fn static_handler(uri: axum::http::Uri) -> Response {
|
|||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
.header(header::CONTENT_TYPE, mime)
|
.header(header::CONTENT_TYPE, mime)
|
||||||
.body(Body::from(file.data.into_owned()))
|
.body(Body::from(file.data.into_owned()))
|
||||||
.unwrap()
|
.unwrap_or_else(|_| {
|
||||||
|
Response::builder()
|
||||||
|
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||||
|
.body(Body::from("Internal server error"))
|
||||||
|
.unwrap()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
None => Response::builder()
|
None => Response::builder()
|
||||||
.status(StatusCode::NOT_FOUND)
|
.status(StatusCode::NOT_FOUND)
|
||||||
@@ -655,37 +656,44 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
fn empty_test_state() -> AppState {
|
||||||
async fn test_healthz_returns_ok() {
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
let state = test_state();
|
db::init_schema(&conn).unwrap();
|
||||||
|
|
||||||
|
AppState {
|
||||||
|
conn: Arc::new(Mutex::new(conn)),
|
||||||
|
config: Arc::new(Config {
|
||||||
|
allowed_origins: vec![],
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn spawn_test_server(state: AppState) -> String {
|
||||||
let app = build_router(state);
|
let app = build_router(state);
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let addr = listener.local_addr().unwrap();
|
let addr = listener.local_addr().unwrap();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
axum::serve(listener, app).await.unwrap();
|
axum::serve(listener, app).await.unwrap();
|
||||||
});
|
});
|
||||||
|
format!("http://{addr}")
|
||||||
|
}
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/healthz"))
|
#[tokio::test]
|
||||||
.await
|
async fn test_healthz_returns_ok() {
|
||||||
.unwrap();
|
let state = test_state();
|
||||||
|
let base = spawn_test_server(state).await;
|
||||||
|
|
||||||
|
let resp = reqwest::get(format!("{base}/healthz")).await.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_species_returns_all() {
|
async fn test_get_species_returns_all() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/species")).await.unwrap();
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/species"))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body: Vec<SpeciesDto> = resp.json().await.unwrap();
|
let body: Vec<SpeciesDto> = resp.json().await.unwrap();
|
||||||
@@ -696,17 +704,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_no_filter() {
|
async fn test_get_landings_no_filter() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/landings")).await.unwrap();
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/landings"))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body: Vec<LandingDto> = resp.json().await.unwrap();
|
let body: Vec<LandingDto> = resp.json().await.unwrap();
|
||||||
@@ -717,15 +717,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_filter_by_species() {
|
async fn test_get_landings_filter_by_species() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/landings?species=COD"))
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/landings?species=COD"))
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
@@ -738,15 +732,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_filter_by_measure() {
|
async fn test_get_landings_filter_by_measure() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/landings?measure=MASS"))
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/landings?measure=MASS"))
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
@@ -759,17 +747,10 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_range_filter() {
|
async fn test_get_landings_range_filter() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
// Test single month range (same from and to)
|
|
||||||
let resp = reqwest::get(format!(
|
let resp = reqwest::get(format!(
|
||||||
"http://{addr}/api/landings?month_from=2024M01&month_to=2024M01"
|
"{base}/api/landings?month_from=2024M01&month_to=2024M01"
|
||||||
))
|
))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -779,32 +760,23 @@ mod tests {
|
|||||||
assert!(body.iter().all(|l| l.month == "2024M01"));
|
assert!(body.iter().all(|l| l.month == "2024M01"));
|
||||||
assert_eq!(body.len(), 4);
|
assert_eq!(body.len(), 4);
|
||||||
|
|
||||||
// Test multi-month range
|
|
||||||
let resp = reqwest::get(format!(
|
let resp = reqwest::get(format!(
|
||||||
"http://{addr}/api/landings?month_from=2024M01&month_to=2024M02"
|
"{base}/api/landings?month_from=2024M01&month_to=2024M02"
|
||||||
))
|
))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body: Vec<LandingDto> = resp.json().await.unwrap();
|
let body: Vec<LandingDto> = resp.json().await.unwrap();
|
||||||
assert!(body.len() > 0);
|
assert!(!body.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_summary_monthly_aggregates() {
|
async fn test_get_summary_monthly_aggregates() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/summary")).await.unwrap();
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/summary"))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
let body: SummaryDto = resp.json().await.unwrap();
|
let body: SummaryDto = resp.json().await.unwrap();
|
||||||
@@ -823,17 +795,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_summary_top_species() {
|
async fn test_get_summary_top_species() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/summary")).await.unwrap();
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/summary"))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let body: SummaryDto = resp.json().await.unwrap();
|
let body: SummaryDto = resp.json().await.unwrap();
|
||||||
|
|
||||||
@@ -845,17 +809,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_summary_price_trend() {
|
async fn test_get_summary_price_trend() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/summary")).await.unwrap();
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/summary"))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let body: SummaryDto = resp.json().await.unwrap();
|
let body: SummaryDto = resp.json().await.unwrap();
|
||||||
|
|
||||||
@@ -877,18 +833,41 @@ mod tests {
|
|||||||
assert!((feb.price_per_kg.unwrap() - 5.0).abs() < f64::EPSILON);
|
assert!((feb.price_per_kg.unwrap() - 5.0).abs() < f64::EPSILON);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_get_summary_with_species_filter() {
|
||||||
|
let state = test_state();
|
||||||
|
let base = spawn_test_server(state).await;
|
||||||
|
|
||||||
|
let resp = reqwest::get(format!("{base}/api/summary?species=COD"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let body: SummaryDto = resp.json().await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(body.monthly.len(), 2);
|
||||||
|
|
||||||
|
let jan = body.monthly.iter().find(|m| m.month == "2024M01").unwrap();
|
||||||
|
assert!((jan.total_mass.unwrap() - 1000.0).abs() < f64::EPSILON);
|
||||||
|
assert!((jan.total_value.unwrap() - 5000.0).abs() < f64::EPSILON);
|
||||||
|
|
||||||
|
assert!(!body.top_species.is_empty());
|
||||||
|
for ts in &body.top_species {
|
||||||
|
assert_eq!(ts.species_code, "COD");
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(body.price_trend.len(), 2);
|
||||||
|
for pt in &body.price_trend {
|
||||||
|
assert!(pt.price_per_kg.is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_faroese_label_preserved() {
|
async fn test_get_landings_faroese_label_preserved() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/landings?species=COD"))
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/landings?species=COD"))
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
@@ -901,15 +880,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_empty_result() {
|
async fn test_get_landings_empty_result() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/landings?species=NONEXISTENT"))
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/landings?species=NONEXISTENT"))
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
@@ -921,15 +894,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_landings_limit_applied() {
|
async fn test_get_landings_limit_applied() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/landings?limit=2"))
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/landings?limit=2"))
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
@@ -941,15 +908,9 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_export_parquet() {
|
async fn test_export_parquet() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let resp = reqwest::get(format!("{base}/api/export.parquet"))
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let resp = reqwest::get(format!("http://{addr}/api/export.parquet"))
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
@@ -963,18 +924,27 @@ mod tests {
|
|||||||
assert_eq!(&bytes[..4], b"PAR1");
|
assert_eq!(&bytes[..4], b"PAR1");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_export_parquet_empty_db() {
|
||||||
|
let state = empty_test_state();
|
||||||
|
let base = spawn_test_server(state).await;
|
||||||
|
|
||||||
|
let resp = reqwest::get(format!("{base}/api/export.parquet"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let bytes = resp.bytes().await.unwrap();
|
||||||
|
assert_eq!(&bytes[..4], b"PAR1");
|
||||||
|
assert!(bytes.len() > 4);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_concurrent_export_requests() {
|
async fn test_concurrent_export_requests() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let app = build_router(state);
|
let base = spawn_test_server(state).await;
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
let url = format!("{base}/api/export.parquet");
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let url = format!("http://{addr}/api/export.parquet");
|
|
||||||
let (resp1, resp2) = tokio::join!(reqwest::get(&url), reqwest::get(&url));
|
let (resp1, resp2) = tokio::join!(reqwest::get(&url), reqwest::get(&url));
|
||||||
|
|
||||||
let resp1 = resp1.unwrap();
|
let resp1 = resp1.unwrap();
|
||||||
@@ -992,27 +962,57 @@ mod tests {
|
|||||||
assert!(!bytes2.is_empty());
|
assert!(!bytes2.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_static_handler_falls_back_to_index_html() {
|
||||||
|
let state = test_state();
|
||||||
|
let base = spawn_test_server(state).await;
|
||||||
|
|
||||||
|
let resp = reqwest::get(format!("{base}/nonexistent-path-xyz"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let body = resp.text().await.unwrap();
|
||||||
|
assert!(body.contains("<html"));
|
||||||
|
assert!(body.contains("<title>Fisheries Dashboard"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_sql_injection_safe() {
|
async fn test_sql_injection_safe() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let state_clone = state.clone();
|
let base = spawn_test_server(state.clone()).await;
|
||||||
|
|
||||||
let app = build_router(state.clone());
|
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
|
||||||
axum::serve(listener, app).await.unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let malicious_params = "?species='; DROP TABLE landings; --";
|
let malicious_params = "?species='; DROP TABLE landings; --";
|
||||||
let url = format!("http://{addr}/api/landings{malicious_params}");
|
let url = format!("{base}/api/landings{malicious_params}");
|
||||||
let resp = reqwest::get(&url).await.unwrap();
|
let resp = reqwest::get(&url).await.unwrap();
|
||||||
|
|
||||||
assert_eq!(resp.status(), StatusCode::OK);
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
let count: i64 = tokio::task::spawn_blocking(move || {
|
let count: i64 = tokio::task::spawn_blocking(move || {
|
||||||
let conn = state_clone.conn.blocking_lock();
|
let conn = state.conn.blocking_lock();
|
||||||
|
conn.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
||||||
|
.unwrap()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(count, 6);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_sql_injection_all_params_safe() {
|
||||||
|
let state = test_state();
|
||||||
|
let base = spawn_test_server(state.clone()).await;
|
||||||
|
|
||||||
|
let malicious_params =
|
||||||
|
"?month_from=2024M01'; DROP TABLE landings;--&month_to=2024M12&species=COD";
|
||||||
|
let url = format!("{base}/api/landings{malicious_params}");
|
||||||
|
let resp = reqwest::get(&url).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let count: i64 = tokio::task::spawn_blocking(move || {
|
||||||
|
let conn = state.conn.blocking_lock();
|
||||||
conn.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
conn.query_row("SELECT COUNT(*) FROM landings", [], |row| row.get(0))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
})
|
})
|
||||||
|
|||||||
+17
-17
@@ -15,7 +15,7 @@ const DIM_MEASURE: &str = "measure";
|
|||||||
const SENTINEL_VALUE: f64 = -1.0;
|
const SENTINEL_VALUE: f64 = -1.0;
|
||||||
|
|
||||||
fn is_sentinel(v: f64) -> bool {
|
fn is_sentinel(v: f64) -> bool {
|
||||||
v.total_cmp(&SENTINEL_VALUE) == std::cmp::Ordering::Equal
|
v == SENTINEL_VALUE
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch_metadata(client: &Client, url: &str) -> Result<(LookupMap, MetadataResponse)> {
|
pub async fn fetch_metadata(client: &Client, url: &str) -> Result<(LookupMap, MetadataResponse)> {
|
||||||
@@ -322,17 +322,17 @@ pub fn parse_row(row_index: usize, dataset: &Dataset, lookup_maps: &LookupMap) -
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn data_row_to_landing(row: &DataRow) -> Landing {
|
pub fn data_row_to_landing(row: DataRow) -> Landing {
|
||||||
Landing {
|
Landing {
|
||||||
month: row.month.clone(),
|
month: row.month,
|
||||||
species_code: row.species_code.clone(),
|
species_code: row.species_code,
|
||||||
species_label: row.species_label.clone(),
|
species_label: row.species_label,
|
||||||
gear_code: row.gear_code.clone(),
|
gear_code: row.gear_code,
|
||||||
zone_code: row.zone_code.clone(),
|
zone_code: row.zone_code,
|
||||||
processing_code: row.processing_code.clone(),
|
processing_code: row.processing_code,
|
||||||
preservation_code: row.preservation_code.clone(),
|
preservation_code: row.preservation_code,
|
||||||
shipsize_code: row.shipsize_code.clone(),
|
shipsize_code: row.shipsize_code,
|
||||||
measure_code: row.measure_code.clone(),
|
measure_code: row.measure_code,
|
||||||
value: row.value,
|
value: row.value,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -608,13 +608,13 @@ mod tests {
|
|||||||
let lookup_maps = mock_lookup_maps();
|
let lookup_maps = mock_lookup_maps();
|
||||||
|
|
||||||
let row = parse_row(0, &dataset.dataset, &lookup_maps).expect("parse failed");
|
let row = parse_row(0, &dataset.dataset, &lookup_maps).expect("parse failed");
|
||||||
let landing = data_row_to_landing(&row);
|
let landing = data_row_to_landing(row);
|
||||||
|
|
||||||
assert_eq!(landing.month, row.month);
|
assert_eq!(landing.month, "2015M01");
|
||||||
assert_eq!(landing.species_code, row.species_code);
|
assert_eq!(landing.species_code, "148XXXXXXX00000");
|
||||||
assert_eq!(landing.species_label, row.species_label);
|
assert_eq!(landing.species_label, "Sild");
|
||||||
assert_eq!(landing.measure_code, row.measure_code);
|
assert_eq!(landing.measure_code, "MASS");
|
||||||
assert_eq!(landing.value, row.value);
|
assert_eq!(landing.value, Some(1234.5));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+67
-8
@@ -1,3 +1,4 @@
|
|||||||
|
// src/main.rs
|
||||||
mod api;
|
mod api;
|
||||||
mod cli;
|
mod cli;
|
||||||
mod db;
|
mod db;
|
||||||
@@ -19,7 +20,6 @@ struct LoggerGuard {
|
|||||||
|
|
||||||
fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
|
fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
|
||||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||||
|
|
||||||
let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json");
|
let is_json = std::env::var("HAGFISH_LOG_FORMAT").as_deref() == Ok("json");
|
||||||
|
|
||||||
match log_file_path {
|
match log_file_path {
|
||||||
@@ -28,7 +28,7 @@ fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
|
|||||||
let (non_blocking, guard) = tracing_appender::non_blocking(file);
|
let (non_blocking, guard) = tracing_appender::non_blocking(file);
|
||||||
|
|
||||||
let subscriber = tracing_subscriber::fmt()
|
let subscriber = tracing_subscriber::fmt()
|
||||||
.with_env_filter(filter.clone())
|
.with_env_filter(filter)
|
||||||
.with_writer(non_blocking);
|
.with_writer(non_blocking);
|
||||||
|
|
||||||
if is_json {
|
if is_json {
|
||||||
@@ -42,7 +42,7 @@ fn init_logging(log_file_path: Option<&str>) -> LoggerGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
let subscriber = tracing_subscriber::fmt().with_env_filter(filter.clone());
|
let subscriber = tracing_subscriber::fmt().with_env_filter(filter);
|
||||||
|
|
||||||
if is_json {
|
if is_json {
|
||||||
subscriber.json().init();
|
subscriber.json().init();
|
||||||
@@ -89,15 +89,20 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
if default_path.exists() {
|
if default_path.exists() {
|
||||||
types::Config::load(default_path)?
|
types::Config::load(default_path)?
|
||||||
} else {
|
} else {
|
||||||
tracing::warn!("No config file found, using defaults");
|
eprintln!("WARN: No config file found, using defaults");
|
||||||
types::Config::default()
|
types::Config::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Initialize logging AFTER config is loaded so we have log_file_path
|
|
||||||
let _logger_guard = init_logging(config.log_file_path.as_deref());
|
let _logger_guard = init_logging(config.log_file_path.as_deref());
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
duckdb_path = %config.duckdb_path,
|
||||||
|
bind_address = %config.bind_address,
|
||||||
|
"configuration loaded"
|
||||||
|
);
|
||||||
|
|
||||||
match cli.command {
|
match cli.command {
|
||||||
cli::Command::Serve => run_serve(&config).await,
|
cli::Command::Serve => run_serve(&config).await,
|
||||||
cli::Command::Ingest { full } => run_ingest(&config, full).await,
|
cli::Command::Ingest { full } => run_ingest(&config, full).await,
|
||||||
@@ -134,7 +139,9 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let conn = db::init(&config.duckdb_path)?;
|
let conn = db::init(&config.duckdb_path)?;
|
||||||
let client = reqwest::Client::new();
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(std::time::Duration::from_secs(60))
|
||||||
|
.build()?;
|
||||||
|
|
||||||
let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url)
|
let (lookup_maps, metadata) = ingest::fetch_metadata(&client, &config.data_source_url)
|
||||||
.await
|
.await
|
||||||
@@ -219,7 +226,7 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
|
|||||||
for j in 0..data.dataset.value.len() {
|
for j in 0..data.dataset.value.len() {
|
||||||
match ingest::parse_row(j, &data.dataset, &lookup_maps) {
|
match ingest::parse_row(j, &data.dataset, &lookup_maps) {
|
||||||
Ok(row) => {
|
Ok(row) => {
|
||||||
landings.push(ingest::data_row_to_landing(&row));
|
landings.push(ingest::data_row_to_landing(row));
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
@@ -260,7 +267,6 @@ async fn run_ingest(config: &types::Config, full: bool) -> anyhow::Result<()> {
|
|||||||
"batch complete"
|
"batch complete"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Backoff between batches to avoid rate limiting
|
|
||||||
if i < batches.len() - 1 {
|
if i < batches.len() - 1 {
|
||||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||||
}
|
}
|
||||||
@@ -300,3 +306,56 @@ fn run_export(config: &types::Config, out: &Path) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::types::Config;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_default_values() {
|
||||||
|
let config = Config::default();
|
||||||
|
assert_eq!(config.duckdb_path, "hagfish.db");
|
||||||
|
assert_eq!(config.bind_address, "127.0.0.1:8090");
|
||||||
|
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_run_export_writes_valid_parquet() {
|
||||||
|
let tmp_dir = tempfile::tempdir().unwrap();
|
||||||
|
let db_path = tmp_dir.path().join("test.db");
|
||||||
|
let parquet_path = tmp_dir.path().join("export.parquet");
|
||||||
|
|
||||||
|
let conn = db::init(db_path.to_str().unwrap()).unwrap();
|
||||||
|
|
||||||
|
let rows = vec![crate::types::Landing {
|
||||||
|
month: "2024M01".to_string(),
|
||||||
|
species_code: "COD".to_string(),
|
||||||
|
species_label: "Toskur".to_string(),
|
||||||
|
gear_code: "TOTAL".to_string(),
|
||||||
|
zone_code: "TOTAL".to_string(),
|
||||||
|
processing_code: "TOTAL".to_string(),
|
||||||
|
preservation_code: "TOTAL".to_string(),
|
||||||
|
shipsize_code: "TOTAL".to_string(),
|
||||||
|
measure_code: "MASS".to_string(),
|
||||||
|
value: Some(1000.0),
|
||||||
|
}];
|
||||||
|
|
||||||
|
db::upsert_landings(&conn, &rows).unwrap();
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
|
let config = Config {
|
||||||
|
duckdb_path: db_path.to_str().unwrap().to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = run_export(&config, &parquet_path);
|
||||||
|
assert!(result.is_ok());
|
||||||
|
|
||||||
|
let metadata = std::fs::metadata(&parquet_path).unwrap();
|
||||||
|
assert!(metadata.len() > 0);
|
||||||
|
|
||||||
|
let file_bytes = std::fs::read(&parquet_path).unwrap();
|
||||||
|
assert_eq!(&file_bytes[..4], b"PAR1");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -265,3 +265,71 @@ pub enum IngestError {
|
|||||||
pub type LookupMap = HashMap<String, HashMap<String, String>>;
|
pub type LookupMap = HashMap<String, HashMap<String, String>>;
|
||||||
|
|
||||||
pub type Result<T> = std::result::Result<T, IngestError>;
|
pub type Result<T> = std::result::Result<T, IngestError>;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_missing_file_hard_error() {
|
||||||
|
let result = Config::load(Path::new("/nonexistent/path/config.json"));
|
||||||
|
assert!(result.is_err());
|
||||||
|
let err_msg = result.unwrap_err().to_string();
|
||||||
|
assert!(err_msg.contains("Failed to read config file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_invalid_json_error() {
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
std::fs::write(tmp.path(), "not valid json {{{").unwrap();
|
||||||
|
|
||||||
|
let result = Config::load(tmp.path());
|
||||||
|
assert!(result.is_err());
|
||||||
|
let err_msg = result.unwrap_err().to_string();
|
||||||
|
assert!(err_msg.contains("Failed to parse config file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_valid_file() {
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
let content = r#"{
|
||||||
|
"duckdb_path": "test.db",
|
||||||
|
"bind_address": "127.0.0.1:9090",
|
||||||
|
"data_source_url": "http://example.com",
|
||||||
|
"log_file_path": null,
|
||||||
|
"allowed_origins": ["https://example.com"]
|
||||||
|
}"#;
|
||||||
|
std::fs::write(tmp.path(), content).unwrap();
|
||||||
|
|
||||||
|
let config = Config::load(tmp.path()).unwrap();
|
||||||
|
assert_eq!(config.duckdb_path, "test.db");
|
||||||
|
assert_eq!(config.bind_address, "127.0.0.1:9090");
|
||||||
|
assert_eq!(config.data_source_url, "http://example.com");
|
||||||
|
assert!(config.log_file_path.is_none());
|
||||||
|
assert_eq!(config.allowed_origins, vec!["https://example.com"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_load_defaults_allowed_origins() {
|
||||||
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
let content = r#"{
|
||||||
|
"duckdb_path": "test.db",
|
||||||
|
"bind_address": "127.0.0.1:9090",
|
||||||
|
"data_source_url": "http://example.com"
|
||||||
|
}"#;
|
||||||
|
std::fs::write(tmp.path(), content).unwrap();
|
||||||
|
|
||||||
|
let config = Config::load(tmp.path()).unwrap();
|
||||||
|
assert!(config.allowed_origins.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_default_values() {
|
||||||
|
let config = Config::default();
|
||||||
|
assert_eq!(config.duckdb_path, "hagfish.db");
|
||||||
|
assert_eq!(config.bind_address, "127.0.0.1:8090");
|
||||||
|
assert!(config.data_source_url.contains("statbank.hagstova.fo"));
|
||||||
|
assert!(config.log_file_path.is_some());
|
||||||
|
assert!(config.allowed_origins.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user