warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: ASRX hybrid pipeline, identity history, worker fixes, checkpoint system

Browse Source
This commit is contained in:
Accusys
2026-06-02 07:13:23 +08:00
parent e3066c3f49
commit e1572907ae
198 changed files with 43705 additions and 8910 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/trace_agent_api.rs
+332 -87
View File
@@ -25,14 +25,19 @@ pub fn trace_agent_routes() -> Router<crate::api::types::AppState> {
"/api/v1/file/:file_uuid/trace/:trace_id/thumbnail",
get(get_trace_thumbnail),
)
.route(
"/api/v1/file/:file_uuid/stranger/:stranger_id/representative-face",
get(get_stranger_representative_face),
)
.route(
"/api/v1/file/:file_uuid/stranger/:stranger_id/thumbnail",
get(get_stranger_thumbnail),
)
.route(
"/api/v1/file/:file_uuid/identities/:identity_uuid_a/co-occur-with/:identity_uuid_b",
get(get_cooccurrence),
)
.route(
"/api/v1/file/:file_uuid/tkg/rebuild",
post(rebuild_tkg),
)
.route("/api/v1/file/:file_uuid/tkg/rebuild", post(rebuild_tkg))
.route(
"/api/v1/file/:file_uuid/representative-frame",
get(get_representative_frame),
@@ -54,8 +59,8 @@ struct TracesRequest {
struct TraceInfo {
trace_id: i32,
face_count: i64,
start_frame: i32,
end_frame: i32,
start_frame: i64,
end_frame: i64,
start_time: f64,
end_time: f64,
duration_sec: f64,
@@ -110,8 +115,8 @@ async fn list_traces_sorted(
"SELECT tt.*, fd.id AS sample_face_id FROM (
SELECT trace_id::int AS trace_id,
COUNT(*) AS face_count,
MIN(frame_number)::int AS start_frame,
MAX(frame_number)::int AS end_frame,
MIN(frame_number)::bigint AS start_frame,
MAX(frame_number)::bigint AS end_frame,
(MAX(frame_number) - MIN(frame_number))::float8 AS duration_sec,
AVG(confidence)::float8 AS avg_confidence
FROM {}
@@ -132,7 +137,7 @@ async fn list_traces_sorted(
crate::core::db::schema::table_name("face_detections"),
);
let rows: Vec<(i32, i64, i32, i32, f64, f64, Option<i32>)> = sqlx::query_as(&query)
let rows: Vec<(i32, i64, i64, i64, f64, f64, Option<i32>)> = sqlx::query_as(&query)
.bind(&file_uuid)
.bind(min_faces)
.bind(effective_limit)
@@ -193,8 +198,8 @@ struct TraceFacesQuery {
#[derive(Debug, Serialize)]
struct TraceFaceItem {
id: i32,
start_frame: i32,
end_frame: i32,
start_frame: i64,
end_frame: i64,
start_time: f64,
end_time: f64,
x: Option<i32>,
@@ -260,14 +265,14 @@ async fn list_trace_faces(
let rows: Vec<(
i32,
i32,
i64,
Option<i32>,
Option<i32>,
Option<i32>,
Option<i32>,
f32,
)> = sqlx::query_as(&format!(
"SELECT id, frame_number::int, x, y, width, height, confidence::float4 \
"SELECT id, frame_number, x, y, width, height, confidence::float4 \
FROM {} WHERE file_uuid = $1 AND trace_id = $2 \
ORDER BY frame_number ASC LIMIT $3 OFFSET $4",
crate::core::db::schema::table_name("face_detections")
@@ -405,7 +410,8 @@ where
let video_table = schema::table_name("videos");
let fps: f64 = sqlx::query_scalar(&format!(
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1", video_table
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
video_table
))
.bind(file_uuid)
.fetch_optional(pool)
@@ -414,7 +420,8 @@ where
.unwrap_or(25.0);
let face_count: (i64,) = sqlx::query_as(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND trace_id = $2", fd_table
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND trace_id = $2",
fd_table
))
.bind(file_uuid)
.bind(trace_id)
@@ -422,7 +429,15 @@ where
.await
.map_err(|e| err_fn(anyhow::anyhow!("{}", e)))?;
struct Candidate { frame: i64, x: i32, y: i32, w: i32, h: i32, conf: f64, score: f64 }
struct Candidate {
frame: i64,
x: i32,
y: i32,
w: i32,
h: i32,
conf: f64,
score: f64,
}
let rows = sqlx::query_as::<_, (i64, i32, i32, i32, i32, f64)>(&format!(
"SELECT frame_number::bigint, x, y, width, height, confidence::float8 \
@@ -431,7 +446,8 @@ where
ORDER BY (width::float8 * height::float8) * confidence::float8 DESC LIMIT 10",
fd_table
))
.bind(file_uuid).bind(trace_id)
.bind(file_uuid)
.bind(trace_id)
.fetch_all(pool)
.await
.map_err(|e| err_fn(anyhow::anyhow!("{}", e)))?;
@@ -440,15 +456,25 @@ where
return Err(err_fn(anyhow::anyhow!("No suitable face found")));
}
let candidates: Vec<Candidate> = rows.into_iter()
let candidates: Vec<Candidate> = rows
.into_iter()
.map(|(frame, x, y, w, h, conf)| {
let score = (w as f64 * h as f64) * conf;
Candidate { frame, x, y, w, h, conf, score }
Candidate {
frame,
x,
y,
w,
h,
conf,
score,
}
})
.collect();
let video_path: String = sqlx::query_scalar(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1", video_table
"SELECT file_path FROM {} WHERE file_uuid = $1",
video_table
))
.bind(file_uuid)
.fetch_optional(pool)
@@ -463,16 +489,31 @@ where
for (i, c) in candidates.iter().enumerate() {
let seek = c.frame as f64 / fps;
if let Ok(output) = tokio::process::Command::new("ffmpeg")
.args(["-ss", &format!("{:.2}", seek), "-i", &video_path,
"-vframes", "1", "-vf", &format!("crop={}:{}:{}:{},blurdetect", c.w, c.h, c.x, c.y),
"-f", "null", "-"])
.output().await
.args([
"-ss",
&format!("{:.2}", seek),
"-i",
&video_path,
"-vframes",
"1",
"-vf",
&format!("crop={}:{}:{}:{},blurdetect", c.w, c.h, c.x, c.y),
"-f",
"null",
"-",
])
.output()
.await
{
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stderr.lines() {
if let Some(blur_str) = line.split("blur mean: ").nth(1) {
if let Ok(blur) = blur_str.trim().parse::<f64>() {
if blur < best_blur { best_blur = blur; best = c.frame; best_idx = i; }
if blur < best_blur {
best_blur = blur;
best = c.frame;
best_idx = i;
}
}
}
}
@@ -481,9 +522,17 @@ where
let chosen = &candidates[best_idx];
Ok(RepFaceSelection {
frame: chosen.frame, x: chosen.x, y: chosen.y, w: chosen.w, h: chosen.h,
conf: chosen.conf, blur: best_blur, score: chosen.score,
video_path, fps, face_count: face_count.0,
frame: chosen.frame,
x: chosen.x,
y: chosen.y,
w: chosen.w,
h: chosen.h,
conf: chosen.conf,
blur: best_blur,
score: chosen.score,
video_path,
fps,
face_count: face_count.0,
})
}
@@ -491,19 +540,36 @@ async fn get_representative_face(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
) -> Result<Json<RepFaceResponse>, (StatusCode, Json<serde_json::Value>)> {
let sel = select_rep_face(state.db.pool(), &file_uuid, trace_id, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await?;
get_representative_face_inner(&state, &file_uuid, trace_id).await
}
async fn get_representative_face_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
trace_id: i32,
) -> Result<Json<RepFaceResponse>, (StatusCode, Json<serde_json::Value>)> {
let sel = select_rep_face(state.db.pool(), file_uuid, trace_id, |e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await?;
Ok(Json(RepFaceResponse {
success: true,
file_uuid,
file_uuid: file_uuid.to_string(),
trace_id,
face_count: sel.face_count,
representative: RepFaceResult {
frame_number: sel.frame,
timestamp_secs: sel.frame as f64 / sel.fps,
bbox: RepFaceBbox { x: sel.x, y: sel.y, width: sel.w, height: sel.h },
bbox: RepFaceBbox {
x: sel.x,
y: sel.y,
width: sel.w,
height: sel.h,
},
confidence: sel.conf,
quality_score: sel.score,
blur_score: sel.blur,
@@ -515,34 +581,118 @@ async fn get_trace_thumbnail(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
get_trace_thumbnail_inner(&state, &file_uuid, trace_id).await
}
async fn get_trace_thumbnail_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
trace_id: i32,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
// Step 1: Check for pre-stored face crops in .faces/{file_uuid}/{trace_id}/
// For trace_id=0 (untracked/stranger), check unbound directory instead
let output_dir = crate::core::config::OUTPUT_DIR.as_str();
let trace_id_str = trace_id.to_string();
let trace_dir_name = if trace_id == 0 { "unbound" } else { &trace_id_str };
let trace_dir = std::path::PathBuf::from(output_dir)
.join(".faces")
.join(&file_uuid)
.join(trace_dir_name);
if trace_dir.exists() {
// Find any cached face crop in this trace directory
if let Ok(mut entries) = std::fs::read_dir(&trace_dir) {
while let Some(Ok(entry)) = entries.next() {
let path = entry.path();
if path.extension().map_or(false, |e| e == "jpg") {
tracing::info!("[trace_thumbnail] Using cached face crop: {}", path.display());
let bytes = tokio::fs::read(&path)
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
// Validate cached JPEG
crate::core::thumbnail::validator::validate_jpeg(&bytes).map_err(|e| {
tracing::warn!("[trace_thumbnail] Cached JPEG validation failed: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "Invalid cached JPEG"})),
)
})?;
return Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(bytes))
.unwrap());
}
}
}
}
// Step 2: Fallback to ffmpeg on-demand extraction
let sel = select_rep_face(state.db.pool(), &file_uuid, trace_id, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await?;
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await?;
let seek = sel.frame as f64 / sel.fps;
let tmp = std::env::temp_dir().join(format!("trace_{}_{}.jpg", file_uuid, trace_id));
tracing::debug!("[trace_thumbnail] Fallback to ffmpeg for trace {} frame {}", trace_id, sel.frame);
let status = tokio::process::Command::new("ffmpeg")
.args([
"-ss", &format!("{:.2}", seek),
"-i", &sel.video_path,
"-vframes", "1",
"-vf", &format!("crop={}:{}:{}:{},scale=320:320", sel.w, sel.h, sel.x, sel.y),
"-q:v", "2",
"-y", &tmp.to_string_lossy().to_string(),
"-ss",
&format!("{:.2}", seek),
"-i",
&sel.video_path,
"-vframes",
"1",
"-vf",
&format!("crop={}:{}:{}:{},scale=320:320", sel.w, sel.h, sel.x, sel.y),
"-q:v",
"2",
"-y",
&tmp.to_string_lossy().to_string(),
])
.output()
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
if !status.status.success() {
return Err((StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": "FFmpeg failed"}))));
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "FFmpeg failed"})),
));
}
let bytes = tokio::fs::read(&tmp).await.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
crate::core::thumbnail::validator::validate_jpeg(&bytes).map_err(|e| {
tracing::warn!("[trace_thumbnail] JPEG validation failed: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "Invalid JPEG output"})),
)
})?;
let _ = tokio::fs::remove_file(&tmp).await;
@@ -605,10 +755,16 @@ async fn get_cooccurrence(
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or_else(|| {
(StatusCode::NOT_FOUND, Json(serde_json::json!({"error": "Identity A not found"})))
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Identity A not found"})),
)
})?;
let id_b = sqlx::query_as::<_, (i32, String)>(&format!(
@@ -619,31 +775,38 @@ async fn get_cooccurrence(
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or_else(|| {
(StatusCode::NOT_FOUND, Json(serde_json::json!({"error": "Identity B not found"})))
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Identity B not found"})),
)
})?;
// Stage 2: Find first frame where both identity_ids appear
let cooccur: Option<(i64,)> = sqlx::query_as(
&format!(
"SELECT MIN(fd.frame_number)::bigint FROM {} fd \
let cooccur: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT MIN(fd.frame_number)::bigint FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.identity_id = $2 \
AND fd.frame_number IN ( \
SELECT frame_number FROM {} \
WHERE file_uuid = $1 AND identity_id = $3 \
)",
fd_table, fd_table
)
)
fd_table, fd_table
))
.bind(&file_uuid)
.bind(id_a.0)
.bind(id_b.0)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
let (first_frame,) = cooccur.ok_or_else(|| {
@@ -653,13 +816,17 @@ async fn get_cooccurrence(
// Get fps for timestamp
let video_table = schema::table_name("videos");
let fps: f64 = sqlx::query_scalar(&format!(
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1", video_table
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
video_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.unwrap_or(25.0);
@@ -685,40 +852,67 @@ async fn get_cooccurrence(
// Stage 4: Get representative faces for both traces (reusing select_rep_face)
let rep_a = if let Some((tid,)) = trace_a {
select_rep_face(state.db.pool(), &file_uuid, tid, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await.ok().map(|sel| CoOccurRepFace {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await
.ok()
.map(|sel| CoOccurRepFace {
frame_number: sel.frame,
bbox: RepFaceBbox { x: sel.x, y: sel.y, width: sel.w, height: sel.h },
bbox: RepFaceBbox {
x: sel.x,
y: sel.y,
width: sel.w,
height: sel.h,
},
confidence: sel.conf,
thumbnail_url: format!("/api/v1/file/{}/trace/{}/thumbnail", file_uuid, tid),
})
} else { None };
} else {
None
};
let rep_b = if let Some((tid,)) = trace_b {
select_rep_face(state.db.pool(), &file_uuid, tid, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await.ok().map(|sel| CoOccurRepFace {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await
.ok()
.map(|sel| CoOccurRepFace {
frame_number: sel.frame,
bbox: RepFaceBbox { x: sel.x, y: sel.y, width: sel.w, height: sel.h },
bbox: RepFaceBbox {
x: sel.x,
y: sel.y,
width: sel.w,
height: sel.h,
},
confidence: sel.conf,
thumbnail_url: format!("/api/v1/file/{}/trace/{}/thumbnail", file_uuid, tid),
})
} else { None };
} else {
None
};
// Total co-occurrence frames (from TKG if available, otherwise from face_detections)
let total_cooccurrence_frames: i64 = sqlx::query_scalar(
&format!(
"SELECT COUNT(DISTINCT fd.frame_number)::bigint FROM {} fd \
let total_cooccurrence_frames: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(DISTINCT fd.frame_number)::bigint FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.identity_id = $2 \
AND fd.frame_number IN ( \
SELECT frame_number FROM {} \
WHERE file_uuid = $1 AND identity_id = $3 \
)",
fd_table, fd_table
)
)
.bind(&file_uuid).bind(id_a.0).bind(id_b.0)
.fetch_one(state.db.pool()).await
fd_table, fd_table
))
.bind(&file_uuid)
.bind(id_a.0)
.bind(id_b.0)
.fetch_one(state.db.pool())
.await
.unwrap_or(0);
Ok(Json(CoOccurResponse {
@@ -758,12 +952,7 @@ async fn rebuild_tkg(
State(state): State<crate::api::types::AppState>,
Path(file_uuid): Path<String>,
) -> Json<TkgRebuildResponse> {
let result = crate::core::processor::tkg::build_tkg(
&state.db,
&file_uuid,
&OUTPUT_DIR,
)
.await;
let result = crate::core::processor::tkg::build_tkg(&state.db, &file_uuid, &OUTPUT_DIR).await;
match result {
Ok(r) => Json(TkgRebuildResponse {
@@ -807,14 +996,14 @@ async fn get_representative_frame(
State(state): State<crate::api::types::AppState>,
Path(file_uuid): Path<String>,
) -> Result<Json<RepFrameResponse>, (StatusCode, Json<serde_json::Value>)> {
let result = tkg::query_auto_representative_frame(
state.db.pool(),
&file_uuid,
)
.await
.map_err(|e| {
(StatusCode::NOT_FOUND, Json(serde_json::json!({"error": e.to_string()})))
})?;
let result = tkg::query_auto_representative_frame(state.db.pool(), &file_uuid)
.await
.map_err(|e| {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
let fps = query_fps(state.db.pool(), &file_uuid).await;
@@ -843,3 +1032,59 @@ async fn query_fps(pool: &sqlx::PgPool, file_uuid: &str) -> f64 {
.flatten()
.unwrap_or(25.0)
}
async fn get_stranger_representative_face(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, stranger_id)): Path<(String, i32)>,
) -> Result<Json<RepFaceResponse>, (StatusCode, Json<serde_json::Value>)> {
let faces_table = crate::core::db::schema::table_name("face_detections");
let trace_id: i32 = sqlx::query_scalar(&format!(
"SELECT trace_id FROM {} WHERE file_uuid = $1 AND stranger_id = $2 LIMIT 1",
faces_table
))
.bind(&file_uuid)
.bind(stranger_id)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or((
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Stranger not found"})),
))?;
get_representative_face_inner(&state, &file_uuid, trace_id).await
}
async fn get_stranger_thumbnail(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, stranger_id)): Path<(String, i32)>,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
let faces_table = crate::core::db::schema::table_name("face_detections");
let trace_id: i32 = sqlx::query_scalar(&format!(
"SELECT trace_id FROM {} WHERE file_uuid = $1 AND stranger_id = $2 LIMIT 1",
faces_table
))
.bind(&file_uuid)
.bind(stranger_id)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or((
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Stranger not found"})),
))?;
get_trace_thumbnail_inner(&state, &file_uuid, trace_id).await
}