warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: trace-level matching, health watcher/worker status, timezone config

Browse Source
This commit is contained in:
Accusys
2026-05-21 01:08:30 +08:00
parent 8ede4be159
commit bebaa743ed
60 changed files with 6110 additions and 1586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/tmdb_api.rs
+445 -21
View File
@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
use crate::api::server::AppState;
use crate::core::config;
use crate::core::db::PostgresDb;
use crate::core::db::{PostgresDb, QdrantDb};
use crate::core::tmdb;
#[derive(Debug, Serialize)]
@@ -64,10 +64,44 @@ struct FileUuidParam {
file_uuid: String,
}
#[derive(Debug, Deserialize)]
struct TmdbFetchRequest {
file_uuid: String,
}
#[derive(Debug, Serialize)]
struct TmdbFetchMemberResult {
name: String,
character: Option<String>,
aliases: Vec<String>,
metadata: serde_json::Value,
status: String,
has_json: bool,
has_jpg: bool,
error: Option<String>,
}
#[derive(Debug, Serialize)]
struct TmdbFetchResponse {
success: bool,
movie_title: Option<String>,
tmdb_id: Option<u64>,
results: Vec<TmdbFetchMemberResult>,
summary: serde_json::Value,
}
pub fn tmdb_routes() -> Router<AppState> {
Router::new()
.route("/api/v1/agents/tmdb/prefetch", post(tmdb_prefetch))
.route("/api/v1/file/:file_uuid/tmdb-probe", post(tmdb_probe_handler))
.route(
"/api/v1/file/:file_uuid/tmdb-probe",
post(tmdb_probe_handler),
)
.route("/api/v1/tmdb/fetch", post(tmdb_fetch))
.route(
"/api/v1/agents/tmdb/match/:file_uuid",
post(tmdb_match_handler),
)
.route("/api/v1/resource/tmdb", get(tmdb_resource_status))
.route("/api/v1/resource/tmdb/check", post(tmdb_resource_check))
}
@@ -79,9 +113,10 @@ async fn tmdb_prefetch(
let file_uuid = req.file_uuid;
// Verify file exists in DB
let file_exists: bool = sqlx::query_scalar(
&format!("SELECT COUNT(*) > 0 FROM {} WHERE file_uuid = $1", crate::core::db::schema::table_name("videos"))
)
let file_exists: bool = sqlx::query_scalar(&format!(
"SELECT COUNT(*) > 0 FROM {} WHERE file_uuid = $1",
crate::core::db::schema::table_name("videos")
))
.bind(&file_uuid)
.fetch_one(state.db.pool())
.await
@@ -182,18 +217,22 @@ async fn tmdb_probe_handler(
let file_uuid = params.file_uuid;
// Verify file exists
let file_exists: bool = sqlx::query_scalar(
&format!("SELECT COUNT(*) > 0 FROM {} WHERE file_uuid = $1", crate::core::db::schema::table_name("videos"))
)
let file_exists: bool = sqlx::query_scalar(&format!(
"SELECT COUNT(*) > 0 FROM {} WHERE file_uuid = $1",
crate::core::db::schema::table_name("videos")
))
.bind(&file_uuid)
.fetch_one(state.db.pool())
.await
.unwrap_or(false);
if !file_exists {
return Err((StatusCode::NOT_FOUND, Json(serde_json::json!({
"error": "Video not found", "file_uuid": file_uuid
}))));
return Err((
StatusCode::NOT_FOUND,
Json(serde_json::json!({
"error": "Video not found", "file_uuid": file_uuid
})),
));
}
match tmdb::probe::probe_from_cache(&state.db, &file_uuid).await {
@@ -214,7 +253,10 @@ async fn tmdb_probe_handler(
.await
{
for uuid in rows {
let _ = crate::core::identity::storage::save_identity_file_by_pool(&pool, &uuid).await;
let _ = crate::core::identity::storage::save_identity_file_by_pool(
&pool, &uuid,
)
.await;
}
}
});
@@ -245,24 +287,26 @@ async fn tmdb_probe_handler(
message: "No TMDb cache found. Run tmdb-prefetch first.".to_string(),
}))
} else {
Err((StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({
"error": msg, "file_uuid": file_uuid
}))))
Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": msg, "file_uuid": file_uuid
})),
))
}
}
}
}
async fn tmdb_resource_status(
State(state): State<AppState>,
) -> Json<TmdbResourceResponse> {
async fn tmdb_resource_status(State(state): State<AppState>) -> Json<TmdbResourceResponse> {
let status = tmdb::status::quick_status();
let identities_seeded = tmdb::status::count_tmdb_identities(state.db.pool())
.await
.unwrap_or(0);
let identities_with_embedding = tmdb::status::count_tmdb_identities_with_embedding(state.db.pool())
.await
.unwrap_or(0);
let identities_with_embedding =
tmdb::status::count_tmdb_identities_with_embedding(state.db.pool())
.await
.unwrap_or(0);
let cache_files = tmdb::status::count_cache_files();
Json(TmdbResourceResponse {
@@ -303,3 +347,383 @@ async fn tmdb_resource_check() -> Json<TmdbCheckResponse> {
status,
})
}
async fn tmdb_fetch(
State(state): State<AppState>,
Json(req): Json<TmdbFetchRequest>,
) -> Result<Json<TmdbFetchResponse>, (StatusCode, Json<serde_json::Value>)> {
let file_uuid = req.file_uuid;
let filename: Option<String> = sqlx::query_scalar(&format!(
"SELECT file_name FROM {} WHERE file_uuid = $1",
crate::core::db::schema::table_name("videos")
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.flatten();
let filename = filename.ok_or_else(|| {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "File not found"})),
)
})?;
// Run probe to create identities
match tmdb::probe::probe_movie(&state.db, &filename, &file_uuid).await {
Ok(Some(probe_result)) => {
let mut member_results = Vec::new();
// Read the cache to get cast list with names and profile URLs
if let Ok(cache) = tmdb::cache::read_tmdb_cache(&file_uuid) {
for member in &cache.cast {
let name = member.name.clone();
let character = if member.character.is_empty() {
None
} else {
Some(member.character.clone())
};
let aliases = member.also_known_as.clone();
let profile_url = member
.profile_path
.as_ref()
.map(|p| format!("https://image.tmdb.org/t/p/w185{}", p));
let metadata = serde_json::json!({
"tmdb_id": member.id,
"name": member.name,
"character": member.character,
"aliases": member.also_known_as,
"profile_path": member.profile_path,
"order": member.order,
"biography": member.biography,
"birthday": member.birthday,
"place_of_birth": member.place_of_birth,
"imdb_id": member.imdb_id,
"known_for_department": member.known_for_department,
"popularity": member.popularity,
"deathday": member.deathday,
"gender": member.gender,
"homepage": member.homepage,
});
let identity_row = sqlx::query_as::<_, (i32, uuid::Uuid)>(&format!(
"SELECT id, uuid FROM {} WHERE name = $1 AND source = 'tmdb' LIMIT 1",
crate::core::db::schema::table_name("identities")
))
.bind(&name)
.fetch_optional(state.db.pool())
.await;
match identity_row {
Ok(Some((identity_id, uuid))) => {
let clean = uuid.to_string().replace('-', "");
let dir = crate::core::identity::storage::identity_dir(&clean);
std::fs::create_dir_all(&dir).ok();
let json_result = crate::core::identity::storage::save_identity_file(
&state.db, &clean,
)
.await;
let has_json = json_result.is_ok();
let has_jpg = if let Some(url) = &profile_url {
let jpg_path = dir.join("profile.jpg");
if jpg_path.exists() {
true
} else if let Ok(resp) = reqwest::get(url).await {
if let Ok(bytes) = resp.bytes().await {
std::fs::write(&jpg_path, &bytes).is_ok()
} else {
false
}
} else {
false
}
} else {
false
};
// Push face_embedding to Qdrant if available
let face_collection = format!(
"{}_faces",
crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':')
);
let emb_row: Option<(Vec<f32>,)> = sqlx::query_as(
&format!(
"SELECT face_embedding::real[] FROM {} WHERE uuid = $1 AND face_embedding IS NOT NULL",
crate::core::db::schema::table_name("identities")
)
)
.bind(&uuid)
.fetch_optional(state.db.pool())
.await
.unwrap_or(None);
if let Some((embedding,)) = emb_row {
let qdrant = QdrantDb::new();
qdrant.ensure_collection(&face_collection, 512).await.ok();
let _ = qdrant
.upsert_vector_to_collection(
&face_collection,
identity_id as u64,
&embedding,
Some(serde_json::json!({
"identity_id": identity_id,
"name": name,
"source": "tmdb",
})),
)
.await;
}
let status = if has_json && has_jpg {
"success"
} else {
"partial"
};
let error = if !has_json {
Some(format!("{:?}", json_result.err()))
} else if !has_jpg {
Some("profile download failed".to_string())
} else {
None
};
member_results.push(TmdbFetchMemberResult {
name: name.clone(),
character: character.clone(),
aliases: aliases.clone(),
metadata: metadata.clone(),
status: status.to_string(),
has_json,
has_jpg,
error,
});
}
Ok(None) => {
member_results.push(TmdbFetchMemberResult {
name: name.clone(),
character: character.clone(),
aliases: aliases.clone(),
metadata: metadata.clone(),
status: "skipped".to_string(),
has_json: false,
has_jpg: false,
error: None,
});
}
Err(e) => {
member_results.push(TmdbFetchMemberResult {
name: name.clone(),
character: character.clone(),
aliases: aliases.clone(),
metadata: metadata.clone(),
status: "error".to_string(),
has_json: false,
has_jpg: false,
error: Some(format!("DB error: {}", e)),
});
}
}
}
}
let total = member_results.len();
let success_count = member_results
.iter()
.filter(|r| r.status == "success")
.count();
let json_count = member_results.iter().filter(|r| r.has_json).count();
let jpg_count = member_results.iter().filter(|r| r.has_jpg).count();
Ok(Json(TmdbFetchResponse {
success: true,
movie_title: Some(probe_result.title),
tmdb_id: Some(probe_result.tmdb_id),
results: member_results,
summary: serde_json::json!({
"total": total,
"success": success_count,
"with_json": json_count,
"with_jpg": jpg_count,
}),
}))
}
Ok(None) => Err((
StatusCode::NOT_FOUND,
Json(serde_json::json!({
"error": "No movie found for this filename"
})),
)),
Err(e) => Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": e.to_string()
})),
)),
}
}
#[derive(Debug, Serialize)]
struct TmdbMatchResponse {
success: bool,
file_uuid: String,
bindings_created: usize,
tmdb_identities_available: usize,
message: String,
}
async fn tmdb_match_handler(
Path(params): Path<FileUuidParam>,
State(state): State<AppState>,
) -> Result<Json<TmdbMatchResponse>, (StatusCode, Json<serde_json::Value>)> {
let file_uuid = params.file_uuid;
// Verify file exists
let file_exists: bool = sqlx::query_scalar(&format!(
"SELECT COUNT(*) > 0 FROM {} WHERE file_uuid = $1",
crate::core::db::schema::table_name("videos")
))
.bind(&file_uuid)
.fetch_one(state.db.pool())
.await
.unwrap_or(false);
if !file_exists {
return Err((
StatusCode::NOT_FOUND,
Json(serde_json::json!({
"error": "Video not found", "file_uuid": file_uuid
})),
));
}
// Get all TMDb identities with face_embedding
let tmdb_rows = sqlx::query_as::<_, (i32, String, Vec<f32>)>(
&format!(
"SELECT id, name, face_embedding::real[] FROM {} WHERE source='tmdb' AND face_embedding IS NOT NULL",
crate::core::db::schema::table_name("identities")
)
)
.fetch_all(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
})?;
if tmdb_rows.is_empty() {
return Ok(Json(TmdbMatchResponse {
success: true,
file_uuid,
bindings_created: 0,
tmdb_identities_available: 0,
message: "No TMDb identities with face embeddings".to_string(),
}));
}
let face_collection = format!(
"{}_faces",
crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':')
);
let qdrant = QdrantDb::new();
let _ = qdrant.ensure_collection(&face_collection, 512).await;
let trace_collection = format!(
"{}_traces",
crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':')
);
let _ = qdrant.ensure_collection(&trace_collection, 512).await;
// Sync trace embeddings (idempotent)
if let Err(e) = crate::core::db::qdrant_db::sync_trace_embeddings(&file_uuid).await {
tracing::error!("[TKG-MATCH] Trace sync failed: {}", e);
}
let mut total_bindings = 0usize;
for (tmdb_id, tmdb_name, tmdb_embedding) in &tmdb_rows {
// Search Qdrant trace collection with this TMDb embedding
let results = match qdrant
.search_face_collection(
&trace_collection,
tmdb_embedding,
100,
"source",
"tmdb",
Some(&file_uuid),
)
.await
{
Ok(r) => r,
Err(e) => {
tracing::warn!("[TKG-MATCH] Qdrant search failed for {}: {}", tmdb_name, e);
continue;
}
};
// Filter results by threshold and file_uuid
let filtered: Vec<_> = results
.into_iter()
.filter(|(score, payload)| {
*score >= 0.50
&& payload.get("file_uuid").and_then(|v| v.as_str()) == Some(&file_uuid)
})
.collect();
if filtered.is_empty() {
continue;
}
// Bind matched traces directly
let mut bound_count = 0usize;
for (_score, payload) in &filtered {
if let Some(tid) = payload.get("trace_id").and_then(|v| v.as_i64()) {
let r = sqlx::query(&format!(
"UPDATE {} SET identity_id=$1 WHERE file_uuid=$2 AND trace_id=$3",
crate::core::db::schema::table_name("face_detections")
))
.bind(tmdb_id)
.bind(&file_uuid)
.bind(tid as i32)
.execute(state.db.pool())
.await;
if let Ok(result) = r {
bound_count += result.rows_affected() as usize;
}
}
}
if bound_count > 0 {
tracing::info!(
"[TKG-MATCH] {}: bound {} traces to TMDb identity {}",
tmdb_name,
bound_count,
tmdb_id
);
}
total_bindings += bound_count;
}
Ok(Json(TmdbMatchResponse {
success: true,
file_uuid,
bindings_created: total_bindings,
tmdb_identities_available: tmdb_rows.len(),
message: format!("{} traces matched to TMDb identities", total_bindings),
}))
}