warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: trace-level matching, health watcher/worker status, timezone config

Browse Source
This commit is contained in:
Accusys
2026-05-21 01:08:30 +08:00
parent 8ede4be159
commit bebaa743ed
60 changed files with 6110 additions and 1586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/tmdb/cache.rs
+6 -4
View File
@@ -65,7 +65,11 @@ pub fn tmdb_cache_path(file_uuid: &str) -> PathBuf {
pub fn read_tmdb_cache(file_uuid: &str) -> Result<TmdbCache> {
let path = tmdb_cache_path(file_uuid);
if !path.exists() {
anyhow::bail!("TMDb cache not found: {} (expected: {})", file_uuid, path.display());
anyhow::bail!(
"TMDb cache not found: {} (expected: {})",
file_uuid,
path.display()
);
}
let content = std::fs::read_to_string(&path)
.with_context(|| format!("Failed to read TMDb cache: {}", path.display()))?;
@@ -96,9 +100,7 @@ pub fn count_cache_files() -> usize {
match std::fs::read_dir(&dir) {
Ok(entries) => entries
.filter_map(|e| e.ok())
.filter(|e| {
e.file_name().to_string_lossy().ends_with(".tmdb.json")
})
.filter(|e| e.file_name().to_string_lossy().ends_with(".tmdb.json"))
.count(),
Err(_) => 0,
}
src/core/tmdb/face_agent.rs
+41 -29
View File
@@ -46,11 +46,12 @@ pub async fn match_faces_against_tmdb(db: &PostgresDb, file_uuid: &str) -> Resul
// Step 2: Load face_detections grouped by trace_id
let fd_table = schema::table_name("face_detections");
let fd_rows = sqlx::query_as::<_, (i32, Vec<f32>)>(
&format!("SELECT trace_id, embedding FROM {} \
let fd_rows = sqlx::query_as::<_, (i32, Vec<f32>)>(&format!(
"SELECT trace_id, embedding FROM {} \
WHERE file_uuid=$1 AND trace_id IS NOT NULL AND embedding IS NOT NULL \
ORDER BY trace_id", fd_table),
)
ORDER BY trace_id",
fd_table
))
.bind(file_uuid)
.fetch_all(pool)
.await?;
@@ -156,9 +157,10 @@ pub async fn match_faces_against_tmdb(db: &PostgresDb, file_uuid: &str) -> Resul
let fd_table = schema::table_name("face_detections");
let mut after_qc = HashMap::new();
for (&tid, &(id, ref name)) in &matched {
let cnt: i64 = sqlx::query_scalar(
&format!("SELECT COUNT(*) FROM {} WHERE file_uuid=$1 AND trace_id=$2", fd_table),
)
let cnt: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid=$1 AND trace_id=$2",
fd_table
))
.bind(file_uuid)
.bind(tid)
.fetch_one(pool)
@@ -194,9 +196,10 @@ pub async fn match_faces_against_tmdb(db: &PostgresDb, file_uuid: &str) -> Resul
// Step 5: Update DB
let mut updated = 0usize;
for (&tid, &(id, _)) in &matched {
let r = sqlx::query(
&format!("UPDATE {} SET identity_id=$1 WHERE file_uuid=$2 AND trace_id=$3", fd_table),
)
let r = sqlx::query(&format!(
"UPDATE {} SET identity_id=$1 WHERE file_uuid=$2 AND trace_id=$3",
fd_table
))
.bind(id)
.bind(file_uuid)
.bind(tid)
@@ -223,9 +226,8 @@ pub async fn match_faces_against_tmdb(db: &PostgresDb, file_uuid: &str) -> Resul
async fn quality_check_temporal_collisions(pool: &sqlx::PgPool, file_uuid: &str) -> Result<usize> {
let fd_table = schema::table_name("face_detections");
// Find all collision pairs: same identity, same frame, different trace
let collisions = sqlx::query_as::<_, (i32, i32, i32, i32)>(
&format!(
"SELECT a.identity_id, a.trace_id, b.trace_id, a.frame_number \
let collisions = sqlx::query_as::<_, (i32, i32, i32, i32)>(&format!(
"SELECT a.identity_id, a.trace_id, b.trace_id, a.frame_number \
FROM {} a \
JOIN {} b \
ON a.file_uuid = b.file_uuid \
@@ -235,9 +237,8 @@ async fn quality_check_temporal_collisions(pool: &sqlx::PgPool, file_uuid: &str)
AND a.identity_id IS NOT NULL \
AND a.identity_id = b.identity_id \
ORDER BY a.identity_id, a.frame_number",
fd_table, fd_table
),
)
fd_table, fd_table
))
.bind(file_uuid)
.fetch_all(pool)
.await?;
@@ -256,25 +257,36 @@ async fn quality_check_temporal_collisions(pool: &sqlx::PgPool, file_uuid: &str)
let mut unbound = 0usize;
for ((id, ta, tb), overlap_frames) in &collision_groups {
// Get face detection count for each trace
let cnt_a: i64 = sqlx::query_scalar(
&format!("SELECT COUNT(*) FROM {} WHERE file_uuid=$1 AND trace_id=$2 AND identity_id=$3", fd_table)
)
.bind(file_uuid).bind(ta).bind(id)
.fetch_one(pool).await.unwrap_or(0);
let cnt_a: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid=$1 AND trace_id=$2 AND identity_id=$3",
fd_table
))
.bind(file_uuid)
.bind(ta)
.bind(id)
.fetch_one(pool)
.await
.unwrap_or(0);
let cnt_b: i64 = sqlx::query_scalar(
&format!("SELECT COUNT(*) FROM {} WHERE file_uuid=$1 AND trace_id=$2 AND identity_id=$3", fd_table)
)
.bind(file_uuid).bind(tb).bind(id)
.fetch_one(pool).await.unwrap_or(0);
let cnt_b: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid=$1 AND trace_id=$2 AND identity_id=$3",
fd_table
))
.bind(file_uuid)
.bind(tb)
.bind(id)
.fetch_one(pool)
.await
.unwrap_or(0);
// Unbind the trace with fewer detections (likely the false positive)
let victim = if cnt_a <= cnt_b { *ta } else { *tb };
let victim_cnt = if cnt_a <= cnt_b { cnt_a } else { cnt_b };
sqlx::query(
&format!("UPDATE {} SET identity_id=NULL WHERE file_uuid=$1 AND trace_id=$2", fd_table),
)
sqlx::query(&format!(
"UPDATE {} SET identity_id=NULL WHERE file_uuid=$1 AND trace_id=$2",
fd_table
))
.bind(file_uuid)
.bind(victim)
.execute(pool)
src/core/tmdb/probe.rs
+45 -17
View File
@@ -45,7 +45,14 @@ fn extract_movie_name(filename: &str) -> Option<String> {
.file_stem()
.and_then(|s| s.to_str())?;
let cleaned = name.replace(['.', '_'], " ").trim().to_string();
// Take only the part before year patterns or separators
let cleaned = name
.replace(['.', '_'], " ")
.split(|c: char| c == '(' || c == '[' || c == '│' || c == '|')
.next()
.unwrap_or(&name)
.trim()
.to_string();
if cleaned.is_empty() || cleaned.len() < 3 {
return None;
@@ -53,10 +60,7 @@ fn extract_movie_name(filename: &str) -> Option<String> {
Some(cleaned)
}
pub async fn probe_from_cache(
db: &PostgresDb,
file_uuid: &str,
) -> Result<TmdbProbeResult> {
pub async fn probe_from_cache(db: &PostgresDb, file_uuid: &str) -> Result<TmdbProbeResult> {
let cache = crate::core::tmdb::cache::read_tmdb_cache(file_uuid)?;
if cache.identities.is_empty() && !cache.cast.is_empty() {
return create_identities_from_data(db, file_uuid, &cache.movie, &cache.cast).await;
@@ -83,7 +87,8 @@ async fn upsert_identities_from_disk(
}
match std::fs::read_to_string(&path) {
Ok(content) => {
match serde_json::from_str::<crate::core::identity::storage::IdentityFile>(&content) {
match serde_json::from_str::<crate::core::identity::storage::IdentityFile>(&content)
{
Ok(identity_file) => {
let identities_table = crate::core::db::schema::table_name("identities");
let result = sqlx::query(&format!(
@@ -106,21 +111,35 @@ async fn upsert_identities_from_disk(
match result {
Ok(_) => {
info!("[TMDB] Upserted identity: {} (uuid={})", identity_file.name, identity_file.identity_uuid);
info!(
"[TMDB] Upserted identity: {} (uuid={})",
identity_file.name, identity_file.identity_uuid
);
identities_created += 1;
}
Err(e) => {
warn!("[TMDB] Failed to upsert identity '{}': {}", identity_file.name, e);
warn!(
"[TMDB] Failed to upsert identity '{}': {}",
identity_file.name, e
);
}
}
}
Err(e) => {
warn!("[TMDB] Failed to parse identity file {}: {}", path.display(), e);
warn!(
"[TMDB] Failed to parse identity file {}: {}",
path.display(),
e
);
}
}
}
Err(e) => {
warn!("[TMDB] Failed to read identity file {}: {}", path.display(), e);
warn!(
"[TMDB] Failed to read identity file {}: {}",
path.display(),
e
);
}
}
}
@@ -181,7 +200,9 @@ pub async fn create_identities_from_data(
continue;
}
let profile_url = member.profile_path.as_ref()
let profile_url = member
.profile_path
.as_ref()
.map(|p| format!("https://image.tmdb.org/t/p/w185{}", p));
let metadata = serde_json::json!({
@@ -226,8 +247,13 @@ pub async fn create_identities_from_data(
member.name, member.character, uuid_str
);
identities_created += 1;
if let Err(e) = crate::core::identity::storage::save_identity_file(db, &uuid_str).await {
warn!("[TMDB] Failed to save identity file for {}: {}", member.name, e);
if let Err(e) =
crate::core::identity::storage::save_identity_file(db, &uuid_str).await
{
warn!(
"[TMDB] Failed to save identity file for {}: {}",
member.name, e
);
}
// Download and save TMDb profile image locally
if let Some(url) = &profile_url {
@@ -393,8 +419,10 @@ pub async fn probe_movie(
overview: movie.overview.clone(),
poster_path: movie.poster_path.clone(),
};
let cache_cast: Vec<cache::TmdbCastMember> = credits.cast.iter().map(|m| {
cache::TmdbCastMember {
let cache_cast: Vec<cache::TmdbCastMember> = credits
.cast
.iter()
.map(|m| cache::TmdbCastMember {
id: m.id,
name: m.name.clone(),
character: m.character.clone(),
@@ -410,8 +438,8 @@ pub async fn probe_movie(
deathday: None,
gender: None,
homepage: None,
}
}).collect();
})
.collect();
// Write TMDb cache so probe_from_cache can be used next time
let cache_obj = cache::TmdbCache {
src/core/tmdb/status.rs
+13 -7
View File
@@ -60,7 +60,11 @@ pub async fn check_tmdb_api() -> TmdbResourceStatus {
enabled: *config::tmdb::PROBE_ENABLED,
api_reachable: Some(reachable),
api_latency_ms: Some(latency),
api_error: if reachable { None } else { Some(format!("HTTP {}", resp.status())) },
api_error: if reachable {
None
} else {
Some(format!("HTTP {}", resp.status()))
},
last_check_at: Some(chrono::Utc::now().to_rfc3339()),
}
}
@@ -84,9 +88,10 @@ pub fn count_cache_files() -> usize {
pub async fn count_tmdb_identities(pool: &sqlx::PgPool) -> Result<i64> {
let identities_table = crate::core::db::schema::table_name("identities");
let count: i64 = sqlx::query_scalar(
&format!("SELECT COUNT(*) FROM {} WHERE source = 'tmdb'", identities_table)
)
let count: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE source = 'tmdb'",
identities_table
))
.fetch_one(pool)
.await?;
Ok(count)
@@ -94,9 +99,10 @@ pub async fn count_tmdb_identities(pool: &sqlx::PgPool) -> Result<i64> {
pub async fn count_tmdb_identities_with_embedding(pool: &sqlx::PgPool) -> Result<i64> {
let identities_table = crate::core::db::schema::table_name("identities");
let count: i64 = sqlx::query_scalar(
&format!("SELECT COUNT(*) FROM {} WHERE source = 'tmdb' AND face_embedding IS NOT NULL", identities_table)
)
let count: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE source = 'tmdb' AND face_embedding IS NOT NULL",
identities_table
))
.fetch_one(pool)
.await?;
Ok(count)