warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: Phase 1 handover - schema migration, correction mechanism, API fixes

Browse Source 
Schema changes: dev.chunks->dev.chunk, remove old_chunk_id/chunk_index
Correction: asr-1.json format, generate/apply scripts
API: 37/37 endpoints fixed and tested
Docs: HANDOVER_V2.0.md for M4
This commit is contained in:
Accusys
2026-05-11 07:03:22 +08:00
parent ef894a44ad
commit 39ba5ddf76
147 changed files with 19843 additions and 3053 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/pipeline_status.py
+27 -14
View File
@@ -108,7 +108,7 @@ def check_job(uuid: str) -> dict:
stages = []
t0 = time.time()
# 1. ASR
# 1. ASR (pass 1: faster-whisper small)
t = time.time()
f = OUTPUT_DIR / f"{uuid}.asr.json"
ok = f.exists() and f.stat().st_size > 0
@@ -118,10 +118,10 @@ def check_job(uuid: str) -> dict:
with open(f) as fh: d = json.load(fh)
segs = len(d.get("segments", []))
except: ok = False
stages.append({"name": "ASR", "passed": ok and segs > 0, "detail": f"{segs} seg" if ok else file_size(str(f)),
stages.append({"name": "ASR", "passed": ok and segs > 0, "detail": f"faster-whisper ({segs})" if ok else file_size(str(f)),
"elapsed": round(time.time() - t, 1)})
# 2. ASRX
# 2. ASRX (ECAPA-TDNN speaker diarization)
t = time.time()
f = OUTPUT_DIR / f"{uuid}.asrx.json"
ok = f.exists() and f.stat().st_size > 0
@@ -131,15 +131,28 @@ def check_job(uuid: str) -> dict:
with open(f) as fh: d = json.load(fh)
segs = len(d.get("segments", []))
except: ok = False
stages.append({"name": "ASRX", "passed": ok and segs > 0, "detail": f"{segs} seg" if ok else file_size(str(f)),
stages.append({"name": "ASRX", "passed": ok and segs > 0, "detail": f"ECAPA-TDNN ({segs})" if ok else file_size(str(f)),
"elapsed": round(time.time() - t, 1)})
# 3. Sentence Chunks
# 3. ASR2 (pass 2: correct split segments)
t = time.time()
cnt = int(psql(f"SELECT count(*) FROM dev.chunks WHERE file_uuid='{uuid}' AND chunk_type='sentence'"))
stages.append({"name": "Sentence", "passed": cnt > 0, "detail": f"{cnt} chunks", "elapsed": round(time.time() - t, 1)})
f2 = OUTPUT_DIR / f"{uuid}.asr-1.json"
ok2 = f2.exists() and f2.stat().st_size > 0
cnt2 = 0
if ok2:
try:
with open(f2) as fh: d2 = json.load(fh)
cnt2 = len(d2.get("kept", [])) + sum(len(c["corrected"]) for c in d2.get("corrections", []))
except: ok2 = False
stages.append({"name": "ASR2", "passed": ok2 and cnt2 > 0, "detail": f"{cnt2} chunks (asr-1.json)" if ok2 else file_size(str(f2)),
"elapsed": round(time.time() - t, 1)})
# 4. Vectorization
# 4. Sentence Chunks (DB)
t = time.time()
cnt = int(psql(f"SELECT count(*) FROM dev.chunk WHERE file_uuid='{uuid}' AND chunk_type='sentence'"))
stages.append({"name": "Sentence", "passed": cnt > 0, "detail": f"{cnt} DB", "elapsed": round(time.time() - t, 1)})
# 5. Vectorization
t = time.time()
vec = int(psql(f"SELECT count(*) FROM dev.chunk_vectors WHERE uuid='{uuid}'"))
qdrant_ok = False
@@ -161,7 +174,7 @@ def check_job(uuid: str) -> dict:
"detail": f"{vec} PG, Qdrant={'ok' if qdrant_ok else '?'}",
"elapsed": round(time.time() - t, 1)})
# 5. Face Trace
# 6. Face Trace
t = time.time()
traces = int(psql(f"SELECT count(DISTINCT trace_id) FROM dev.face_detections WHERE file_uuid='{uuid}' AND trace_id IS NOT NULL"))
faces = int(psql(f"SELECT count(*) FROM dev.face_detections WHERE file_uuid='{uuid}' AND trace_id IS NOT NULL"))
@@ -169,7 +182,7 @@ def check_job(uuid: str) -> dict:
"detail": f"{traces} traces, {faces} faces",
"elapsed": round(time.time() - t, 1)})
# 6. TKG
# 7. TKG
t = time.time()
nodes = int(psql(f"SELECT count(*) FROM dev.tkg_nodes WHERE file_uuid='{uuid}'"))
edges = int(psql(f"SELECT count(*) FROM dev.tkg_edges WHERE file_uuid='{uuid}'"))
@@ -177,16 +190,16 @@ def check_job(uuid: str) -> dict:
"detail": f"{nodes} nodes, {edges} edges",
"elapsed": round(time.time() - t, 1)})
# 7. Trace Chunks
# 8. Trace Chunks
t = time.time()
tc = int(psql(f"SELECT count(*) FROM dev.chunks WHERE file_uuid='{uuid}' AND chunk_type='trace'"))
tc = int(psql(f"SELECT count(*) FROM dev.chunk WHERE file_uuid='{uuid}' AND chunk_type='trace'"))
stages.append({"name": "TraceChunks", "passed": tc > 0, "detail": f"{tc} chunks",
"elapsed": round(time.time() - t, 1)})
# 8. Phase 1 Release
# 9. Phase 1 Release
t = time.time()
p1 = PROJECT / "release" / "phase1" / "latest"
p1_files = [p1 / "RELEASE_INFO.txt", p1 / "chunks.csv", p1 / "vectors.csv"]
p1_files = [p1 / "RELEASE_INFO.txt", p1 / "schema.sql", p1 / "snapshots"]
p1_ok = all(f.exists() for f in p1_files)
p1_size = sum(f.stat().st_size for f in p1.rglob("*") if f.is_file()) // 1024 if p1.exists() else 0
stages.append({"name": "Phase1", "passed": p1_ok,