feat: trace-level matching, health watcher/worker status, timezone config
This commit is contained in:
Accusys
@@ -28,7 +28,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
DB_URL = os.getenv("DATABASE_URL", "postgresql://accusys@localhost:5432/momentry")
|
||||
SCHEMA = os.getenv("DATABASE_SCHEMA", "dev")
|
||||
OUTPUT_DIR = os.getenv("MOMENTRY_OUTPUT_DIR", "/Users/accusys/momentry/output_dev")
|
||||
OLLAMA_URL = "http://localhost:11434/api"
|
||||
EMBEDDING_URL = os.getenv("EMBEDDING_URL", "http://localhost:11436/v1/embeddings")
|
||||
|
||||
def load_speaker_map(file_uuid: str) -> dict:
|
||||
"""Load speaker→identity mapping from DB (generalized, not hardcoded)"""
|
||||
@@ -64,7 +64,7 @@ CURRENT_VERSIONS = {
|
||||
"embedding_agent": "nomic-embed-768d/v1",
|
||||
}
|
||||
|
||||
LLM_URL = os.getenv("MOMENTRY_LLM_SUMMARY_URL", "http://127.0.0.1:8081/v1/chat/completions")
|
||||
LLM_URL = os.getenv("MOMENTRY_LLM_URL", os.getenv("MOMENTRY_LLM_SUMMARY_URL", "http://127.0.0.1:8082/v1/chat/completions"))
|
||||
LLM_MODEL = os.getenv("MOMENTRY_LLM_SUMMARY_MODEL", "gemma4")
|
||||
|
||||
|
||||
@@ -97,7 +97,7 @@ def build_child_chunks(data: dict, file_uuid: str) -> List[dict]:
|
||||
s, e = cs["start_time"], cs["end_time"]
|
||||
|
||||
children = []
|
||||
for seg in asr_segs:
|
||||
for seg_idx, seg in enumerate(asr_segs):
|
||||
st, en = seg.get("start", 0), seg.get("end", 0)
|
||||
text = seg.get("text", "").strip()
|
||||
if st < s or en > e or not text: continue
|
||||
@@ -117,11 +117,11 @@ def build_child_chunks(data: dict, file_uuid: str) -> List[dict]:
|
||||
"start": st, "end": en, "text": text,
|
||||
"speaker_id": spk_id, "speaker_name": character,
|
||||
"speaker_confidence": spk_conf,
|
||||
"chunk_id": f"{file_uuid}_{st:.0f}_{en:.0f}",
|
||||
"chunk_id": f"{file_uuid}_{seg_idx}",
|
||||
})
|
||||
|
||||
# Boundary overlap: even empty scenes get partial children
|
||||
for seg in asr_segs:
|
||||
for seg_idx, seg in enumerate(asr_segs):
|
||||
st, en = seg.get("start", 0), seg.get("end", 0)
|
||||
text = seg.get("text", "").strip()
|
||||
if not text: continue
|
||||
@@ -141,7 +141,7 @@ def build_child_chunks(data: dict, file_uuid: str) -> List[dict]:
|
||||
"start": st, "end": en, "text": text,
|
||||
"speaker_id": spk_id, "speaker_name": character,
|
||||
"speaker_confidence": spk_conf,
|
||||
"chunk_id": f"{file_uuid}_{st:.0f}_{en:.0f}",
|
||||
"chunk_id": f"{file_uuid}_{seg_idx}",
|
||||
"overlap_type": "partial",
|
||||
})
|
||||
|
||||
@@ -215,14 +215,17 @@ def generate_llm_child_summary(child: dict, parent_summary: str) -> Optional[str
|
||||
# ===== Embedding (Ollama nomic-embed) =====
|
||||
|
||||
def embed_text(text: str, max_retries: int = 3) -> Optional[List[float]]:
|
||||
"""Get embedding via Ollama nomic-embed-text"""
|
||||
"""Get embedding via EmbeddingGemma server"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
resp = requests.post(f"{OLLAMA_URL}/embeddings", json={
|
||||
"model": "nomic-embed-text-v2-moe", "prompt": text,
|
||||
resp = requests.post(EMBEDDING_URL, json={
|
||||
"input": [text],
|
||||
}, timeout=30)
|
||||
if resp.status_code == 200:
|
||||
return resp.json()["embedding"]
|
||||
data = resp.json()
|
||||
items = data.get("data", [])
|
||||
if items:
|
||||
return items[0]["embedding"]
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
print(f" ⚠️ Embedding failed: {e}")
|
||||
@@ -244,7 +247,7 @@ def store_chunks(file_uuid: str, scenes: List[dict], mode: str, do_embed: bool,
|
||||
|
||||
# Get base chunk_index
|
||||
cur.execute(
|
||||
f"SELECT COALESCE(MAX(chunk_index), 0) FROM {SCHEMA}.chunks WHERE file_uuid = %s",
|
||||
f"SELECT COALESCE(MAX(chunk_index), 0) FROM {SCHEMA}.chunk WHERE file_uuid = %s",
|
||||
(file_uuid,),
|
||||
)
|
||||
next_index = (cur.fetchone()[0] or 0) + 1
|
||||
@@ -255,20 +258,38 @@ def store_chunks(file_uuid: str, scenes: List[dict], mode: str, do_embed: bool,
|
||||
|
||||
parent_id = f"{mode}_parent_{file_uuid}_{scene['start_time']:.0f}_{scene['end_time']:.0f}"
|
||||
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO {SCHEMA}.chunks (chunk_id, old_chunk_id, file_uuid, chunk_type, chunk_index,
|
||||
start_time, end_time, content, text_content, parent_chunk_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s)
|
||||
ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE
|
||||
SET content = EXCLUDED.content, text_content = EXCLUDED.text_content
|
||||
""",
|
||||
(parent_id, parent_id, file_uuid, parent_type, next_index,
|
||||
scene["start_time"], scene["end_time"],
|
||||
json.dumps({"summary": parent_text, "mode": mode, "type": "parent",
|
||||
"source_versions": CURRENT_VERSIONS}),
|
||||
parent_text, None),
|
||||
)
|
||||
parent_embedding = embed_text(parent_text) if do_embed else None
|
||||
if do_embed and parent_embedding:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO {SCHEMA}.chunk (chunk_id, old_chunk_id, file_uuid, chunk_type, chunk_index,
|
||||
start_time, end_time, content, text_content, parent_chunk_id, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s::vector)
|
||||
ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE
|
||||
SET content = EXCLUDED.content, text_content = EXCLUDED.text_content,
|
||||
embedding = EXCLUDED.embedding
|
||||
""",
|
||||
(parent_id, parent_id, file_uuid, parent_type, next_index,
|
||||
scene["start_time"], scene["end_time"],
|
||||
json.dumps({"summary": parent_text, "mode": mode, "type": "parent",
|
||||
"source_versions": CURRENT_VERSIONS}),
|
||||
parent_text, None, parent_embedding),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO {SCHEMA}.chunk (chunk_id, old_chunk_id, file_uuid, chunk_type, chunk_index,
|
||||
start_time, end_time, content, text_content, parent_chunk_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s)
|
||||
ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE
|
||||
SET content = EXCLUDED.content, text_content = EXCLUDED.text_content
|
||||
""",
|
||||
(parent_id, parent_id, file_uuid, parent_type, next_index,
|
||||
scene["start_time"], scene["end_time"],
|
||||
json.dumps({"summary": parent_text, "mode": mode, "type": "parent",
|
||||
"source_versions": CURRENT_VERSIONS}),
|
||||
parent_text, None),
|
||||
)
|
||||
next_index += 1
|
||||
parent_count += 1
|
||||
|
||||
@@ -276,22 +297,42 @@ def store_chunks(file_uuid: str, scenes: List[dict], mode: str, do_embed: bool,
|
||||
child_id = child["chunk_id"]
|
||||
child_text = generate_story_child_summary(child, parent_text) if mode == "story" else generate_llm_child_summary(child, parent_text)
|
||||
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO {SCHEMA}.chunks (chunk_id, old_chunk_id, file_uuid, chunk_type, chunk_index,
|
||||
start_time, end_time, content, text_content, parent_chunk_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s)
|
||||
ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE
|
||||
SET content = EXCLUDED.content, text_content = EXCLUDED.text_content,
|
||||
parent_chunk_id = EXCLUDED.parent_chunk_id
|
||||
""",
|
||||
(child_id, child_id, file_uuid, child_type, next_index,
|
||||
child["start"], child["end"],
|
||||
json.dumps({"speaker": child["speaker_name"], "text": child["text"], "mode": mode,
|
||||
"speaker_confidence": child.get("speaker_confidence", 0),
|
||||
"source_versions": CURRENT_VERSIONS}),
|
||||
child_text, parent_id),
|
||||
)
|
||||
child_embedding = embed_text(child_text) if do_embed else None
|
||||
if do_embed and child_embedding:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO {SCHEMA}.chunk (chunk_id, old_chunk_id, file_uuid, chunk_type, chunk_index,
|
||||
start_time, end_time, content, text_content, parent_chunk_id, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s::vector)
|
||||
ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE
|
||||
SET content = EXCLUDED.content, text_content = EXCLUDED.text_content,
|
||||
parent_chunk_id = EXCLUDED.parent_chunk_id,
|
||||
embedding = EXCLUDED.embedding
|
||||
""",
|
||||
(child_id, child_id, file_uuid, child_type, next_index,
|
||||
child["start"], child["end"],
|
||||
json.dumps({"speaker": child["speaker_name"], "text": child["text"], "mode": mode,
|
||||
"speaker_confidence": child.get("speaker_confidence", 0),
|
||||
"source_versions": CURRENT_VERSIONS}),
|
||||
child_text, parent_id, child_embedding),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
f"""
|
||||
INSERT INTO {SCHEMA}.chunk (chunk_id, old_chunk_id, file_uuid, chunk_type, chunk_index,
|
||||
start_time, end_time, content, text_content, parent_chunk_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s)
|
||||
ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE
|
||||
SET content = EXCLUDED.content, text_content = EXCLUDED.text_content,
|
||||
parent_chunk_id = EXCLUDED.parent_chunk_id
|
||||
""",
|
||||
(child_id, child_id, file_uuid, child_type, next_index,
|
||||
child["start"], child["end"],
|
||||
json.dumps({"speaker": child["speaker_name"], "text": child["text"], "mode": mode,
|
||||
"speaker_confidence": child.get("speaker_confidence", 0),
|
||||
"source_versions": CURRENT_VERSIONS}),
|
||||
child_text, parent_id),
|
||||
)
|
||||
next_index += 1
|
||||
child_count += 1
|
||||
|
||||
@@ -304,7 +345,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(description="Story Processor V2.0")
|
||||
parser.add_argument("--file-uuid", required=True)
|
||||
parser.add_argument("--mode", choices=["story", "llm"], default="story")
|
||||
parser.add_argument("--max-scenes", type=int, default=300)
|
||||
parser.add_argument("--max-scenes", type=int, default=99999)
|
||||
parser.add_argument("--embed", action="store_true", help="Generate embeddings (Ollama)")
|
||||
parser.add_argument("--no-db", action="store_true", help="Skip DB storage")
|
||||
args = parser.parse_args()
|
||||
|
||||
Reference in New Issue
Block a user