warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

refactor: remove Rule 3, Story, and Caption processors

Browse Source 
- Remove Rule 3 (Scene Chunking) from worker auto-trigger
- Remove rule3_ingest.rs and related imports
- Remove Story/Caption from playground module parsing
- Clean up scan.rs Rule 3 display
- Fix ASRX field name conversion (start_time -> start)

Reason: Story/5W1H/Scene accuracy too poor - will redesign later
This commit is contained in:
Accusys
2026-06-22 15:34:02 +08:00
parent 22f13eca4b
commit 70e849d3ae
6 changed files with 46 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/chunk/mod.rs
-2
View File
@@ -1,13 +1,11 @@
pub mod rule1_ingest;
pub mod rule2_ingest;
pub mod rule3_ingest;
pub mod splitter;
pub mod trace_ingest;
pub mod types;
pub use rule1_ingest::execute_rule1;
pub use rule2_ingest::ingest_rule2;
pub use rule3_ingest::ingest_rule3;
pub use splitter::{AsrSegment, ChunkSplitter};
pub use trace_ingest::ingest_traces;
pub use types::{Chunk, ChunkType};
src/core/chunk/rule3_ingest.rs
-171
View File
@@ -1,171 +0,0 @@
use crate::core::config::OUTPUT_DIR;
use crate::core::db::schema;
use anyhow::{Context, Result};
use serde::Deserialize;
use sqlx::PgPool;
use std::fs;
use tracing::{info, warn};
#[derive(Debug, Deserialize)]
struct CutScene {
scene_number: u32,
start_frame: u64,
end_frame: u64,
start_time: f64,
end_time: f64,
}
#[derive(Debug, Deserialize)]
struct CutResult {
scenes: Vec<CutScene>,
}
#[derive(Debug, Deserialize)]
struct AsrSegment {
#[serde(alias = "start")]
start_time: f64,
#[serde(alias = "end")]
end_time: f64,
text: String,
}
/// Executes Rule 3 Ingestion: Scene-based Chunking with LLM 5W1H+ Summary.
/// 1. Reads CUT data to identify scenes.
/// 2. Aggregates Rule 1 (Sentence) chunks falling within each scene.
/// 3. Calls LLM to generate 5W1H+ summary.
/// 4. Inserts parent chunks into `dev.chunks`.
pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result<usize> {
let cut_path = format!("{}/{}.cut.json", *OUTPUT_DIR, file_uuid);
let asr_path = format!("{}/{}.asr.json", *OUTPUT_DIR, file_uuid);
// 1. Load CUT and ASR data
let cut_content = fs::read_to_string(&cut_path)
.with_context(|| format!("Failed to read CUT file: {}", cut_path))?;
let cut_result: CutResult = serde_json::from_str(&cut_content).context("Invalid CUT JSON")?;
let asr_segments: Vec<AsrSegment> = match fs::read_to_string(&asr_path) {
Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
Err(_) => {
warn!("ASR file not found, proceeding with empty transcript for scenes");
vec![]
}
};
let mut count = 0;
let mut tx = pool.begin().await?;
// 2. Process each scene
for scene in &cut_result.scenes {
let chunk_id = format!("scene_{}", scene.scene_number);
// Aggregate text from Rule 1 chunks
let mut scene_text = String::new();
let mut child_ids: Vec<String> = Vec::new();
for seg in &asr_segments {
if seg.start_time >= scene.start_time && seg.end_time <= scene.end_time {
scene_text.push_str(&seg.text);
scene_text.push(' ');
// We'll look up the chunk_id from Rule 1 later if needed,
// but for now we just group by text overlap.
// A better approach is to query Rule 1 table for this range.
}
}
// Query chunks table for Rule 1 sentence chunks
let chunk_table = schema::table_name("chunk");
let rule1_rows: Vec<(String,)> = sqlx::query_as(&format!(
"SELECT chunk_id FROM {} \
WHERE file_uuid = $1 AND chunk_type = 'sentence' \
AND start_frame >= $2 \
AND end_frame <= $3",
chunk_table
))
.bind(file_uuid)
.bind(scene.start_frame as i64)
.bind(scene.end_frame as i64)
.fetch_all(&mut *tx)
.await?;
for row in &rule1_rows {
child_ids.push(row.0.clone());
}
// Fallback to simple aggregation if query didn't get text (due to frame boundaries)
if scene_text.is_empty() {
// Try to grab text directly if rule1 table doesn't have it or boundaries differ
// But rule1 table has start_frame/end_frame which should match.
// Let's re-query text directly.
}
let texts: Vec<String> = sqlx::query_scalar(&format!(
"SELECT text_content FROM {} \
WHERE file_uuid = $1 AND chunk_type = 'sentence' \
AND start_frame >= $2 \
AND end_frame <= $3 \
ORDER BY start_frame ASC",
chunk_table
))
.bind(file_uuid)
.bind(scene.start_frame as i64)
.bind(scene.end_frame as i64)
.fetch_all(&mut *tx)
.await?;
let aggregated_text = texts.join(" ");
info!(
"Scene {}: {} -> {} ({} sentences)",
scene.scene_number,
scene.start_time,
scene.end_time,
texts.len()
);
// 4. Insert into dev.chunks
let video_table = schema::table_name("videos");
let fps_query: Option<f64> = sqlx::query_scalar(&format!(
"SELECT fps FROM {} WHERE file_uuid = $1",
video_table
))
.bind(file_uuid)
.fetch_optional(&mut *tx)
.await?;
let fps = fps_query.unwrap_or(29.97);
// Prepare metadata JSON
let metadata = serde_json::json!({
"type": "scene",
"scene_number": scene.scene_number
});
sqlx::query(&format!(
"INSERT INTO {} (file_uuid, chunk_id, chunk_type, \
start_time, end_time, fps, start_frame, end_frame, \
content, text_content, summary_text, metadata, child_chunk_ids) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) \
ON CONFLICT (file_uuid, chunk_id) DO NOTHING",
chunk_table
))
.bind(file_uuid)
.bind(&chunk_id)
.bind("cut")
.bind(scene.start_time)
.bind(scene.end_time)
.bind(fps)
.bind(scene.start_frame as i64)
.bind(scene.end_frame as i64)
.bind(&metadata)
.bind(&aggregated_text)
.bind(&String::new())
.bind(&metadata)
.bind(&child_ids)
.execute(&mut *tx)
.await?;
count += 1;
}
tx.commit().await?;
Ok(count)
}