momentry_core/src/api/scan.rs

use axum::{
    extract::{Path, Query, State},
    http::StatusCode,
    response::Json,
    routing::get,
    Router,
};
use serde::{Deserialize, Serialize};

use super::types::AppState;
use crate::core::db::schema;

#[derive(Debug, Serialize, Deserialize)]
struct ScannedFileInfo {
    file_name: String,
    relative_path: String,
    file_path: String,
    file_size: u64,
    modified_time: String,
    is_registered: bool,
    file_uuid: Option<String>,
    status: Option<String>,
    registration_time: Option<String>,
    job_id: Option<i32>,
}

#[derive(Debug, Serialize, Deserialize)]
struct ScanFilesResponse {
    files: Vec<ScannedFileInfo>,
    total: usize,
    filtered_total: usize,
    page: usize,
    page_size: usize,
    total_pages: usize,
    registered_count: usize,
    unregistered_count: usize,
    total_chunks: i64,
    searchable_chunks: i64,
    pending_videos: i64,
}

#[derive(Debug, Deserialize)]
struct ScanFilesQuery {
    limit: Option<usize>,
    page: Option<usize>,
    page_size: Option<usize>,
    pattern: Option<String>,
    sort_by: Option<String>,
    sort_order: Option<String>,
}

fn scan_directory_recursive(
    dir: &std::path::Path,
    root: &std::path::Path,
    allowed_extensions: &[&str],
    registered_paths: &std::collections::HashMap<
        String,
        (String, String, Option<String>, Option<i32>),
    >,
    files: &mut Vec<ScannedFileInfo>,
) {
    if let Ok(entries) = std::fs::read_dir(dir) {
        for entry in entries.flatten() {
            let path = entry.path();
            if path.is_dir() {
                if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
                    if name.starts_with('.') {
                        return;
                    }
                }
                scan_directory_recursive(&path, root, allowed_extensions, registered_paths, files);
            } else if path.is_file() {
                if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
                    if allowed_extensions.contains(&ext.to_lowercase().as_str()) {
                        if let Ok(meta) = entry.metadata() {
                            let abs_path = path.to_string_lossy().to_string();
                            let rel_path = path
                                .strip_prefix(root)
                                .map(|p| p.to_string_lossy().to_string())
                                .unwrap_or_else(|_| abs_path.clone());

                            let file_name = path
                                .file_name()
                                .map(|n| n.to_string_lossy().to_string())
                                .unwrap_or_default();

                            let modified_time = meta
                                .modified()
                                .ok()
                                .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
                                .map(|d| {
                                    chrono::DateTime::from_timestamp(d.as_secs() as i64, 0)
                                        .map(|dt| dt.to_rfc3339())
                                        .unwrap_or_default()
                                })
                                .unwrap_or_default();

                            match registered_paths.get(&abs_path) {
                                Some((uuid, status, reg_time, jid)) if status != "unregistered" => {
                                    files.push(ScannedFileInfo {
                                        file_name,
                                        relative_path: rel_path,
                                        file_path: abs_path,
                                        file_size: meta.len(),
                                        modified_time,
                                        is_registered: true,
                                        file_uuid: Some(uuid.clone()),
                                        status: Some(status.clone()),
                                        registration_time: reg_time.clone(),
                                        job_id: *jid,
                                    });
                                }
                                _ => {
                                    files.push(ScannedFileInfo {
                                        file_name,
                                        relative_path: rel_path,
                                        file_path: abs_path,
                                        file_size: meta.len(),
                                        modified_time,
                                        is_registered: false,
                                        file_uuid: None,
                                        status: Some("unregistered".to_string()),
                                        registration_time: None,
                                        job_id: None,
                                    });
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

async fn scan_files(
    State(state): State<AppState>,
    Query(params): Query<ScanFilesQuery>,
) -> Result<Json<ScanFilesResponse>, StatusCode> {
    let demo_dir_str = std::env::var("MOMENTRY_SFTP_ROOT")
        .unwrap_or_else(|_| "/Users/accusys/momentry/var/sftpgo/data/demo".to_string());
    let demo_dir = std::path::Path::new(&demo_dir_str);

    let allowed_extensions = vec![
        "mp4", "mov", "mkv", "avi", "webm", "jpg", "jpeg", "png", "gif", "webp",
    ];

    let table = schema::table_name("videos");
    let mj_table = schema::table_name("monitor_jobs");
    let registered_db: Vec<(String, String, String, String, Option<String>, Option<i32>)> =
        sqlx::query_as(&format!(
            "SELECT v.file_path, v.file_name, v.file_uuid, v.status, v.registration_time::text, \
                latest_job.id as job_id \
         FROM {} v \
         LEFT JOIN LATERAL ( \
             SELECT id FROM {} WHERE uuid = v.file_uuid ORDER BY id DESC LIMIT 1 \
         ) latest_job ON true \
         ORDER BY v.id",
            table, mj_table
        ))
        .fetch_all(state.db.pool())
        .await
        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;

    let registered_paths: std::collections::HashMap<
        String,
        (String, String, Option<String>, Option<i32>),
    > = registered_db
        .into_iter()
        .map(|(path, _name, uuid, status, reg_time, jid)| (path, (uuid, status, reg_time, jid)))
        .collect();

    let mut result_files = Vec::new();

    if demo_dir.exists() {
        scan_directory_recursive(
            demo_dir,
            demo_dir,
            &allowed_extensions,
            &registered_paths,
            &mut result_files,
        );
    }

    let desc = params.sort_order.as_deref().unwrap_or("asc") == "desc";
    match params.sort_by.as_deref().unwrap_or("name") {
        "size" => {
            if desc {
                result_files.sort_by(|a, b| b.file_size.cmp(&a.file_size));
            } else {
                result_files.sort_by(|a, b| a.file_size.cmp(&b.file_size));
            }
        }
        "modified" | "time" => {
            if desc {
                result_files.sort_by(|a, b| b.modified_time.cmp(&a.modified_time));
            } else {
                result_files.sort_by(|a, b| a.modified_time.cmp(&b.modified_time));
            }
        }
        "status" => {
            if desc {
                result_files
                    .sort_by(|a, b| b.status.cmp(&a.status).then(b.file_name.cmp(&a.file_name)));
            } else {
                result_files
                    .sort_by(|a, b| a.status.cmp(&b.status).then(a.file_name.cmp(&b.file_name)));
            }
        }
        _ => {
            if desc {
                result_files.sort_by(|a, b| {
                    a.is_registered
                        .cmp(&b.is_registered)
                        .then(b.file_name.cmp(&a.file_name))
                });
            } else {
                result_files.sort_by(|a, b| {
                    b.is_registered
                        .cmp(&a.is_registered)
                        .then(a.file_name.cmp(&b.file_name))
                });
            }
        }
    }

    let total_all = result_files.len();
    let registered_count = result_files.iter().filter(|f| f.is_registered).count();
    let unregistered_count = result_files.iter().filter(|f| !f.is_registered).count();

    let filtered: Vec<ScannedFileInfo> = if let Some(ref pat) = params.pattern {
        let re = match regex::Regex::new(&format!("(?i){}", pat)) {
            Ok(r) => r,
            Err(_) => return Err(StatusCode::BAD_REQUEST),
        };
        result_files
            .into_iter()
            .filter(|f| re.is_match(&f.file_name))
            .collect()
    } else {
        result_files
    };

    let filtered_total = filtered.len();

    let page = params.page.unwrap_or(1).max(1);
    let page_size = params
        .page_size
        .or(params.limit)
        .unwrap_or(filtered_total.max(1));
    let total_pages = if page_size > 0 {
        (filtered_total + page_size - 1) / page_size
    } else {
        1
    };
    let start = (page - 1) * page_size;
    let files: Vec<ScannedFileInfo> = filtered.into_iter().skip(start).take(page_size).collect();

    let table_videos = schema::table_name("videos");
    let table_chunks = schema::table_name("chunk");
    let total_chunks: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {}", table_chunks))
        .fetch_one(state.db.pool())
        .await
        .unwrap_or(0);
    let searchable_chunks: i64 = sqlx::query_scalar(&format!(
        "SELECT COUNT(*) FROM {} WHERE vector_id IS NOT NULL",
        table_chunks
    ))
    .fetch_one(state.db.pool())
    .await
    .unwrap_or(0);
    let pending_videos: i64 = sqlx::query_scalar(&format!(
        "SELECT COUNT(*) FROM {} WHERE status = 'pending'",
        table_videos
    ))
    .fetch_one(state.db.pool())
    .await
    .unwrap_or(0);

    Ok(Json(ScanFilesResponse {
        files,
        total: total_all,
        filtered_total,
        page,
        page_size,
        total_pages,
        registered_count,
        unregistered_count,
        total_chunks,
        searchable_chunks,
        pending_videos,
    }))
}

#[derive(Debug, Serialize)]
struct SftpgoStatusResponse {
    username: String,
    home_dir: String,
    files_count: i64,
    registered_videos: Vec<RegisteredVideo>,
    last_login: Option<String>,
}

#[derive(Debug, Serialize)]
struct RegisteredVideo {
    uuid: String,
    file_name: String,
    status: String,
}

async fn get_sftpgo_status(
    State(state): State<AppState>,
) -> Result<Json<SftpgoStatusResponse>, StatusCode> {
    let demo_dir = "/Users/accusys/momentry/var/sftpgo/data/demo";

    let files_count: i64 = std::fs::read_dir(demo_dir)
        .map(|entries| entries.count() as i64)
        .unwrap_or(0);

    let table_videos = schema::table_name("videos");

    let registered_videos: Vec<(String, String, String)> = sqlx::query_as(&format!(
        "SELECT file_uuid, file_name, status FROM {} WHERE file_path LIKE '%demo%' ORDER BY id",
        table_videos
    ))
    .fetch_all(state.db.pool())
    .await
    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;

    let registered_videos = registered_videos
        .into_iter()
        .map(|(uuid, file_name, status)| RegisteredVideo {
            uuid,
            file_name,
            status,
        })
        .collect();

    Ok(Json(SftpgoStatusResponse {
        username: "demo".to_string(),
        home_dir: demo_dir.to_string(),
        files_count,
        registered_videos,
        last_login: None,
    }))
}

#[derive(Debug, Serialize)]
struct IngestionStep {
    name: String,
    status: String,
    detail: Option<String>,
}

#[derive(Debug, Serialize)]
struct IdentityRef {
    uuid: String,
    name: String,
}

#[derive(Debug, Serialize)]
struct IngestionStatusResponse {
    file_uuid: String,
    steps: Vec<IngestionStep>,
    related_identities: Vec<IdentityRef>,
    strangers: i64,
}

async fn get_ingestion_status(
    State(state): State<AppState>,
    Path(file_uuid): Path<String>,
) -> Result<Json<IngestionStatusResponse>, StatusCode> {
    let pool = state.db.pool();
    let chunk = schema::table_name("chunk");
    let fd = schema::table_name("face_detections");
    let identities = schema::table_name("identities");

    let scene_meta_path = format!(
        "{}/{}.scene_meta.json",
        crate::core::config::OUTPUT_DIR.as_str(),
        file_uuid
    );
    let scene_meta_ok = std::path::Path::new(&scene_meta_path).exists();

    macro_rules! count_sql {
        ($sql:expr) => {
            sqlx::query_scalar::<_, i64>($sql)
                .fetch_one(pool)
                .await
                .unwrap_or(0)
        };
    }

    let sentence_count = count_sql!(&format!(
        "SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'sentence'"
    ));
    let sentence_embedded = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'sentence' AND embedding IS NOT NULL"));
    let scene_count = count_sql!(&format!(
        "SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut'"
    ));
    let face_total = count_sql!(&format!(
        "SELECT COUNT(*) FROM {fd} WHERE file_uuid = '{file_uuid}'"
    ));
    let trace_count = count_sql!(&format!("SELECT COUNT(DISTINCT trace_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL"));
    let trace_chunks = count_sql!(&format!(
        "SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'trace'"
    ));
    let identity_count = count_sql!(&format!("SELECT COUNT(DISTINCT identity_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND identity_id IS NOT NULL"));
    let tkg_nodes = count_sql!(&format!(
        "SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
        schema::table_name("tkg_nodes")
    ));
    let tkg_edges = count_sql!(&format!(
        "SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
        schema::table_name("tkg_edges")
    ));
    let scene_5w1h = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut' AND summary_text IS NOT NULL AND summary_text != ''"));

    let related_identities: Vec<IdentityRef> =
        match sqlx::query_as::<_, (String, String)>(&format!(
            "SELECT DISTINCT i.uuid::text, i.name FROM {identities} i \
         JOIN {fd} fd ON fd.identity_id = i.id \
         WHERE fd.file_uuid = '{file_uuid}' AND fd.identity_id IS NOT NULL \
         ORDER BY i.name"
        ))
        .fetch_all(pool)
        .await
        {
            Ok(rows) => rows
                .into_iter()
                .map(|(uuid, name)| IdentityRef {
                    uuid: uuid.replace('-', ""),
                    name,
                })
                .collect(),
            Err(e) => {
                tracing::error!("related_identities query failed: {}", e);
                vec![]
            }
        };

    let strangers = count_sql!(&format!(
        "SELECT COUNT(DISTINCT trace_id) FROM {fd} \
         WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL AND identity_id IS NULL"
    ));

    macro_rules! step {
        ($name:expr, $done:expr, $detail:expr) => {
            IngestionStep {
                name: $name.into(),
                status: if $done { "done" } else { "pending" }.into(),
                detail: $detail,
            }
        };
    }

    let steps = vec![
        step!(
            "rule1_sentence",
            sentence_count > 0,
            Some(format!("{sentence_count} sentence chunks"))
        ),
        step!(
            "auto_vectorize",
            sentence_embedded > 0,
            Some(format!("{sentence_embedded} embedded"))
        ),
        step!(
            "rule3_scene",
            scene_count > 0,
            Some(format!("{scene_count} scene chunks"))
        ),
        step!(
            "face_trace",
            trace_count > 0,
            Some(format!("{trace_count} traces / {face_total} detections"))
        ),
        step!(
            "trace_chunks",
            trace_chunks > 0,
            Some(format!("{trace_chunks} trace chunks"))
        ),
        step!(
            "tkg",
            tkg_nodes > 0 || tkg_edges > 0,
            Some(format!("{tkg_nodes} nodes, {tkg_edges} edges"))
        ),
        step!(
            "identity_match",
            identity_count > 0,
            Some(format!("{identity_count} identities matched"))
        ),
        step!("scene_metadata", scene_meta_ok, None),
        step!(
            "5w1h",
            scene_5w1h > 0,
            Some(format!("{scene_5w1h} scenes with 5W1H"))
        ),
    ];

    Ok(Json(IngestionStatusResponse {
        file_uuid,
        steps,
        related_identities,
        strangers,
    }))
}

pub fn scan_routes() -> Router<AppState> {
    Router::new()
        .route("/api/v1/files/scan", get(scan_files))
        .route("/api/v1/stats/sftpgo", get(get_sftpgo_status))
        .route(
            "/api/v1/stats/ingestion-status/:file_uuid",
            get(get_ingestion_status),
        )
}