479 lines
18 KiB
Python
479 lines
18 KiB
Python
#!/opt/homebrew/bin/python3.11
|
|
"""
|
|
Momentry Dashboard — Flask web app
|
|
Reads pipeline status + Redis + system health on demand
|
|
"""
|
|
|
|
import json, os, subprocess, sys, platform
|
|
from pathlib import Path
|
|
from flask import Flask, jsonify, render_template_string
|
|
|
|
app = Flask(__name__)
|
|
|
|
PROJECT = Path(__file__).resolve().parent.parent
|
|
|
|
# System role detection
|
|
HOSTNAME = platform.node()
|
|
IS_M5 = "MacBook" in HOSTNAME or "M5" in HOSTNAME
|
|
SYSTEM_ROLE = "M5 (MacBook Pro)" if IS_M5 else "M4 (Mac Mini)"
|
|
SYSTEM_COLOR = "#58a6ff" if IS_M5 else "#f0883e"
|
|
|
|
|
|
def run_status_json():
|
|
"""Run pipeline_status.py and return parsed JSON"""
|
|
r = subprocess.run(
|
|
[sys.executable, str(PROJECT / "scripts/pipeline_status.py"), "--json"],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
return json.loads(r.stdout)
|
|
|
|
|
|
def run_redis_info():
|
|
"""Fetch key Redis metrics"""
|
|
result = {}
|
|
try:
|
|
r = subprocess.run(
|
|
["redis-cli", "-a", "accusys", "INFO", "all"],
|
|
capture_output=True, text=True, timeout=5,
|
|
)
|
|
for line in r.stdout.split("\n"):
|
|
line = line.strip()
|
|
if ":" not in line or line.startswith("#"):
|
|
continue
|
|
k, v = line.split(":", 1)
|
|
if k in ("total_system_memory_human", "used_memory_human",
|
|
"used_memory_peak_human", "total_connections_received",
|
|
"total_commands_processed", "keyspace_hits", "keyspace_misses",
|
|
"connected_clients", "uptime_in_seconds"):
|
|
result[k] = v if not v.endswith("_human") else v
|
|
result["keyspace_hits"] = int(result.get("keyspace_hits", 0))
|
|
result["keyspace_misses"] = int(result.get("keyspace_misses", 0))
|
|
hit_rate = result["keyspace_hits"] / max(result["keyspace_hits"] + result["keyspace_misses"], 1) * 100
|
|
result["hit_rate_pct"] = round(hit_rate, 1)
|
|
except Exception as e:
|
|
result["error"] = str(e)
|
|
|
|
# Get momentry keys
|
|
try:
|
|
r = subprocess.run(
|
|
["redis-cli", "-a", "accusys", "KEYS", "momentry_dev:*"],
|
|
capture_output=True, text=True, timeout=5,
|
|
)
|
|
keys = [k for k in r.stdout.strip().split("\n") if k]
|
|
result["momentry_keys"] = len(keys)
|
|
# Sample a few interesting keys
|
|
sample = {}
|
|
for k in keys:
|
|
if k.endswith(":health") or k.endswith(":job:") or ":processor:" in k:
|
|
pass
|
|
if len(sample) >= 5:
|
|
break
|
|
result["key_sample"] = keys[:10]
|
|
except:
|
|
result["momentry_keys"] = 0
|
|
result["key_sample"] = []
|
|
|
|
return result
|
|
|
|
|
|
def run_db_info():
|
|
"""Fetch DB metrics + current processing file"""
|
|
psql = "/Users/accusys/pgsql/18.3/bin/psql"
|
|
cmd = [psql, "-U", "accusys", "-d", "momentry", "-t", "-A"]
|
|
result = {}
|
|
try:
|
|
r = subprocess.run(cmd + ["-c", """
|
|
SELECT 'videos', count(*) FROM dev.videos
|
|
UNION ALL SELECT 'chunks', count(*) FROM dev.chunks
|
|
UNION ALL SELECT 'face_detections', count(*) FROM dev.face_detections
|
|
UNION ALL SELECT 'identities', count(*) FROM dev.identities
|
|
UNION ALL SELECT 'tkg_nodes', count(*) FROM dev.tkg_nodes
|
|
UNION ALL SELECT 'tkg_edges', count(*) FROM dev.tkg_edges
|
|
"""], capture_output=True, text=True, timeout=10)
|
|
for line in r.stdout.strip().split("\n"):
|
|
if not line.strip() or "|" not in line:
|
|
continue
|
|
parts = line.split("|")
|
|
result[parts[0].strip()] = int(parts[1])
|
|
except:
|
|
pass
|
|
|
|
# 所有檔案的 pipeline 進度（依檔案名去重，取最新）
|
|
try:
|
|
r = subprocess.run(cmd + ["-c", """
|
|
SELECT DISTINCT ON (v.file_name)
|
|
v.file_uuid, v.file_name, v.status,
|
|
COALESCE(v.processing_status::text, '{}') as pstatus,
|
|
m.status as job_status
|
|
FROM dev.videos v
|
|
LEFT JOIN dev.monitor_jobs m ON m.uuid = v.file_uuid
|
|
WHERE v.status IN ('completed', 'processing')
|
|
OR m.status IS NOT NULL
|
|
ORDER BY v.file_name, GREATEST(
|
|
COALESCE(v.registration_time::timestamp, '1970-01-01'),
|
|
COALESCE(m.updated_at, '1970-01-01')
|
|
) DESC
|
|
LIMIT 20
|
|
"""], capture_output=True, text=True, timeout=10)
|
|
seen_names = set()
|
|
files = []
|
|
for line in r.stdout.strip().split("\n"):
|
|
if not line.strip() or "|" not in line:
|
|
continue
|
|
parts = line.split("|", 4)
|
|
if len(parts) < 5:
|
|
continue
|
|
name = parts[1].strip()
|
|
if name in seen_names:
|
|
continue
|
|
seen_names.add(name)
|
|
f = {"uuid": parts[0].strip(), "name": name,
|
|
"status": parts[2].strip(), "job_status": parts[4].strip()}
|
|
try:
|
|
ps = json.loads(parts[3]) if parts[3] and parts[3] != '{}' else {}
|
|
f["progress"] = ps.get("progress", {})
|
|
except:
|
|
f["progress"] = {}
|
|
files.append(f)
|
|
result["files"] = files
|
|
except Exception as e:
|
|
result["files_error"] = str(e)
|
|
|
|
return result
|
|
|
|
|
|
@app.route("/")
|
|
def index():
|
|
return render_template_string(TEMPLATE)
|
|
|
|
|
|
@app.route("/api/status")
|
|
def api_status():
|
|
return jsonify(run_status_json())
|
|
|
|
|
|
@app.route("/api/redis")
|
|
def api_redis():
|
|
return jsonify(run_redis_info())
|
|
|
|
|
|
@app.route("/api/db")
|
|
def api_db():
|
|
return jsonify(run_db_info())
|
|
|
|
|
|
@app.route("/api/all")
|
|
def api_all():
|
|
return jsonify({
|
|
"system": {"hostname": HOSTNAME, "role": SYSTEM_ROLE, "is_m5": IS_M5},
|
|
"status": run_status_json(),
|
|
"redis": run_redis_info(),
|
|
"db": run_db_info(),
|
|
})
|
|
|
|
|
|
TEMPLATE = """<!DOCTYPE html>
|
|
<html lang="zh-TW">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>Momentry Dashboard</title>
|
|
<style>
|
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
background: #0d1117; color: #c9d1d9; padding: 20px; }
|
|
.container { max-width: 1200px; margin: 0 auto; }
|
|
h1 { font-size: 24px; margin-bottom: 20px; color: #58a6ff; }
|
|
h2 { font-size: 16px; margin-bottom: 12px; color: #8b949e; text-transform: uppercase; letter-spacing: 1px; }
|
|
.section { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 20px; margin-bottom: 20px; }
|
|
.row { display: flex; gap: 16px; flex-wrap: wrap; }
|
|
.col { flex: 1; min-width: 300px; }
|
|
table { width: 100%; border-collapse: collapse; font-size: 14px; }
|
|
th, td { padding: 8px 12px; text-align: left; border-bottom: 1px solid #21262d; }
|
|
th { color: #8b949e; font-weight: 600; }
|
|
.pass { color: #3fb950; font-weight: bold; }
|
|
.fail { color: #f85149; font-weight: bold; }
|
|
.badge { display: inline-block; padding: 2px 8px; border-radius: 12px; font-size: 12px; font-weight: 600; }
|
|
.badge-ok { background: #1b3a1b; color: #3fb950; }
|
|
.badge-err { background: #3a1b1b; color: #f85149; }
|
|
.badge-warn { background: #3a321b; color: #d29922; }
|
|
.stat-value { font-size: 28px; font-weight: 700; }
|
|
.stat-label { font-size: 12px; color: #8b949e; margin-top: 4px; }
|
|
.stat-card { background: #0d1117; border: 1px solid #30363d; border-radius: 6px; padding: 16px; text-align: center; }
|
|
.refresh-bar { display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px; }
|
|
.last-updated { color: #8b949e; font-size: 13px; }
|
|
button { background: #238636; color: white; border: none; padding: 8px 20px; border-radius: 6px; cursor: pointer; font-size: 14px; }
|
|
button:hover { background: #2ea043; }
|
|
.progress-bar { height: 6px; background: #21262d; border-radius: 3px; margin-top: 8px; }
|
|
.progress-fill { height: 100%; border-radius: 3px; background: #238636; transition: width 0.5s; }
|
|
@media (max-width: 768px) { .col { min-width: 100%; } }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="refresh-bar">
|
|
<h1>Momentry Dashboard <span style="font-size:14px;background:#1f2937;color:#{{'58a6ff' if IS_M5 else 'f0883e'}};padding:4px 12px;border-radius:12px;margin-left:8px;vertical-align:middle">🤖 {{ SYSTEM_ROLE }}</span></h1>
|
|
<div style="display:flex;align-items:center;gap:8px">
|
|
<span class="last-updated" id="lastUpdated">—</span>
|
|
<button onclick="copyStatus()" style="background:#1f6feb;padding:6px 14px;font-size:13px">📋 Copy</button>
|
|
<button onclick="fetchAll()" style="background:#238636;padding:6px 14px;font-size:13px">⟳ Refresh</button>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>✅ Pipeline Checklist</h2>
|
|
<table id="checklist"><tr><td colspan="3">Loading...</td></tr></table>
|
|
</div>
|
|
</div>
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>💻 System Health</h2>
|
|
<div id="health" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
<div class="section">
|
|
<h2>🛠 Services</h2>
|
|
<div id="services" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section" id="fileProgressSection">
|
|
<h2>📁 Pipeline Progress</h2>
|
|
<div id="fileProgress" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>⚡ Redis</h2>
|
|
<div id="redis" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
</div>
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>🗄 Database</h2>
|
|
<div id="db" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>⏱ Processor Timing</h2>
|
|
<table id="timing"><tr><td>Loading...</td></tr></table>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function fetchAll() {
|
|
const ts = new Date().toISOString().slice(11,19);
|
|
document.getElementById('lastUpdated').textContent = '🔄 ' + ts;
|
|
|
|
try {
|
|
const all = await (await fetch('/api/all')).json();
|
|
_lastData = all;
|
|
const status = all.status;
|
|
renderChecklist(status.job);
|
|
renderHealth(status.health);
|
|
renderTiming(status.health?.processors);
|
|
if (all.redis) renderRedis(all.redis);
|
|
if (all.db) { renderDb(all.db); renderFileProgress(all.db); }
|
|
document.getElementById('lastUpdated').textContent = '✅ ' + ts;
|
|
} catch(e) {
|
|
document.getElementById('checklist').innerHTML = '<tr><td class="fail">Error: ' + e.message + '</td></tr>';
|
|
// Fallback: try separate endpoints
|
|
try {
|
|
const s = await (await fetch('/api/status')).json(); renderChecklist(s.job); renderHealth(s.health); renderTiming(s.health?.processors);
|
|
} catch(e2) {}
|
|
try {
|
|
const r = await (await fetch('/api/redis')).json(); renderRedis(r);
|
|
} catch(e2) {}
|
|
try {
|
|
const d = await (await fetch('/api/db')).json(); renderDb(d); renderFileProgress(d);
|
|
} catch(e2) {}
|
|
}
|
|
}
|
|
|
|
function renderChecklist(job) {
|
|
if (!job || !job.stages) return;
|
|
let h = '<tr><th>Stage</th><th>Status</th><th>Detail</th><th>Time</th></tr>';
|
|
for (const s of job.stages) {
|
|
const cls = s.passed ? 'pass' : 'fail';
|
|
const icon = s.passed ? '✅' : '❌';
|
|
h += '<tr><td>' + s.name + '</td><td class="' + cls + '">' + icon + '</td><td>' + s.detail + '</td><td>' + s.elapsed + 's</td></tr>';
|
|
}
|
|
const totalCls = job.passed ? 'pass' : 'fail';
|
|
h += '<tr style="font-weight:bold;border-top:2px solid #30363d"><td>TOTAL</td><td class="' + totalCls + '">' + (job.passed ? '✅' : '❌') + '</td><td></td><td>' + job.total_elapsed + 's</td></tr>';
|
|
document.getElementById('checklist').innerHTML = h;
|
|
}
|
|
|
|
function renderHealth(h) {
|
|
if (!h) return;
|
|
const memPct = h.memory_used_mb ? (h.memory_used_mb / 49152 * 100).toFixed(1) : '?';
|
|
const memBar = Math.min(parseFloat(memPct), 100);
|
|
const barColor = memBar > 85 ? '#f85149' : memBar > 70 ? '#d29922' : '#3fb950';
|
|
document.getElementById('health').innerHTML = `
|
|
<div class="row">
|
|
<div class="col"><div class="stat-card"><div class="stat-value">${h.cpu_load_1m ?? '?'}</div><div class="stat-label">CPU Load (1m)</div></div></div>
|
|
<div class="col"><div class="stat-card"><div class="stat-value">${memPct}%</div><div class="stat-label">Memory</div><div class="progress-bar"><div class="progress-fill" style="width:${memBar}%;background:${barColor}"></div></div></div></div>
|
|
<div class="col"><div class="stat-card"><div class="stat-value">${h.disk_use_pct ?? '?'}</div><div class="stat-label">Disk Used</div></div></div>
|
|
</div>
|
|
`;
|
|
const svc = h.services || {};
|
|
document.getElementById('services').innerHTML = Object.entries(svc).map(([k,v]) =>
|
|
'<span style="margin-right:16px">' + (v ? '✅' : '❌') + ' ' + k + '</span>'
|
|
).join('');
|
|
}
|
|
|
|
function renderTiming(procs) {
|
|
if (!procs) return;
|
|
let h = '<tr><th>Processor</th><th>Duration</th></tr>';
|
|
for (const p of procs) {
|
|
const d = p.duration_secs;
|
|
const dur = d ? (d < 60 ? d + 's' : d < 3600 ? Math.floor(d/60) + 'm ' + (d%60) + 's' : Math.floor(d/3600) + 'h ' + Math.floor((d%3600)/60) + 'm') : 'running';
|
|
h += '<tr><td>' + p.name + '</td><td>' + dur + '</td></tr>';
|
|
}
|
|
document.getElementById('timing').innerHTML = h;
|
|
}
|
|
|
|
function renderRedis(r) {
|
|
if (!r) return;
|
|
let h = '<div class="row">';
|
|
const cards = [
|
|
{k:'used_memory_human', l:'Memory Used'},
|
|
{k:'total_system_memory_human', l:'System Memory'},
|
|
{k:'connected_clients', l:'Clients'},
|
|
{k:'hit_rate_pct', l:'Hit Rate'},
|
|
{k:'momentry_keys', l:'Momentry Keys'},
|
|
{k:'uptime_in_seconds', l:'Uptime'},
|
|
];
|
|
for (const c of cards) {
|
|
let v = r[c.k] ?? '—';
|
|
if (c.k === 'uptime_in_seconds' && typeof v === 'number') {
|
|
v = v > 86400 ? Math.round(v/86400) + 'd' : Math.round(v/3600) + 'h';
|
|
}
|
|
if (c.k === 'hit_rate_pct' && typeof v === 'number') v = v.toFixed(1) + '%';
|
|
h += '<div class="col"><div class="stat-card"><div class="stat-value">' + v + '</div><div class="stat-label">' + c.l + '</div></div></div>';
|
|
}
|
|
h += '</div>';
|
|
if (r.key_sample && r.key_sample.length) {
|
|
h += '<div style="margin-top:12px;font-size:12px;color:#8b949e">Recent keys: ' + r.key_sample.slice(0,6).join(', ') + '</div>';
|
|
}
|
|
document.getElementById('redis').innerHTML = h;
|
|
}
|
|
|
|
const PIPELINE_STAGES = ['cut','scene','asr','asrx','yolo','ocr','face','pose','visual_chunk','story'];
|
|
|
|
function renderFileProgress(d) {
|
|
const el = document.getElementById('fileProgress');
|
|
if (!d || !d.files || d.files.length === 0) {
|
|
el.innerHTML = '<div style="color:#8b949e">No files found</div>';
|
|
return;
|
|
}
|
|
let h = '<table><tr><th>File</th><th>Status</th>';
|
|
for (const s of PIPELINE_STAGES) h += '<th style="font-size:11px">' + s.slice(0,4) + '</th>';
|
|
h += '</tr>';
|
|
for (const f of d.files) {
|
|
const name = f.name.length > 50 ? f.name.slice(0,50) + '...' : f.name;
|
|
const statusIcon = f.job_status === 'running' ? '▶️' : f.job_status === 'pending' ? '⏳' : f.status === 'completed' ? '✅' : '❌';
|
|
const progress = f.progress || {};
|
|
h += '<tr><td style="max-width:300px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="' + f.name + '">' + name + '</td>'
|
|
+ '<td>' + statusIcon + ' ' + (f.job_status || f.status) + '</td>';
|
|
for (const s of PIPELINE_STAGES) {
|
|
const ps = progress[s.toUpperCase()] || {};
|
|
const st = ps.status || '';
|
|
let icon = '⬜';
|
|
if (st === 'completed') icon = '✅';
|
|
else if (st === 'running') icon = '⏳';
|
|
else if (st === 'failed') icon = '❌';
|
|
h += '<td style="text-align:center;font-size:13px">' + icon + '</td>';
|
|
}
|
|
h += '</tr>';
|
|
}
|
|
h += '</table>';
|
|
el.innerHTML = h;
|
|
}
|
|
|
|
function renderDb(d) {
|
|
if (!d) return;
|
|
const rows = ['videos','chunks','face_detections','identities','tkg_nodes','tkg_edges'];
|
|
let h = '<div class="row">';
|
|
for (const key of rows) {
|
|
const v = d[key] ?? 0;
|
|
h += '<div class="col"><div class="stat-card"><div class="stat-value">' + v.toLocaleString() + '</div><div class="stat-label">' + key.replace(/_/g,' ') + '</div></div></div>';
|
|
}
|
|
h += '</div>';
|
|
document.getElementById('db').innerHTML = h;
|
|
}
|
|
|
|
let _lastData = null;
|
|
function copyStatus() {
|
|
if (!_lastData) { alert('No data loaded yet'); return; }
|
|
const d = _lastData;
|
|
const job = d.status?.job;
|
|
const h = d.status?.health;
|
|
const db = d.db;
|
|
const r = d.redis;
|
|
let lines = [];
|
|
lines.push('Momentry Pipeline Status');
|
|
lines.push('='.repeat(50));
|
|
lines.push('System: ' + (d.system?.role || '?') + ' | ' + new Date().toISOString().slice(0,19).replace('T',' '));
|
|
lines.push('');
|
|
if (job?.stages) {
|
|
lines.push('── Checklist ──');
|
|
for (const s of job.stages) {
|
|
lines.push(' ' + (s.passed ? '✅' : '❌') + ' ' + s.name.padEnd(14) + s.detail);
|
|
}
|
|
lines.push(' ' + (job.passed ? '✅' : '❌') + ' TOTAL'.padEnd(14) + job.total_elapsed + 's');
|
|
lines.push('');
|
|
}
|
|
if (h) {
|
|
lines.push('── Health ──');
|
|
lines.push(' CPU: ' + (h.cpu_load_1m ?? '?') + ' Memory: ' + (h.memory_used_mb ?? '?') + 'MB GPU: ' + (h.gpu_available ? '✅' : '❌'));
|
|
if (h.services) {
|
|
lines.push(' Services: ' + Object.entries(h.services).map(([k,v]) => k + '=' + (v ? '✓' : '✗')).join(' '));
|
|
}
|
|
lines.push('');
|
|
}
|
|
if (r) {
|
|
lines.push('── Redis ──');
|
|
lines.push(' Keys: ' + (r.momentry_keys ?? '?') + ' Hit Rate: ' + (r.hit_rate_pct ?? '?') + '% Uptime: ' + (r.uptime_in_seconds ? Math.round(r.uptime_in_seconds/3600)+'h' : '?'));
|
|
lines.push('');
|
|
}
|
|
if (db) {
|
|
lines.push('── Database ──');
|
|
const tbls = ['videos','chunks','face_detections','identities','tkg_nodes','tkg_edges'];
|
|
for (const t of tbls) {
|
|
if (db[t] !== undefined) lines.push(' ' + t + ': ' + db[t].toLocaleString());
|
|
}
|
|
if (db.files) {
|
|
lines.push('');
|
|
lines.push('── Files ──');
|
|
for (const f of db.files) {
|
|
lines.push(' ' + (f.job_status === 'running' ? '▶️' : f.job_status === 'pending' ? '⏳' : f.status === 'completed' ? '✅' : '❌') + ' ' + f.name.slice(0,60));
|
|
}
|
|
}
|
|
lines.push('');
|
|
}
|
|
const text = lines.join('\n');
|
|
navigator.clipboard.writeText(text).then(() => {
|
|
const btn = event.target;
|
|
const orig = btn.textContent;
|
|
btn.textContent = '✅ Copied!';
|
|
setTimeout(() => btn.textContent = orig, 2000);
|
|
}).catch(() => alert('Copy failed'));
|
|
}
|
|
|
|
fetchAll();
|
|
setInterval(fetchAll, 15000);
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
if __name__ == "__main__":
|
|
port = int(os.environ.get("DASHBOARD_PORT", 5050))
|
|
print(f"Momentry Dashboard: http://0.0.0.0:{port}")
|
|
app.run(host="0.0.0.0", port=port, debug=False)
|