feat: add migrations, test scripts, and utility tools
- Add database migrations (006-028) for face recognition, identity, file_uuid - Add test scripts for ASR, face, search, processing - Add portal frontend (Tauri) - Add config, benchmark, and monitoring utilities - Add model checkpoints and pretrained model references
This commit is contained in:
Warren
275
monitor_processing_completion.py
Normal file
275
monitor_processing_completion.py
Normal file
@@ -0,0 +1,275 @@
|
||||
#!/opt/homebrew/bin/python3.11
|
||||
"""
|
||||
监控 ASR/CUT 处理完成情况
|
||||
Monitor ASR/CUT processing completion
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def get_system_load():
|
||||
"""获取系统负载"""
|
||||
load_avg = os.getloadavg()
|
||||
cpu_percent = psutil.cpu_percent(interval=1)
|
||||
memory = psutil.virtual_memory()
|
||||
|
||||
return {
|
||||
"load_1min": load_avg[0],
|
||||
"load_5min": load_avg[1],
|
||||
"load_15min": load_avg[2],
|
||||
"cpu_percent": cpu_percent,
|
||||
"memory_percent": memory.percent,
|
||||
"memory_used_gb": memory.used / (1024**3),
|
||||
"memory_total_gb": memory.total / (1024**3),
|
||||
}
|
||||
|
||||
|
||||
def find_processor_processes():
|
||||
"""查找处理器进程"""
|
||||
processors = {
|
||||
"asr": [],
|
||||
"cut": [],
|
||||
"ocr": [],
|
||||
"yolo": [],
|
||||
"face": [],
|
||||
"pose": [],
|
||||
"asrx": [],
|
||||
"caption": [],
|
||||
"story": [],
|
||||
}
|
||||
|
||||
for proc in psutil.process_iter(
|
||||
["pid", "name", "cmdline", "cpu_percent", "memory_percent"]
|
||||
):
|
||||
try:
|
||||
cmdline = " ".join(proc.info["cmdline"]) if proc.info["cmdline"] else ""
|
||||
|
||||
# 检查各种处理器
|
||||
if "asr_processor" in cmdline:
|
||||
processors["asr"].append(
|
||||
{
|
||||
"pid": proc.pid,
|
||||
"cpu": proc.info.get("cpu_percent", 0),
|
||||
"memory": proc.info.get("memory_percent", 0),
|
||||
"cmdline": cmdline[:100] + "..."
|
||||
if len(cmdline) > 100
|
||||
else cmdline,
|
||||
}
|
||||
)
|
||||
elif "cut_processor" in cmdline:
|
||||
processors["cut"].append(
|
||||
{
|
||||
"pid": proc.pid,
|
||||
"cpu": proc.info.get("cpu_percent", 0),
|
||||
"memory": proc.info.get("memory_percent", 0),
|
||||
"cmdline": cmdline[:100] + "..."
|
||||
if len(cmdline) > 100
|
||||
else cmdline,
|
||||
}
|
||||
)
|
||||
elif "ocr_processor" in cmdline:
|
||||
processors["ocr"].append(proc.pid)
|
||||
elif "yolo_processor" in cmdline:
|
||||
processors["yolo"].append(proc.pid)
|
||||
elif "face_processor" in cmdline:
|
||||
processors["face"].append(proc.pid)
|
||||
elif "pose_processor" in cmdline:
|
||||
processors["pose"].append(proc.pid)
|
||||
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
|
||||
return processors
|
||||
|
||||
|
||||
def check_output_files():
|
||||
"""检查输出文件"""
|
||||
output_dir = "/Users/accusys/momentry/output"
|
||||
if not os.path.exists(output_dir):
|
||||
return {}
|
||||
|
||||
files = {}
|
||||
for filename in os.listdir(output_dir):
|
||||
if filename.endswith(".json"):
|
||||
# 提取处理器类型
|
||||
if "_asr_" in filename:
|
||||
processor = "asr"
|
||||
elif "_cut_" in filename:
|
||||
processor = "cut"
|
||||
elif "_ocr_" in filename:
|
||||
processor = "ocr"
|
||||
elif "_yolo_" in filename:
|
||||
processor = "yolo"
|
||||
elif "_face_" in filename:
|
||||
processor = "face"
|
||||
elif "_pose_" in filename:
|
||||
processor = "pose"
|
||||
elif "_asrx_" in filename:
|
||||
processor = "asrx"
|
||||
elif "_caption_" in filename:
|
||||
processor = "caption"
|
||||
elif "_story_" in filename:
|
||||
processor = "story"
|
||||
else:
|
||||
continue
|
||||
|
||||
if processor not in files:
|
||||
files[processor] = []
|
||||
|
||||
filepath = os.path.join(output_dir, filename)
|
||||
try:
|
||||
size = os.path.getsize(filepath)
|
||||
mtime = os.path.getmtime(filepath)
|
||||
files[processor].append(
|
||||
{
|
||||
"filename": filename,
|
||||
"size": size,
|
||||
"mtime": datetime.fromtimestamp(mtime),
|
||||
"age_seconds": time.time() - mtime,
|
||||
}
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 按修改时间排序
|
||||
for processor in files:
|
||||
files[processor].sort(key=lambda x: x["mtime"], reverse=True)
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 80)
|
||||
print("ASR/CUT 处理完成监控")
|
||||
print(f"时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("=" * 80)
|
||||
|
||||
# 获取系统状态
|
||||
system = get_system_load()
|
||||
print(f"\n📊 系统状态:")
|
||||
print(
|
||||
f" 负载: {system['load_1min']:.2f} (1min), {system['load_5min']:.2f} (5min), {system['load_15min']:.2f} (15min)"
|
||||
)
|
||||
print(f" CPU使用率: {system['cpu_percent']:.1f}%")
|
||||
print(
|
||||
f" 内存: {system['memory_percent']:.1f}% ({system['memory_used_gb']:.1f}GB / {system['memory_total_gb']:.1f}GB)"
|
||||
)
|
||||
|
||||
# 查找处理器进程
|
||||
processors = find_processor_processes()
|
||||
|
||||
print(f"\n🔍 处理器进程:")
|
||||
for processor_type, procs in processors.items():
|
||||
if procs:
|
||||
if (
|
||||
processor_type in ["asr", "cut"]
|
||||
and isinstance(procs, list)
|
||||
and len(procs) > 0
|
||||
):
|
||||
# 对于 ASR 和 CUT，显示详细信息
|
||||
print(f" {processor_type.upper()}: {len(procs)} 个进程")
|
||||
for proc in procs[:3]: # 只显示前3个
|
||||
print(
|
||||
f" PID {proc['pid']}: CPU {proc['cpu']:.1f}%, 内存 {proc['memory']:.1f}%"
|
||||
)
|
||||
print(f" 命令: {proc['cmdline']}")
|
||||
if len(procs) > 3:
|
||||
print(f" ... 还有 {len(procs) - 3} 个进程")
|
||||
else:
|
||||
print(f" {processor_type.upper()}: {len(procs)} 个进程")
|
||||
|
||||
# 检查输出文件
|
||||
output_files = check_output_files()
|
||||
|
||||
print(f"\n📁 输出文件统计:")
|
||||
for processor_type, files in output_files.items():
|
||||
if files:
|
||||
latest = files[0]
|
||||
print(f" {processor_type.upper()}: {len(files)} 个文件")
|
||||
print(
|
||||
f" 最新: {latest['filename']} ({latest['size']} 字节, {latest['age_seconds']:.0f} 秒前)"
|
||||
)
|
||||
|
||||
# 分析状态
|
||||
print(f"\n📈 状态分析:")
|
||||
|
||||
# 检查 ASR 处理
|
||||
asr_procs = len(processors.get("asr", []))
|
||||
asr_files = len(output_files.get("asr", []))
|
||||
|
||||
if asr_procs > 0:
|
||||
total_cpu = sum(p["cpu"] for p in processors["asr"])
|
||||
print(f" ASR处理: {asr_procs} 个进程运行中 (总CPU: {total_cpu:.1f}%)")
|
||||
|
||||
if total_cpu > 100:
|
||||
print(f" ⚠️ CPU使用率很高，可能正在处理视频")
|
||||
elif total_cpu < 10:
|
||||
print(f" ✅ CPU使用率正常，可能接近完成")
|
||||
else:
|
||||
print(f" ASR处理: 没有运行中的进程")
|
||||
if asr_files > 0:
|
||||
print(f" ✅ 已完成 {asr_files} 个处理任务")
|
||||
|
||||
# 检查 CUT 处理
|
||||
cut_procs = len(processors.get("cut", []))
|
||||
cut_files = len(output_files.get("cut", []))
|
||||
|
||||
if cut_procs > 0:
|
||||
print(f" CUT处理: {cut_procs} 个进程运行中")
|
||||
else:
|
||||
print(f" CUT处理: 没有运行中的进程")
|
||||
if cut_files > 0:
|
||||
print(f" ✅ 已完成 {cut_files} 个处理任务")
|
||||
|
||||
# 系统负载分析
|
||||
if system["load_1min"] > 8:
|
||||
print(f" ⚠️ 系统负载很高 ({system['load_1min']:.1f})")
|
||||
print(f" 建议等待处理完成后再进行其他操作")
|
||||
elif system["load_1min"] > 4:
|
||||
print(f" ℹ️ 系统负载中等 ({system['load_1min']:.1f})")
|
||||
else:
|
||||
print(f" ✅ 系统负载正常 ({system['load_1min']:.1f})")
|
||||
|
||||
# 内存分析
|
||||
if system["memory_percent"] > 90:
|
||||
print(f" ⚠️ 内存使用率很高 ({system['memory_percent']:.1f}%)")
|
||||
print(f" 建议监控内存使用情况")
|
||||
elif system["memory_percent"] > 80:
|
||||
print(f" ℹ️ 内存使用率较高 ({system['memory_percent']:.1f}%)")
|
||||
|
||||
print(f"\n⏱️ 监控将持续运行，按 Ctrl+C 停止")
|
||||
print("=" * 80)
|
||||
|
||||
return {
|
||||
"system": system,
|
||||
"processors": processors,
|
||||
"output_files": output_files,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
# 第一次运行
|
||||
data = main()
|
||||
|
||||
# 持续监控
|
||||
interval = 30 # 秒
|
||||
print(f"\n开始持续监控，每 {interval} 秒更新一次...\n")
|
||||
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
print("\n" + "=" * 80)
|
||||
print(f"更新: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("=" * 80)
|
||||
data = main()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n\n监控已停止")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"\n错误: {e}")
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user