b54c2def30
- Add database migrations (006-028) for face recognition, identity, file_uuid - Add test scripts for ASR, face, search, processing - Add portal frontend (Tauri) - Add config, benchmark, and monitoring utilities - Add model checkpoints and pretrained model references
276 lines
8.8 KiB
Python
276 lines
8.8 KiB
Python
#!/opt/homebrew/bin/python3.11
|
||
"""
|
||
监控 ASR/CUT 处理完成情况
|
||
Monitor ASR/CUT processing completion
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import time
|
||
import psutil
|
||
from datetime import datetime
|
||
|
||
|
||
def get_system_load():
|
||
"""获取系统负载"""
|
||
load_avg = os.getloadavg()
|
||
cpu_percent = psutil.cpu_percent(interval=1)
|
||
memory = psutil.virtual_memory()
|
||
|
||
return {
|
||
"load_1min": load_avg[0],
|
||
"load_5min": load_avg[1],
|
||
"load_15min": load_avg[2],
|
||
"cpu_percent": cpu_percent,
|
||
"memory_percent": memory.percent,
|
||
"memory_used_gb": memory.used / (1024**3),
|
||
"memory_total_gb": memory.total / (1024**3),
|
||
}
|
||
|
||
|
||
def find_processor_processes():
|
||
"""查找处理器进程"""
|
||
processors = {
|
||
"asr": [],
|
||
"cut": [],
|
||
"ocr": [],
|
||
"yolo": [],
|
||
"face": [],
|
||
"pose": [],
|
||
"asrx": [],
|
||
"caption": [],
|
||
"story": [],
|
||
}
|
||
|
||
for proc in psutil.process_iter(
|
||
["pid", "name", "cmdline", "cpu_percent", "memory_percent"]
|
||
):
|
||
try:
|
||
cmdline = " ".join(proc.info["cmdline"]) if proc.info["cmdline"] else ""
|
||
|
||
# 检查各种处理器
|
||
if "asr_processor" in cmdline:
|
||
processors["asr"].append(
|
||
{
|
||
"pid": proc.pid,
|
||
"cpu": proc.info.get("cpu_percent", 0),
|
||
"memory": proc.info.get("memory_percent", 0),
|
||
"cmdline": cmdline[:100] + "..."
|
||
if len(cmdline) > 100
|
||
else cmdline,
|
||
}
|
||
)
|
||
elif "cut_processor" in cmdline:
|
||
processors["cut"].append(
|
||
{
|
||
"pid": proc.pid,
|
||
"cpu": proc.info.get("cpu_percent", 0),
|
||
"memory": proc.info.get("memory_percent", 0),
|
||
"cmdline": cmdline[:100] + "..."
|
||
if len(cmdline) > 100
|
||
else cmdline,
|
||
}
|
||
)
|
||
elif "ocr_processor" in cmdline:
|
||
processors["ocr"].append(proc.pid)
|
||
elif "yolo_processor" in cmdline:
|
||
processors["yolo"].append(proc.pid)
|
||
elif "face_processor" in cmdline:
|
||
processors["face"].append(proc.pid)
|
||
elif "pose_processor" in cmdline:
|
||
processors["pose"].append(proc.pid)
|
||
|
||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||
continue
|
||
|
||
return processors
|
||
|
||
|
||
def check_output_files():
|
||
"""检查输出文件"""
|
||
output_dir = "/Users/accusys/momentry/output"
|
||
if not os.path.exists(output_dir):
|
||
return {}
|
||
|
||
files = {}
|
||
for filename in os.listdir(output_dir):
|
||
if filename.endswith(".json"):
|
||
# 提取处理器类型
|
||
if "_asr_" in filename:
|
||
processor = "asr"
|
||
elif "_cut_" in filename:
|
||
processor = "cut"
|
||
elif "_ocr_" in filename:
|
||
processor = "ocr"
|
||
elif "_yolo_" in filename:
|
||
processor = "yolo"
|
||
elif "_face_" in filename:
|
||
processor = "face"
|
||
elif "_pose_" in filename:
|
||
processor = "pose"
|
||
elif "_asrx_" in filename:
|
||
processor = "asrx"
|
||
elif "_caption_" in filename:
|
||
processor = "caption"
|
||
elif "_story_" in filename:
|
||
processor = "story"
|
||
else:
|
||
continue
|
||
|
||
if processor not in files:
|
||
files[processor] = []
|
||
|
||
filepath = os.path.join(output_dir, filename)
|
||
try:
|
||
size = os.path.getsize(filepath)
|
||
mtime = os.path.getmtime(filepath)
|
||
files[processor].append(
|
||
{
|
||
"filename": filename,
|
||
"size": size,
|
||
"mtime": datetime.fromtimestamp(mtime),
|
||
"age_seconds": time.time() - mtime,
|
||
}
|
||
)
|
||
except:
|
||
pass
|
||
|
||
# 按修改时间排序
|
||
for processor in files:
|
||
files[processor].sort(key=lambda x: x["mtime"], reverse=True)
|
||
|
||
return files
|
||
|
||
|
||
def main():
|
||
print("=" * 80)
|
||
print("ASR/CUT 处理完成监控")
|
||
print(f"时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||
print("=" * 80)
|
||
|
||
# 获取系统状态
|
||
system = get_system_load()
|
||
print(f"\n📊 系统状态:")
|
||
print(
|
||
f" 负载: {system['load_1min']:.2f} (1min), {system['load_5min']:.2f} (5min), {system['load_15min']:.2f} (15min)"
|
||
)
|
||
print(f" CPU使用率: {system['cpu_percent']:.1f}%")
|
||
print(
|
||
f" 内存: {system['memory_percent']:.1f}% ({system['memory_used_gb']:.1f}GB / {system['memory_total_gb']:.1f}GB)"
|
||
)
|
||
|
||
# 查找处理器进程
|
||
processors = find_processor_processes()
|
||
|
||
print(f"\n🔍 处理器进程:")
|
||
for processor_type, procs in processors.items():
|
||
if procs:
|
||
if (
|
||
processor_type in ["asr", "cut"]
|
||
and isinstance(procs, list)
|
||
and len(procs) > 0
|
||
):
|
||
# 对于 ASR 和 CUT，显示详细信息
|
||
print(f" {processor_type.upper()}: {len(procs)} 个进程")
|
||
for proc in procs[:3]: # 只显示前3个
|
||
print(
|
||
f" PID {proc['pid']}: CPU {proc['cpu']:.1f}%, 内存 {proc['memory']:.1f}%"
|
||
)
|
||
print(f" 命令: {proc['cmdline']}")
|
||
if len(procs) > 3:
|
||
print(f" ... 还有 {len(procs) - 3} 个进程")
|
||
else:
|
||
print(f" {processor_type.upper()}: {len(procs)} 个进程")
|
||
|
||
# 检查输出文件
|
||
output_files = check_output_files()
|
||
|
||
print(f"\n📁 输出文件统计:")
|
||
for processor_type, files in output_files.items():
|
||
if files:
|
||
latest = files[0]
|
||
print(f" {processor_type.upper()}: {len(files)} 个文件")
|
||
print(
|
||
f" 最新: {latest['filename']} ({latest['size']} 字节, {latest['age_seconds']:.0f} 秒前)"
|
||
)
|
||
|
||
# 分析状态
|
||
print(f"\n📈 状态分析:")
|
||
|
||
# 检查 ASR 处理
|
||
asr_procs = len(processors.get("asr", []))
|
||
asr_files = len(output_files.get("asr", []))
|
||
|
||
if asr_procs > 0:
|
||
total_cpu = sum(p["cpu"] for p in processors["asr"])
|
||
print(f" ASR处理: {asr_procs} 个进程运行中 (总CPU: {total_cpu:.1f}%)")
|
||
|
||
if total_cpu > 100:
|
||
print(f" ⚠️ CPU使用率很高，可能正在处理视频")
|
||
elif total_cpu < 10:
|
||
print(f" ✅ CPU使用率正常，可能接近完成")
|
||
else:
|
||
print(f" ASR处理: 没有运行中的进程")
|
||
if asr_files > 0:
|
||
print(f" ✅ 已完成 {asr_files} 个处理任务")
|
||
|
||
# 检查 CUT 处理
|
||
cut_procs = len(processors.get("cut", []))
|
||
cut_files = len(output_files.get("cut", []))
|
||
|
||
if cut_procs > 0:
|
||
print(f" CUT处理: {cut_procs} 个进程运行中")
|
||
else:
|
||
print(f" CUT处理: 没有运行中的进程")
|
||
if cut_files > 0:
|
||
print(f" ✅ 已完成 {cut_files} 个处理任务")
|
||
|
||
# 系统负载分析
|
||
if system["load_1min"] > 8:
|
||
print(f" ⚠️ 系统负载很高 ({system['load_1min']:.1f})")
|
||
print(f" 建议等待处理完成后再进行其他操作")
|
||
elif system["load_1min"] > 4:
|
||
print(f" ℹ️ 系统负载中等 ({system['load_1min']:.1f})")
|
||
else:
|
||
print(f" ✅ 系统负载正常 ({system['load_1min']:.1f})")
|
||
|
||
# 内存分析
|
||
if system["memory_percent"] > 90:
|
||
print(f" ⚠️ 内存使用率很高 ({system['memory_percent']:.1f}%)")
|
||
print(f" 建议监控内存使用情况")
|
||
elif system["memory_percent"] > 80:
|
||
print(f" ℹ️ 内存使用率较高 ({system['memory_percent']:.1f}%)")
|
||
|
||
print(f"\n⏱️ 监控将持续运行，按 Ctrl+C 停止")
|
||
print("=" * 80)
|
||
|
||
return {
|
||
"system": system,
|
||
"processors": processors,
|
||
"output_files": output_files,
|
||
}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
# 第一次运行
|
||
data = main()
|
||
|
||
# 持续监控
|
||
interval = 30 # 秒
|
||
print(f"\n开始持续监控，每 {interval} 秒更新一次...\n")
|
||
|
||
while True:
|
||
time.sleep(interval)
|
||
print("\n" + "=" * 80)
|
||
print(f"更新: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||
print("=" * 80)
|
||
data = main()
|
||
|
||
except KeyboardInterrupt:
|
||
print(f"\n\n监控已停止")
|
||
sys.exit(0)
|
||
except Exception as e:
|
||
print(f"\n错误: {e}")
|
||
sys.exit(1)
|