feat: Swift Face Pose integration + TKG 方案 B

Major Changes:
- swift_face_pose: output pose angles (yaw/pitch/roll) in face.json
- face_processor.py: call swift_face_pose (dual output: face.json + pose.json)
- Face struct: add pose_angle field
- TKG 方案 B: gaze/lip_track nodes from face.json (no face_detections dependency)
- Chunk cleanup: delete old data before rebuild (avoid duplicate key)
- Hand nodes: classify by hand_type + gesture (15 combinations)
- HAND_OBJECT edges: bbox spatial matching (174 matches)

Test Results:
- Blake Jones: 8 faces, pose_angle ✓, 66 nodes, 174 edges
- FilmRiot: 394 faces, pose_angle ✓, 35 nodes, 39 edges
- Left hands: 132, Right hands: 2

Architecture:
- All TKG nodes built from JSON files (face.json, hand.json, yolo.json)
- Swift processors: sample_interval=3 (Face/Pose/Hand sync)
- Cleanup functions: delete_tkg_nodes_by_uuid, delete_tkg_edges_by_uuid
This commit is contained in:
Accusys
2026-06-23 05:47:24 +08:00
parent e1e2da2140
commit 766a1d9a6d
17 changed files with 1108 additions and 47 deletions

View File

@@ -33,7 +33,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from redis_publisher import RedisPublisher
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
SWIFT_BIN = os.path.join(SCRIPT_DIR, "swift_processors", ".build", "debug", "swift_face")
SWIFT_BIN = os.path.join(SCRIPT_DIR, "swift_processors", ".build", "debug", "swift_face_pose")
FACENET_PATH = os.path.join(SCRIPT_DIR, "..", "models", "facenet512.mlpackage")
# Pose angle classification from roll/yaw
@@ -106,23 +106,29 @@ class FaceProcessorVision:
return None
def process_with_swift(self) -> Dict:
"""Step 1: Run swift_face to get bbox + pose"""
print(f"[FACE_V2] Step 1: Vision detection...")
"""Step 1: Run swift_face_pose to get bbox + pose (generates face.json + pose.json)"""
print(f"[FACE_V2] Step 1: Vision detection (face + pose)...")
# Build swift_face if needed
# Build swift_face_pose if needed
if not os.path.exists(SWIFT_BIN):
build_dir = os.path.join(SCRIPT_DIR, "swift_processors")
print(f"[FACE_V2] Building swift_face in {build_dir}...")
print(f"[FACE_V2] Building swift_face_pose in {build_dir}...")
subprocess.run(
["swift", "build", "-c", "debug", "--product", "swift_face"],
["swift", "build", "-c", "debug", "--product", "swift_face_pose"],
cwd=build_dir, check=True
)
swift_out = self.output_path.replace(".json", "_detect.json")
swift_face_out = self.output_path.replace(".json", "_detect.json")
# Pose output: same directory, but replace "face" with "pose" in filename
output_dir = os.path.dirname(self.output_path)
output_basename = os.path.basename(self.output_path)
pose_basename = output_basename.replace("face", "pose")
swift_pose_out = os.path.join(output_dir, pose_basename)
cmd = [
SWIFT_BIN,
self.video_path,
swift_out,
swift_face_out,
swift_pose_out,
"--sample-interval", str(self.sample_interval),
]
if self.uuid:
@@ -130,7 +136,7 @@ class FaceProcessorVision:
print(f"[FACE_V2] Running: {' '.join(cmd)}")
t0 = time.time()
log_path = swift_out + ".log"
log_path = swift_face_out + ".log"
log_f = open(log_path, "w")
proc = subprocess.Popen(cmd, stdout=log_f, stderr=subprocess.STDOUT, text=True)
last_pct = -1
@@ -155,13 +161,19 @@ class FaceProcessorVision:
stderr_out = proc.stderr.read()
if stderr_out:
print(stderr_out.strip(), file=sys.stderr)
raise RuntimeError(f"swift_face exited with code {proc.returncode}")
raise RuntimeError(f"swift_face_pose exited with code {proc.returncode}")
elapsed = time.time() - t0
print(f"[FACE_V2] Detection done in {elapsed:.1f}s")
with open(swift_out) as f:
return json.load(f)
with open(swift_face_out) as f:
face_data = json.load(f)
# Also check if pose.json was generated (for reference)
if os.path.exists(swift_pose_out):
print(f"[FACE_V2] Pose file generated: {swift_pose_out}")
return face_data
def embed_and_save(self, detection_data: Dict):
"""Step 2: Crop faces + CoreML embedding + save face.json"""