feat: Swift Face Pose integration + TKG 方案 B
Major Changes: - swift_face_pose: output pose angles (yaw/pitch/roll) in face.json - face_processor.py: call swift_face_pose (dual output: face.json + pose.json) - Face struct: add pose_angle field - TKG 方案 B: gaze/lip_track nodes from face.json (no face_detections dependency) - Chunk cleanup: delete old data before rebuild (avoid duplicate key) - Hand nodes: classify by hand_type + gesture (15 combinations) - HAND_OBJECT edges: bbox spatial matching (174 matches) Test Results: - Blake Jones: 8 faces, pose_angle ✓, 66 nodes, 174 edges - FilmRiot: 394 faces, pose_angle ✓, 35 nodes, 39 edges - Left hands: 132, Right hands: 2 Architecture: - All TKG nodes built from JSON files (face.json, hand.json, yolo.json) - Swift processors: sample_interval=3 (Face/Pose/Hand sync) - Cleanup functions: delete_tkg_nodes_by_uuid, delete_tkg_edges_by_uuid
This commit is contained in:
@@ -33,7 +33,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from redis_publisher import RedisPublisher
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
SWIFT_BIN = os.path.join(SCRIPT_DIR, "swift_processors", ".build", "debug", "swift_face")
|
||||
SWIFT_BIN = os.path.join(SCRIPT_DIR, "swift_processors", ".build", "debug", "swift_face_pose")
|
||||
FACENET_PATH = os.path.join(SCRIPT_DIR, "..", "models", "facenet512.mlpackage")
|
||||
|
||||
# Pose angle classification from roll/yaw
|
||||
@@ -106,23 +106,29 @@ class FaceProcessorVision:
|
||||
return None
|
||||
|
||||
def process_with_swift(self) -> Dict:
|
||||
"""Step 1: Run swift_face to get bbox + pose"""
|
||||
print(f"[FACE_V2] Step 1: Vision detection...")
|
||||
"""Step 1: Run swift_face_pose to get bbox + pose (generates face.json + pose.json)"""
|
||||
print(f"[FACE_V2] Step 1: Vision detection (face + pose)...")
|
||||
|
||||
# Build swift_face if needed
|
||||
# Build swift_face_pose if needed
|
||||
if not os.path.exists(SWIFT_BIN):
|
||||
build_dir = os.path.join(SCRIPT_DIR, "swift_processors")
|
||||
print(f"[FACE_V2] Building swift_face in {build_dir}...")
|
||||
print(f"[FACE_V2] Building swift_face_pose in {build_dir}...")
|
||||
subprocess.run(
|
||||
["swift", "build", "-c", "debug", "--product", "swift_face"],
|
||||
["swift", "build", "-c", "debug", "--product", "swift_face_pose"],
|
||||
cwd=build_dir, check=True
|
||||
)
|
||||
|
||||
swift_out = self.output_path.replace(".json", "_detect.json")
|
||||
swift_face_out = self.output_path.replace(".json", "_detect.json")
|
||||
# Pose output: same directory, but replace "face" with "pose" in filename
|
||||
output_dir = os.path.dirname(self.output_path)
|
||||
output_basename = os.path.basename(self.output_path)
|
||||
pose_basename = output_basename.replace("face", "pose")
|
||||
swift_pose_out = os.path.join(output_dir, pose_basename)
|
||||
cmd = [
|
||||
SWIFT_BIN,
|
||||
self.video_path,
|
||||
swift_out,
|
||||
swift_face_out,
|
||||
swift_pose_out,
|
||||
"--sample-interval", str(self.sample_interval),
|
||||
]
|
||||
if self.uuid:
|
||||
@@ -130,7 +136,7 @@ class FaceProcessorVision:
|
||||
|
||||
print(f"[FACE_V2] Running: {' '.join(cmd)}")
|
||||
t0 = time.time()
|
||||
log_path = swift_out + ".log"
|
||||
log_path = swift_face_out + ".log"
|
||||
log_f = open(log_path, "w")
|
||||
proc = subprocess.Popen(cmd, stdout=log_f, stderr=subprocess.STDOUT, text=True)
|
||||
last_pct = -1
|
||||
@@ -155,13 +161,19 @@ class FaceProcessorVision:
|
||||
stderr_out = proc.stderr.read()
|
||||
if stderr_out:
|
||||
print(stderr_out.strip(), file=sys.stderr)
|
||||
raise RuntimeError(f"swift_face exited with code {proc.returncode}")
|
||||
raise RuntimeError(f"swift_face_pose exited with code {proc.returncode}")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
print(f"[FACE_V2] Detection done in {elapsed:.1f}s")
|
||||
|
||||
with open(swift_out) as f:
|
||||
return json.load(f)
|
||||
with open(swift_face_out) as f:
|
||||
face_data = json.load(f)
|
||||
|
||||
# Also check if pose.json was generated (for reference)
|
||||
if os.path.exists(swift_pose_out):
|
||||
print(f"[FACE_V2] Pose file generated: {swift_pose_out}")
|
||||
|
||||
return face_data
|
||||
|
||||
def embed_and_save(self, detection_data: Dict):
|
||||
"""Step 2: Crop faces + CoreML embedding + save face.json"""
|
||||
|
||||
Reference in New Issue
Block a user