feat: add pose and appearance endpoints for face detail display
- Add GET /api/v1/file/:file_uuid/pose endpoint - Add GET /api/v1/file/:file_uuid/appearance endpoint - Keypoint deduplication (highest confidence) - Add face-groups endpoint for Studio integration - Add profile routes (trace-profile, file-profile) - Update API documentation
This commit is contained in:
Accusys
@@ -0,0 +1,300 @@
|
|||||||
|
# Pose & Appearance API POC Plan
|
||||||
|
|
||||||
|
**Date**: 2026-07-19
|
||||||
|
**Author**: Core API Team
|
||||||
|
**Status**: Ready for Implementation
|
||||||
|
**Target**: Studio Team
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Overview
|
||||||
|
|
||||||
|
This document outlines the POC implementation for pose skeleton and appearance color display in Momentry Studio's Face Detail Modal.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Endpoints
|
||||||
|
|
||||||
|
### 2.1 Get Pose
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/v1/file/:file_uuid/pose?frame=:frame_no
|
||||||
|
```
|
||||||
|
|
||||||
|
**Parameters:**
|
||||||
|
|
||||||
|
| Name | Type | Required | Description |
|
||||||
|
|------|------|----------|-------------|
|
||||||
|
| `file_uuid` | string | Yes | File UUID (path parameter) |
|
||||||
|
| `frame` | integer | Yes | Frame number (query parameter) |
|
||||||
|
|
||||||
|
**Response (200):**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"frame": 303,
|
||||||
|
"keypoints": [
|
||||||
|
{"name": "nose", "x": 315.9, "y": 364.2, "confidence": 0.616},
|
||||||
|
{"name": "left_eye", "x": 312.4, "y": 362.8, "confidence": 0.616},
|
||||||
|
{"name": "right_eye", "x": 316.2, "y": 361.4, "confidence": 0.616},
|
||||||
|
{"name": "left_ear", "x": 343.1, "y": 354.7, "confidence": 0.856},
|
||||||
|
{"name": "right_ear", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "left_shoulder", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "right_shoulder", "x": 296.0, "y": 410.8, "confidence": 0.863},
|
||||||
|
{"name": "left_elbow", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "right_elbow", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "left_wrist", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "right_wrist", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "left_hip", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "right_hip", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "left_knee", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "right_knee", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "left_ankle", "x": 0, "y": 0, "confidence": 0},
|
||||||
|
{"name": "right_ankle", "x": 0, "y": 0, "confidence": 0}
|
||||||
|
],
|
||||||
|
"pose_class": "unknown",
|
||||||
|
"confidence": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Fields:**
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `frame` | integer | Frame number |
|
||||||
|
| `keypoints` | array | 17 COCO keypoints (deduplicated) |
|
||||||
|
| `keypoints[].name` | string | Keypoint name |
|
||||||
|
| `keypoints[].x` | float | X coordinate in pixels |
|
||||||
|
| `keypoints[].y` | float | Y coordinate in pixels |
|
||||||
|
| `keypoints[].confidence` | float | Detection confidence 0-1 (0 means not detected) |
|
||||||
|
| `pose_class` | string | Always `"unknown"` in POC |
|
||||||
|
| `confidence` | null | Always `null` in POC |
|
||||||
|
|
||||||
|
**Error Responses:**
|
||||||
|
|
||||||
|
| HTTP | When |
|
||||||
|
|------|------|
|
||||||
|
| 404 | Frame not found or pose.json not found |
|
||||||
|
| 401 | Missing or invalid API key |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.2 Get Appearance
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/v1/file/:file_uuid/appearance?frame=:frame_no
|
||||||
|
```
|
||||||
|
|
||||||
|
**Parameters:**
|
||||||
|
|
||||||
|
| Name | Type | Required | Description |
|
||||||
|
|------|------|----------|-------------|
|
||||||
|
| `file_uuid` | string | Yes | File UUID (path parameter) |
|
||||||
|
| `frame` | integer | Yes | Frame number (query parameter) |
|
||||||
|
|
||||||
|
**Response (200):**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"frame": 0,
|
||||||
|
"dominant_colors": [],
|
||||||
|
"hsv_histogram": [
|
||||||
|
[0.59, 0, 0, ...],
|
||||||
|
[0.97, 0.02, 0.01, ...],
|
||||||
|
[0.004, 0.02, 0.03, ...]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Fields:**
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `frame` | integer | Frame number |
|
||||||
|
| `dominant_colors` | array | Empty array in POC (`[]`) |
|
||||||
|
| `hsv_histogram` | array | 3x30 HSV histogram bins |
|
||||||
|
|
||||||
|
**Error Responses:**
|
||||||
|
|
||||||
|
| HTTP | When |
|
||||||
|
|------|------|
|
||||||
|
| 404 | Frame not found or appearance.json not found |
|
||||||
|
| 401 | Missing or invalid API key |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Implementation Notes
|
||||||
|
|
||||||
|
### 3.1 Data Source
|
||||||
|
|
||||||
|
| Processor | File | Location |
|
||||||
|
|-----------|------|----------|
|
||||||
|
| Pose | `{file_uuid}.pose.json` | `{OUTPUT_DIR}/{file_uuid}/` |
|
||||||
|
| Appearance | `{file_uuid}.appearance.json` | `{OUTPUT_DIR}/{file_uuid}/` |
|
||||||
|
|
||||||
|
### 3.2 Keypoint Deduplication
|
||||||
|
|
||||||
|
The pose processor may output multiple keypoints with the same name (e.g., 8 "nose" entries). Core API will:
|
||||||
|
|
||||||
|
1. Group keypoints by `name`
|
||||||
|
2. Select the one with highest `confidence`
|
||||||
|
3. Return exactly 17 unique keypoints
|
||||||
|
|
||||||
|
If a keypoint is not detected, it will have:
|
||||||
|
```json
|
||||||
|
{"name": "left_shoulder", "x": 0, "y": 0, "confidence": 0}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.3 COCO-17 Keypoint Names
|
||||||
|
|
||||||
|
```
|
||||||
|
nose, left_eye, right_eye, left_ear, right_ear,
|
||||||
|
left_shoulder, right_shoulder, left_elbow, right_elbow,
|
||||||
|
left_wrist, right_wrist, left_hip, right_hip,
|
||||||
|
left_knee, right_knee, left_ankle, right_ankle
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.4 Frame Alignment
|
||||||
|
|
||||||
|
- Pose/appearance `frame` numbers should align with face `best_face_frame`
|
||||||
|
- If frame not found, API returns 404
|
||||||
|
- Frontend should use the face's `best_face_frame` or `start_frame` to query pose/appearance
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Frontend Integration (Studio Team)
|
||||||
|
|
||||||
|
### 4.1 API Builder
|
||||||
|
|
||||||
|
**File**: `src/api/index.ts`
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
case 'get_pose': {
|
||||||
|
const { fileUuid, frame } = a
|
||||||
|
return {
|
||||||
|
url: `/api/v1/file/${fileUuid}/pose?frame=${frame}`,
|
||||||
|
method: 'GET'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'get_appearance': {
|
||||||
|
const { fileUuid, frame } = a
|
||||||
|
return {
|
||||||
|
url: `/api/v1/file/${fileUuid}/appearance?frame=${frame}`,
|
||||||
|
method: 'GET'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 Store Functions
|
||||||
|
|
||||||
|
**File**: `src/store.ts`
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
export async function getPose(fileUuid: string, frame: number): Promise<PoseResponse> {
|
||||||
|
const response = await callApi('get_pose', { fileUuid, frame })
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAppearance(fileUuid: string, frame: number): Promise<AppearanceResponse> {
|
||||||
|
const response = await callApi('get_appearance', { fileUuid, frame })
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.3 TypeScript Types
|
||||||
|
|
||||||
|
**File**: `src/api/types.ts`
|
||||||
|
|
||||||
|
Already defined in Studio codebase:
|
||||||
|
- `PoseKeypoint`
|
||||||
|
- `PoseClass`
|
||||||
|
- `PoseResponse`
|
||||||
|
- `DominantColor`
|
||||||
|
- `AppearanceResponse`
|
||||||
|
|
||||||
|
### 4.4 Usage Example
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// In Face Detail Modal
|
||||||
|
const face = selectedFace
|
||||||
|
const frame = face.best_face_frame || face.start_frame
|
||||||
|
|
||||||
|
// Fetch pose
|
||||||
|
const pose = await getPose(fileUuid, frame)
|
||||||
|
|
||||||
|
// Fetch appearance
|
||||||
|
const appearance = await getAppearance(fileUuid, frame)
|
||||||
|
|
||||||
|
// Render skeleton on canvas
|
||||||
|
renderPoseSkeleton(canvas, pose.keypoints)
|
||||||
|
|
||||||
|
// Display colors (when implemented)
|
||||||
|
displayDominantColors(appearance.dominant_colors)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. POC Limitations
|
||||||
|
|
||||||
|
| Feature | POC Status | Future |
|
||||||
|
|---------|------------|--------|
|
||||||
|
| Pose classification | `"unknown"` | ML model for standing/sitting/etc. |
|
||||||
|
| Dominant colors | Empty array `[]` | HSV extraction algorithm |
|
||||||
|
| Multiple persons | First person only | Match by trace_id |
|
||||||
|
| Confidence | `null` | Aggregated confidence score |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Testing
|
||||||
|
|
||||||
|
### 6.1 Test Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Set variables
|
||||||
|
API="http://localhost:3002"
|
||||||
|
KEY="muser_demo_key_32chars_abcdef1234567890"
|
||||||
|
FILE_UUID="477b24d3030c56b64092c8fea96676df"
|
||||||
|
|
||||||
|
# Test pose endpoint
|
||||||
|
curl -s "$API/api/v1/file/$FILE_UUID/pose?frame=303" \
|
||||||
|
-H "X-API-Key: $KEY" | jq '.keypoints | length'
|
||||||
|
|
||||||
|
# Expected: 17
|
||||||
|
|
||||||
|
# Test appearance endpoint
|
||||||
|
curl -s "$API/api/v1/file/$FILE_UUID/appearance?frame=303" \
|
||||||
|
-H "X-API-Key: $KEY" | jq '.hsv_histogram | length'
|
||||||
|
|
||||||
|
# Expected: 3 (H, S, V channels)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6.2 Expected Response Time
|
||||||
|
|
||||||
|
- Target: < 100ms per request
|
||||||
|
- Cacheable: Yes (pose/appearance data doesn't change)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Timeline
|
||||||
|
|
||||||
|
| Phase | Team | Task | Status |
|
||||||
|
|-------|------|------|--------|
|
||||||
|
| 1 | Core API | Implement endpoints | ✅ Complete |
|
||||||
|
| 2 | Core API | Test & verify | ✅ Complete |
|
||||||
|
| 3 | Core API | Notify Studio | ✅ Complete |
|
||||||
|
| 4 | Studio | Frontend integration | Pending |
|
||||||
|
| 5 | Studio | UI testing | Pending |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Contact
|
||||||
|
|
||||||
|
**Core API Team**: Will notify Studio when Phase 2 is complete via `M4_workspace/` document.
|
||||||
|
|
||||||
|
**Studio Team**: Ready to integrate once Phase 3 notification is received.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Document Version: 1.0*
|
||||||
|
*Last Updated: 2026-07-19*
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
<!-- module: profile -->
|
<!-- module: profile -->
|
||||||
<!-- description: Trace profile and file profile management — read/update face trace names, key frames, aliases, and file paths -->
|
<!-- description: Trace profile, face groups, and file profile management — read/update face trace names, key frames, aliases, and file paths -->
|
||||||
<!-- depends: 01_auth, 07_identity, 15_tkg -->
|
<!-- depends: 01_auth, 07_identity, 15_tkg -->
|
||||||
|
|
||||||
## Profile Management
|
## Profile Management
|
||||||
@@ -158,6 +158,77 @@ curl -s -X PUT "$API/api/v1/trace-profile/group" \
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
### `GET /api/v1/file/:file_uuid/face-groups`
|
||||||
|
|
||||||
|
**Auth**: Required
|
||||||
|
**Scope**: file-level
|
||||||
|
|
||||||
|
Get face groups for a file. Groups face traces by their label (name) and returns both named groups and unassigned traces.
|
||||||
|
|
||||||
|
#### Path Parameters
|
||||||
|
|
||||||
|
| Field | Type | Required | Description |
|
||||||
|
|-------|------|----------|-------------|
|
||||||
|
| `file_uuid` | string | Yes | File UUID |
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s "$API/api/v1/file/$FILE_UUID/face-groups" \
|
||||||
|
-H "X-API-Key: $KEY"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Response (200)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"file_uuid": "d3f9ae8e471a1fc4d47022c66091b920",
|
||||||
|
"face_groups": [
|
||||||
|
{
|
||||||
|
"group_id": 1,
|
||||||
|
"name": "Cary Grant",
|
||||||
|
"trace_ids": [9, 6, 8],
|
||||||
|
"trace_count": 3,
|
||||||
|
"representative_trace": 9,
|
||||||
|
"editable": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"group_id": 2,
|
||||||
|
"name": "Audrey Hepburn",
|
||||||
|
"trace_ids": [1, 11, 4, 10],
|
||||||
|
"trace_count": 4,
|
||||||
|
"representative_trace": 1,
|
||||||
|
"editable": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"total_groups": 2,
|
||||||
|
"unassigned_traces": [5, 7]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `success` | boolean | Always `true` on success |
|
||||||
|
| `file_uuid` | string | File UUID |
|
||||||
|
| `face_groups` | array | List of named face groups |
|
||||||
|
| `face_groups[].group_id` | integer | Sequential group number (starts at 1) |
|
||||||
|
| `face_groups[].name` | string | Group name (from TKG `label`) |
|
||||||
|
| `face_groups[].trace_ids` | integer[] | Trace IDs in this group |
|
||||||
|
| `face_groups[].trace_count` | integer | Number of traces in group |
|
||||||
|
| `face_groups[].representative_trace` | integer | First trace ID in group |
|
||||||
|
| `face_groups[].editable` | boolean | Always `true` (groups can be renamed) |
|
||||||
|
| `total_groups` | integer | Total named groups |
|
||||||
|
| `unassigned_traces` | integer[] | Traces with default names (e.g., "Face Trace 9") |
|
||||||
|
|
||||||
|
#### Notes
|
||||||
|
|
||||||
|
- Unassigned traces have labels starting with "Face Trace " or "Trace "
|
||||||
|
- Groups are sorted by first trace_id in each group
|
||||||
|
- `group_id` is dynamically generated and may change between requests
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### `GET /api/v1/file-profile`
|
### `GET /api/v1/file-profile`
|
||||||
|
|
||||||
**Auth**: Required
|
**Auth**: Required
|
||||||
@@ -264,4 +335,5 @@ curl -s -X PUT "$API/api/v1/file-profile" \
|
|||||||
| `aliases` | `properties->'aliases'` | Multi-language name aliases |
|
| `aliases` | `properties->'aliases'` | Multi-language name aliases |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
*Updated: 2026-07-19 — Added face-groups endpoint for Studio Proxy integration*
|
||||||
*Updated: 2026-07-18 — New profile module: trace-profile (GET, PUT, PUT group) and file-profile (GET, PUT)*
|
*Updated: 2026-07-18 — New profile module: trace-profile (GET, PUT, PUT group) and file-profile (GET, PUT)*
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
<!-- module: pose_appearance -->
|
||||||
|
<!-- description: Pose skeleton and appearance color endpoints for face detail display -->
|
||||||
|
<!-- depends: 01_auth -->
|
||||||
|
|
||||||
|
## Pose & Appearance
|
||||||
|
|
||||||
|
Endpoints for retrieving pose keypoints and appearance color data for face detail modal display.
|
||||||
|
|
||||||
|
### `GET /api/v1/file/:file_uuid/pose`
|
||||||
|
|
||||||
|
**Auth**: Required
|
||||||
|
**Scope**: file-level
|
||||||
|
|
||||||
|
Get pose keypoints for a specific frame. Returns 17 COCO keypoints with deduplication (highest confidence selected for each keypoint name).
|
||||||
|
|
||||||
|
#### Path Parameters
|
||||||
|
|
||||||
|
| Field | Type | Required | Description |
|
||||||
|
|-------|------|----------|-------------|
|
||||||
|
| `file_uuid` | string | Yes | File UUID |
|
||||||
|
|
||||||
|
#### Query Parameters
|
||||||
|
|
||||||
|
| Field | Type | Required | Description |
|
||||||
|
|-------|------|----------|-------------|
|
||||||
|
| `frame` | integer | Yes | Frame number |
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s "$API/api/v1/file/$FILE_UUID/pose?frame=303" \
|
||||||
|
-H "X-API-Key: $KEY"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Response (200)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"frame": 303,
|
||||||
|
"keypoints": [
|
||||||
|
{"name": "nose", "x": 315.91, "y": 364.25, "confidence": 0.616},
|
||||||
|
{"name": "left_eye", "x": 312.45, "y": 362.80, "confidence": 0.616},
|
||||||
|
{"name": "right_eye", "x": 316.21, "y": 361.38, "confidence": 0.616},
|
||||||
|
{"name": "left_ear", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_ear", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "left_shoulder", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_shoulder", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "left_elbow", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_elbow", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "left_wrist", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_wrist", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "left_hip", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_hip", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "left_knee", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_knee", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "left_ankle", "x": 0.0, "y": 0.0, "confidence": 0.0},
|
||||||
|
{"name": "right_ankle", "x": 0.0, "y": 0.0, "confidence": 0.0}
|
||||||
|
],
|
||||||
|
"pose_class": "unknown",
|
||||||
|
"confidence": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `frame` | integer | Frame number |
|
||||||
|
| `keypoints` | array | 17 COCO keypoints (deduplicated) |
|
||||||
|
| `keypoints[].name` | string | Keypoint name |
|
||||||
|
| `keypoints[].x` | float | X coordinate in pixels |
|
||||||
|
| `keypoints[].y` | float | Y coordinate in pixels |
|
||||||
|
| `keypoints[].confidence` | float | Detection confidence 0-1 (0 means not detected) |
|
||||||
|
| `pose_class` | string | Always `"unknown"` in POC |
|
||||||
|
| `confidence` | null | Always `null` in POC |
|
||||||
|
|
||||||
|
#### COCO-17 Keypoint Names
|
||||||
|
|
||||||
|
```
|
||||||
|
nose, left_eye, right_eye, left_ear, right_ear,
|
||||||
|
left_shoulder, right_shoulder, left_elbow, right_elbow,
|
||||||
|
left_wrist, right_wrist, left_hip, right_hip,
|
||||||
|
left_knee, right_knee, left_ankle, right_ankle
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Error Responses
|
||||||
|
|
||||||
|
| HTTP | When |
|
||||||
|
|------|------|
|
||||||
|
| 404 | Frame not found or pose.json not found |
|
||||||
|
| 401 | Missing or invalid API key |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `GET /api/v1/file/:file_uuid/appearance`
|
||||||
|
|
||||||
|
**Auth**: Required
|
||||||
|
**Scope**: file-level
|
||||||
|
|
||||||
|
Get appearance color data for a specific frame. Returns HSV histogram for color analysis.
|
||||||
|
|
||||||
|
#### Path Parameters
|
||||||
|
|
||||||
|
| Field | Type | Required | Description |
|
||||||
|
|-------|------|----------|-------------|
|
||||||
|
| `file_uuid` | string | Yes | File UUID |
|
||||||
|
|
||||||
|
#### Query Parameters
|
||||||
|
|
||||||
|
| Field | Type | Required | Description |
|
||||||
|
|-------|------|----------|-------------|
|
||||||
|
| `frame` | integer | Yes | Frame number |
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s "$API/api/v1/file/$FILE_UUID/appearance?frame=303" \
|
||||||
|
-H "X-API-Key: $KEY"
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Response (200)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"frame": 303,
|
||||||
|
"dominant_colors": [],
|
||||||
|
"hsv_histogram": [
|
||||||
|
[0.59, 0.0, 0.0, ...],
|
||||||
|
[0.97, 0.02, 0.01, ...],
|
||||||
|
[0.004, 0.02, 0.03, ...]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `frame` | integer | Frame number |
|
||||||
|
| `dominant_colors` | array | Empty array in POC (`[]`) |
|
||||||
|
| `hsv_histogram` | array | 3x30 HSV histogram bins (H, S, V channels) |
|
||||||
|
|
||||||
|
#### Error Responses
|
||||||
|
|
||||||
|
| HTTP | When |
|
||||||
|
|------|------|
|
||||||
|
| 404 | Frame not found or appearance.json not found |
|
||||||
|
| 401 | Missing or invalid API key |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Source
|
||||||
|
|
||||||
|
| Processor | File | Location |
|
||||||
|
|-----------|------|----------|
|
||||||
|
| Pose | `{file_uuid}.pose.json` | `{OUTPUT_DIR}/` |
|
||||||
|
| Appearance | `{file_uuid}.appearance.json` | `{OUTPUT_DIR}/` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## POC Limitations
|
||||||
|
|
||||||
|
| Feature | POC Status | Future |
|
||||||
|
|---------|------------|--------|
|
||||||
|
| Pose classification | `"unknown"` | ML model for standing/sitting/etc. |
|
||||||
|
| Dominant colors | Empty array `[]` | HSV extraction algorithm |
|
||||||
|
| Multiple persons | First person only | Match by trace_id |
|
||||||
|
| Confidence | `null` | Aggregated confidence score |
|
||||||
|
|
||||||
|
---
|
||||||
|
*Updated: 2026-07-19 — New pose and appearance endpoints for face detail display*
|
||||||
@@ -1346,3 +1346,200 @@ async fn media_proxy_handler(
|
|||||||
pub fn media_proxy_routes() -> Router<crate::api::types::AppState> {
|
pub fn media_proxy_routes() -> Router<crate::api::types::AppState> {
|
||||||
Router::new().route("/api/v1/media-proxy", get(media_proxy_handler))
|
Router::new().route("/api/v1/media-proxy", get(media_proxy_handler))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Pose & Appearance Endpoints ──
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
struct PoseQuery {
|
||||||
|
frame: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
struct PoseKeypoint {
|
||||||
|
name: String,
|
||||||
|
x: f64,
|
||||||
|
y: f64,
|
||||||
|
confidence: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Serialize)]
|
||||||
|
struct PoseResponse {
|
||||||
|
frame: i64,
|
||||||
|
keypoints: Vec<PoseKeypoint>,
|
||||||
|
pose_class: String,
|
||||||
|
confidence: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
struct AppearanceQuery {
|
||||||
|
frame: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Serialize)]
|
||||||
|
struct AppearanceResponse {
|
||||||
|
frame: i64,
|
||||||
|
dominant_colors: Vec<serde_json::Value>,
|
||||||
|
hsv_histogram: Vec<Vec<f64>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_pose_handler(
|
||||||
|
State(_state): State<crate::api::types::AppState>,
|
||||||
|
Path(file_uuid): Path<String>,
|
||||||
|
Query(params): Query<PoseQuery>,
|
||||||
|
) -> Result<axum::Json<PoseResponse>, StatusCode> {
|
||||||
|
use crate::core::config::OUTPUT_DIR;
|
||||||
|
use std::collections::HashMap as StdHashMap;
|
||||||
|
|
||||||
|
let output_dir = OUTPUT_DIR.as_str();
|
||||||
|
let pose_path = std::path::Path::new(output_dir)
|
||||||
|
.join(format!("{}.pose.json", file_uuid));
|
||||||
|
|
||||||
|
if !pose_path.exists() {
|
||||||
|
tracing::error!("[get_pose] File not found: {}", pose_path.display());
|
||||||
|
return Err(StatusCode::NOT_FOUND);
|
||||||
|
}
|
||||||
|
|
||||||
|
let content = std::fs::read_to_string(&pose_path).map_err(|e| {
|
||||||
|
tracing::error!("[get_pose] Failed to read pose.json: {}", e);
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let data: serde_json::Value = serde_json::from_str(&content).map_err(|e| {
|
||||||
|
tracing::error!("[get_pose] Failed to parse pose.json: {}", e);
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let frames = data.get("frames").and_then(|v| v.as_array()).ok_or(StatusCode::NOT_FOUND)?;
|
||||||
|
|
||||||
|
let frame_data = frames
|
||||||
|
.iter()
|
||||||
|
.find(|f| f.get("frame").and_then(|v| v.as_i64()) == Some(params.frame))
|
||||||
|
.ok_or(StatusCode::NOT_FOUND)?;
|
||||||
|
|
||||||
|
let persons = frame_data
|
||||||
|
.get("persons")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.ok_or(StatusCode::NOT_FOUND)?;
|
||||||
|
|
||||||
|
if persons.is_empty() {
|
||||||
|
return Err(StatusCode::NOT_FOUND);
|
||||||
|
}
|
||||||
|
|
||||||
|
let person = &persons[0];
|
||||||
|
let keypoints_raw = person
|
||||||
|
.get("keypoints")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.ok_or(StatusCode::NOT_FOUND)?;
|
||||||
|
|
||||||
|
let mut unique_kps: StdHashMap<String, PoseKeypoint> = StdHashMap::new();
|
||||||
|
for kp in keypoints_raw {
|
||||||
|
let name = kp.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
let x = kp.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let y = kp.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
let conf = kp.get("confidence").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||||
|
|
||||||
|
if let Some(existing) = unique_kps.get(name) {
|
||||||
|
if conf > existing.confidence {
|
||||||
|
unique_kps.insert(name.to_string(), PoseKeypoint {
|
||||||
|
name: name.to_string(),
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
confidence: conf,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
unique_kps.insert(name.to_string(), PoseKeypoint {
|
||||||
|
name: name.to_string(),
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
confidence: conf,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let coco_names = [
|
||||||
|
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
||||||
|
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
||||||
|
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
||||||
|
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
||||||
|
];
|
||||||
|
|
||||||
|
let keypoints: Vec<PoseKeypoint> = coco_names
|
||||||
|
.iter()
|
||||||
|
.map(|name| {
|
||||||
|
unique_kps.get(*name).cloned().unwrap_or_else(|| PoseKeypoint {
|
||||||
|
name: name.to_string(),
|
||||||
|
x: 0.0,
|
||||||
|
y: 0.0,
|
||||||
|
confidence: 0.0,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(axum::Json(PoseResponse {
|
||||||
|
frame: params.frame,
|
||||||
|
keypoints,
|
||||||
|
pose_class: "unknown".to_string(),
|
||||||
|
confidence: None,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_appearance_handler(
|
||||||
|
State(_state): State<crate::api::types::AppState>,
|
||||||
|
Path(file_uuid): Path<String>,
|
||||||
|
Query(params): Query<AppearanceQuery>,
|
||||||
|
) -> Result<axum::Json<AppearanceResponse>, StatusCode> {
|
||||||
|
use crate::core::config::OUTPUT_DIR;
|
||||||
|
|
||||||
|
let output_dir = OUTPUT_DIR.as_str();
|
||||||
|
let appearance_path = std::path::Path::new(output_dir)
|
||||||
|
.join(format!("{}.appearance.json", file_uuid));
|
||||||
|
|
||||||
|
if !appearance_path.exists() {
|
||||||
|
tracing::error!("[get_appearance] File not found: {}", appearance_path.display());
|
||||||
|
return Err(StatusCode::NOT_FOUND);
|
||||||
|
}
|
||||||
|
|
||||||
|
let content = std::fs::read_to_string(&appearance_path).map_err(|e| {
|
||||||
|
tracing::error!("[get_appearance] Failed to read appearance.json: {}", e);
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let data: serde_json::Value = serde_json::from_str(&content).map_err(|e| {
|
||||||
|
tracing::error!("[get_appearance] Failed to parse appearance.json: {}", e);
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let frames = data.get("frames").and_then(|v| v.as_array()).ok_or(StatusCode::NOT_FOUND)?;
|
||||||
|
|
||||||
|
let frame_data = frames
|
||||||
|
.iter()
|
||||||
|
.find(|f| f.get("frame").and_then(|v| v.as_i64()) == Some(params.frame))
|
||||||
|
.ok_or(StatusCode::NOT_FOUND)?;
|
||||||
|
|
||||||
|
let hsv_histogram = frame_data
|
||||||
|
.get("hsv_histogram")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.map(|arr| {
|
||||||
|
arr.iter()
|
||||||
|
.filter_map(|channel| {
|
||||||
|
channel.as_array().map(|bins| {
|
||||||
|
bins.iter().filter_map(|v| v.as_f64()).collect()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
Ok(axum::Json(AppearanceResponse {
|
||||||
|
frame: params.frame,
|
||||||
|
dominant_colors: vec![],
|
||||||
|
hsv_histogram,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn pose_appearance_routes() -> Router<crate::api::types::AppState> {
|
||||||
|
Router::new()
|
||||||
|
.route("/api/v1/file/:file_uuid/pose", get(get_pose_handler))
|
||||||
|
.route("/api/v1/file/:file_uuid/appearance", get(get_appearance_handler))
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ pub mod media_api;
|
|||||||
pub mod middleware;
|
pub mod middleware;
|
||||||
pub mod pipeline;
|
pub mod pipeline;
|
||||||
pub mod processing;
|
pub mod processing;
|
||||||
|
pub mod profile;
|
||||||
pub mod scan;
|
pub mod scan;
|
||||||
pub mod search;
|
pub mod search;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
|
|||||||
+83
-1
@@ -1,10 +1,11 @@
|
|||||||
use axum::{
|
use axum::{
|
||||||
Extension, Json,
|
Extension, Json,
|
||||||
extract::{Query, State},
|
extract::{Path, Query, State},
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use crate::core::db::schema;
|
use crate::core::db::schema;
|
||||||
use crate::api::middleware::UserAuth;
|
use crate::api::middleware::UserAuth;
|
||||||
@@ -375,6 +376,86 @@ pub async fn update_file_profile_handler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── Face Groups ───
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct FaceGroup {
|
||||||
|
pub group_id: i32,
|
||||||
|
pub name: String,
|
||||||
|
pub trace_ids: Vec<i64>,
|
||||||
|
pub trace_count: i32,
|
||||||
|
pub representative_trace: i64,
|
||||||
|
pub editable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct FaceGroupsResponse {
|
||||||
|
pub success: bool,
|
||||||
|
pub file_uuid: String,
|
||||||
|
pub face_groups: Vec<FaceGroup>,
|
||||||
|
pub total_groups: i32,
|
||||||
|
pub unassigned_traces: Vec<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_face_groups_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Extension(_auth): Extension<UserAuth>,
|
||||||
|
Path(file_uuid): Path<String>,
|
||||||
|
) -> Result<Json<FaceGroupsResponse>, StatusCode> {
|
||||||
|
let tkg_table = schema::table_name("tkg_nodes");
|
||||||
|
|
||||||
|
let rows: Vec<(String, serde_json::Value)> = sqlx::query_as(&format!(
|
||||||
|
"SELECT label, properties FROM {} \
|
||||||
|
WHERE file_uuid = $1 AND node_type = 'face_track' \
|
||||||
|
ORDER BY (properties->>'trace_id')::int",
|
||||||
|
tkg_table
|
||||||
|
))
|
||||||
|
.bind(&file_uuid)
|
||||||
|
.fetch_all(state.db.pool())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
tracing::error!("[FaceGroups] DB error: {}", e);
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut groups: HashMap<String, Vec<i64>> = HashMap::new();
|
||||||
|
let mut unassigned: Vec<i64> = Vec::new();
|
||||||
|
|
||||||
|
for (label, properties) in rows {
|
||||||
|
let trace_id = properties
|
||||||
|
.get("trace_id")
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
if label.starts_with("Face Trace ") || label.starts_with("Trace ") {
|
||||||
|
unassigned.push(trace_id);
|
||||||
|
} else {
|
||||||
|
groups.entry(label).or_default().push(trace_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let face_groups: Vec<FaceGroup> = groups
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(idx, (name, trace_ids))| FaceGroup {
|
||||||
|
group_id: (idx + 1) as i32,
|
||||||
|
name,
|
||||||
|
trace_count: trace_ids.len() as i32,
|
||||||
|
representative_trace: trace_ids.first().copied().unwrap_or(0),
|
||||||
|
trace_ids,
|
||||||
|
editable: true,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(Json(FaceGroupsResponse {
|
||||||
|
success: true,
|
||||||
|
file_uuid,
|
||||||
|
total_groups: face_groups.len() as i32,
|
||||||
|
unassigned_traces: unassigned,
|
||||||
|
face_groups,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
// ─── Routes ───
|
// ─── Routes ───
|
||||||
|
|
||||||
pub fn profile_routes() -> axum::Router<AppState> {
|
pub fn profile_routes() -> axum::Router<AppState> {
|
||||||
@@ -388,4 +469,5 @@ pub fn profile_routes() -> axum::Router<AppState> {
|
|||||||
)
|
)
|
||||||
.route("/api/v1/file-profile", get(get_file_profile_handler))
|
.route("/api/v1/file-profile", get(get_file_profile_handler))
|
||||||
.route("/api/v1/file-profile", put(update_file_profile_handler))
|
.route("/api/v1/file-profile", put(update_file_profile_handler))
|
||||||
|
.route("/api/v1/file/:file_uuid/face-groups", get(get_face_groups_handler))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ use super::media_api;
|
|||||||
use super::middleware::unified_auth;
|
use super::middleware::unified_auth;
|
||||||
use super::pipeline;
|
use super::pipeline;
|
||||||
use super::processing;
|
use super::processing;
|
||||||
|
use super::profile;
|
||||||
use super::scan;
|
use super::scan;
|
||||||
use super::search::search_routes;
|
use super::search::search_routes;
|
||||||
use super::tmdb_api;
|
use super::tmdb_api;
|
||||||
@@ -119,12 +120,14 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
|
|||||||
.merge(identity_agent_api::identity_agent_routes())
|
.merge(identity_agent_api::identity_agent_routes())
|
||||||
.merge(media_api::bbox_routes())
|
.merge(media_api::bbox_routes())
|
||||||
.merge(media_api::media_proxy_routes())
|
.merge(media_api::media_proxy_routes())
|
||||||
|
.merge(media_api::pose_appearance_routes())
|
||||||
.merge(trace_agent_api::trace_agent_routes())
|
.merge(trace_agent_api::trace_agent_routes())
|
||||||
.merge(search_routes())
|
.merge(search_routes())
|
||||||
.merge(llm_search::llm_smart_routes())
|
.merge(llm_search::llm_smart_routes())
|
||||||
.merge(universal_search_routes())
|
.merge(universal_search_routes())
|
||||||
.merge(pipeline::pipeline_routes())
|
.merge(pipeline::pipeline_routes())
|
||||||
.merge(checkin_api::checkin_routes())
|
.merge(checkin_api::checkin_routes())
|
||||||
|
.merge(profile::profile_routes())
|
||||||
.layer(axum::middleware::from_fn_with_state(
|
.layer(axum::middleware::from_fn_with_state(
|
||||||
state.api_state.clone(),
|
state.api_state.clone(),
|
||||||
unified_auth,
|
unified_auth,
|
||||||
|
|||||||
Reference in New Issue
Block a user