# 人物身份整合架构设计

## 概述

将人脸识别（Face Recognition）和声纹识别（ASRX Speaker Diarization）整合，在视频块（Chunk）中标注人物身份。

## 架构设计

### 数据流

```
视频文件
    ↓
┌─────────────────────────────────────────────┐
│  并行处理                                      │
├─────────────────────────────────────────────┤
│  1. Face Detection → face_detections        │
│  2. ASRX → asrx_segments (speaker_id)       │
│  3. Chunk Generation → chunks               │
└─────────────────────────────────────────────┘
    ↓
┌─────────────────────────────────────────────┐
│  时间重叠分析                                  │
├─────────────────────────────────────────────┤
│  匹配规则：                                    │
│  - face_detections.timestamp ∈ [asrx.start, asrx.end]
│  - 提取时间重叠最大的配对                       │
└─────────────────────────────────────────────┘
    ↓
┌─────────────────────────────────────────────┐
│  创建人物身份关联                               │
├─────────────────────────────────────────────┤
│  person_identities (person_id)              │
│  ├─ face_id (外键)                           │
│  ├─ speaker_id (字符串)                      │
│  ├─ confidence (关联置信度)                   │
│  └─ file_uuid (来源视频)                     │
└─────────────────────────────────────────────┘
    ↓
┌─────────────────────────────────────────────┐
│  更新 Chunk 元数据                            │
├─────────────────────────────────────────────┤
│  chunks.metadata: {                         │
│    "person_identities": [                   │
│      {                                      │
│        "person_id": "person_xxx",           │
│        "face_id": "face_123",               │
│        "speaker_id": "SPEAKER_00",          │
│        "confidence": 0.85                   │
│      }                                      │
│    ]                                        │
│  }                                          │
└─────────────────────────────────────────────┘
```

## 数据库表设计

### 1. person_identities（人物身份表）

```sql
CREATE TABLE person_identities (
    id SERIAL PRIMARY KEY,
    person_id VARCHAR(255) NOT NULL UNIQUE,
    
    -- 身份关联
    face_identity_id INTEGER REFERENCES face_identities(id) ON DELETE SET NULL,
    speaker_id VARCHAR(64),  -- SPEAKER_00, SPEAKER_01, etc.
    
    -- 关联信息
    file_uuid VARCHAR(255) NOT NULL,
    confidence DOUBLE PRECISION DEFAULT 0.0,
    
    -- 元数据
    name VARCHAR(255),  -- 人物姓名（手动标注）
    metadata JSONB DEFAULT '{}'::jsonb,
    
    -- 时间戳
    first_appearance_time DOUBLE PRECISION,
    last_appearance_time DOUBLE PRECISION,
    total_appearance_duration DOUBLE PRECISION DEFAULT 0.0,
    appearance_count INTEGER DEFAULT 0,
    
    -- 审计字段
    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
    is_confirmed BOOLEAN DEFAULT FALSE,  -- 用户确认的身份
    
    -- 约束
    CONSTRAINT unique_person_identity UNIQUE (file_uuid, face_identity_id, speaker_id)
);

CREATE INDEX idx_person_identities_file_uuid ON person_identities(file_uuid);
CREATE INDEX idx_person_identities_face ON person_identities(face_identity_id);
CREATE INDEX idx_person_identities_speaker ON person_identities(speaker_id);
CREATE INDEX idx_person_identities_name ON person_identities(name);
```

### 2. person_appearances（人物出场记录表）

```sql
CREATE TABLE person_appearances (
    id SERIAL PRIMARY KEY,
    person_id VARCHAR(255) NOT NULL REFERENCES person_identities(person_id) ON DELETE CASCADE,
    
    -- 出场信息
    file_uuid VARCHAR(255) NOT NULL,
    start_time DOUBLE PRECISION NOT NULL,
    end_time DOUBLE PRECISION NOT NULL,
    duration DOUBLE PRECISION NOT NULL,
    
    -- 来源信息
    face_detection_id INTEGER REFERENCES face_detections(id) ON DELETE SET NULL,
    asrx_segment_id INTEGER,  -- 暂不设外键，ASRX 结果存储在 JSON 中
    
    -- 元数据
    confidence DOUBLE PRECISION DEFAULT 0.0,
    metadata JSONB DEFAULT '{}'::jsonb,
    
    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);

CREATE INDEX idx_person_appearances_person ON person_appearances(person_id);
CREATE INDEX idx_person_appearances_video ON person_appearances(file_uuid);
CREATE INDEX idx_person_appearances_time ON person_appearances(file_uuid, start_time, end_time);
```

### 3. 增强 chunks 表

```sql
-- 在 chunks.metadata 中添加人物身份信息
-- 示例结构：
{
  "person_identities": [
    {
      "person_id": "person_abc123",
      "face_id": "face_456",
      "speaker_id": "SPEAKER_00",
      "confidence": 0.85,
      "name": "张三"
    }
  ],
  "speaker_id": "SPEAKER_00",  -- 主要说话人
  "face_count": 2  // 检测到的人脸数量
}
```

## 核心算法

### 算法 1：时间重叠匹配

```python
def match_face_with_speaker(face_detections, asrx_segments, threshold=0.5):
    """
    根据时间重叠匹配人脸和说话人
    
    参数：
    - face_detections: 人脸检测列表 [{timestamp, face_id, ...}]
    - asrx_segments: ASRX 片段列表 [{start, end, speaker_id, ...}]
    - threshold: 最小重叠比例阈值
    
    返回：
    - 匹配列表 [{face_id, speaker_id, confidence}]
    """
    matches = []
    
    for face in face_detections:
        face_time = face['timestamp']
        
        # 找到时间重叠的 ASRX 片段
        for segment in asrx_segments:
            if segment['start'] <= face_time <= segment['end']:
                # 计算重叠比例
                overlap_duration = min(face_time - segment['start'], 
                                      segment['end'] - face_time)
                total_duration = segment['end'] - segment['start']
                overlap_ratio = overlap_duration / total_duration
                
                if overlap_ratio >= threshold:
                    matches.append({
                        'face_id': face['face_id'],
                        'speaker_id': segment['speaker_id'],
                        'confidence': overlap_ratio,
                        'timestamp': face_time
                    })
    
    return matches
```

### 算法 2：人物身份聚类

```python
def cluster_person_identities(matches, face_embeddings, similarity_threshold=0.7):
    """
    将匹配结果聚类为人物身份
    
    参数：
    - matches: 匹配列表
    - face_embeddings: 人脸嵌入向量 {face_id: embedding}
    - similarity_threshold: 相似度阈值
    
    返回：
    - 人物身份列表 [{person_id, face_ids, speaker_ids}]
    """
    from sklearn.cluster import DBSCAN
    import numpy as np
    
    # 收集所有 face_id 和对应的嵌入向量
    face_ids = list(set(m['face_id'] for m in matches))
    embeddings = [face_embeddings[face_id] for face_id in face_ids]
    
    # 聚类
    clustering = DBSCAN(eps=1-similarity_threshold, min_samples=2, metric='cosine')
    labels = clustering.fit_predict(embeddings)
    
    # 按聚类分组
    person_identities = {}
    for face_id, label in zip(face_ids, labels):
        if label == -1:
            continue  # 噪声
        
        person_id = f"person_{label}"
        if person_id not in person_identities:
            person_identities[person_id] = {
                'person_id': person_id,
                'face_ids': [],
                'speaker_ids': set()
            }
        
        person_identities[person_id]['face_ids'].append(face_id)
        
        # 添加对应的 speaker_id
        for match in matches:
            if match['face_id'] == face_id:
                person_identities[person_id]['speaker_ids'].add(match['speaker_id'])
    
    # 转换 set 为 list
    for person in person_identities.values():
        person['speaker_ids'] = list(person['speaker_ids'])
    
    return list(person_identities.values())
```

### 算法 3：更新 Chunk 人物信息

```python
def update_chunk_person_identities(chunk, person_appearances):
    """
    更新 Chunk 的人物身份信息
    
    参数：
    - chunk: Chunk 对象
    - person_appearances: 人物出场记录列表
    
    返回：
    - 更新后的 Chunk
    """
    chunk_start = chunk['start_time']
    chunk_end = chunk['end_time']
    
    # 找到与 Chunk 时间重叠的人物出场
    overlapping_persons = []
    for appearance in person_appearances:
        if (appearance['start_time'] <= chunk_end and 
            appearance['end_time'] >= chunk_start):
            
            # 计算重叠时长
            overlap_start = max(chunk_start, appearance['start_time'])
            overlap_end = min(chunk_end, appearance['end_time'])
            overlap_duration = overlap_end - overlap_start
            
            overlapping_persons.append({
                'person_id': appearance['person_id'],
                'name': appearance.get('name'),
                'overlap_duration': overlap_duration,
                'confidence': appearance['confidence']
            })
    
    # 按重叠时长排序
    overlapping_persons.sort(key=lambda x: x['overlap_duration'], reverse=True)
    
    # 更新 Chunk 元数据
    metadata = chunk.get('metadata', {})
    metadata['person_identities'] = overlapping_persons
    
    # 设置主要人物（重叠时长最长）
    if overlapping_persons:
        metadata['primary_person'] = overlapping_persons[0]['person_id']
    
    chunk['metadata'] = metadata
    
    return chunk
```

## API 设计

### 1. 创建人物身份关联

```http
POST /api/v1/person/identify
Content-Type: application/json

{
  "file_uuid": "abc123",
  "auto_match": true,
  "match_threshold": 0.5
}

Response:
{
  "success": true,
  "message": "Identified 3 persons",
  "persons": [
    {
      "person_id": "person_0",
      "face_ids": ["face_123", "face_456"],
      "speaker_ids": ["SPEAKER_00"],
      "confidence": 0.85,
      "appearance_count": 15,
      "total_duration": 120.5
    }
  ]
}
```

### 2. 查询人物出场时间轴

```http
GET /api/v1/person/:person_id/timeline?file_uuid=abc123

Response:
{
  "success": true,
  "person_id": "person_0",
  "name": "张三",
  "timeline": [
    {
      "start_time": 10.5,
      "end_time": 25.3,
      "duration": 14.8,
      "confidence": 0.92
    },
    {
      "start_time": 45.0,
      "end_time": 60.2,
      "duration": 15.2,
      "confidence": 0.88
    }
  ],
  "statistics": {
    "total_appearances": 15,
    "total_duration": 120.5,
    "first_appearance": 10.5,
    "last_appearance": 350.2
  }
}
```

### 3. 手动标注人物姓名

```http
PATCH /api/v1/person/:person_id
Content-Type: application/json

{
  "name": "张三",
  "metadata": {
    "role": "主持人",
    "department": "新闻部"
  }
}

Response:
{
  "success": true,
  "message": "Person identity updated",
  "person_id": "person_0"
}
```

### 4. 查询 Chunk 中的人物

```http
GET /api/v1/chunks/:chunk_id/persons

Response:
{
  "success": true,
  "chunk_id": "sentence_0012",
  "persons": [
    {
      "person_id": "person_0",
      "name": "张三",
      "confidence": 0.85,
      "overlap_duration": 3.5
    }
  ]
}
```

## 实现步骤

### Phase 1: 数据库表创建 (Day 1)

1. ✅ 创建迁移文件 `007_person_identity_tables.sql`
2. ✅ 创建 `person_identities` 表
3. ✅ 创建 `person_appearances` 表
4. ✅ 创建索引和约束
5. ✅ 运行迁移测试

### Phase 2: 核心算法实现 (Day 2-3)

1. ⏳ 实现 Rust 结构体
   - `PersonIdentity`
   - `PersonAppearance`
   - `PersonMatch`

2. ⏳ 实现匹配算法
   - `match_face_with_speaker()`
   - `cluster_person_identities()`
   - `update_chunk_person_identities()`

3. ⏳ 实现数据库操作
   - `store_person_identity()`
   - `store_person_appearance()`
   - `update_chunks_with_persons()`

### Phase 3: API 实现 (Day 4)

1. ⏳ 创建 `src/api/person_identity.rs`
2. ⏳ 实现 API 端点
   - `POST /api/v1/person/identify`
   - `GET /api/v1/person/:person_id/timeline`
   - `PATCH /api/v1/person/:person_id`
   - `GET /api/v1/chunks/:chunk_id/persons`

3. ⏳ 添加路由到 `server.rs`

### Phase 4: 集成测试 (Day 5)

1. ⏳ 准备测试视频
2. ⏳ 运行完整处理流程
   - Face Detection
   - ASRX
   - Chunk Generation
   - Person Identity Creation

3. ⏳ 验证结果
   - 数据库记录正确性
   - API 响应正确性
   - 时间轴查询正确性

### Phase 5: 文档和优化 (Day 6)

1. ⏳ 编写 API 文档
2. ⏳ 编写使用指南
3. ⏳ 性能优化
4. ⏳ 错误处理增强

## 性能优化

### 1. 批量插入

```rust
// 使用事务批量插入人物出场记录
pub async fn batch_insert_person_appearances(
    db: &PostgresDb,
    appearances: &[PersonAppearance],
) -> Result<()> {
    let mut tx = db.pool().begin().await?;
    
    for appearance in appearances {
        sqlx::query(r#"
            INSERT INTO person_appearances (
                person_id, file_uuid, start_time, end_time, 
                duration, confidence, metadata
            ) VALUES ($1, $2, $3, $4, $5, $6, $7)
        "#)
        .bind(&appearance.person_id)
        .bind(&appearance.file_uuid)
        .bind(appearance.start_time)
        .bind(appearance.end_time)
        .bind(appearance.duration)
        .bind(appearance.confidence)
        .bind(&appearance.metadata)
        .execute(&mut *tx)
        .await?;
    }
    
    tx.commit().await?;
    Ok(())
}
```

### 2. 索引优化

```sql
-- 为常用查询添加复合索引
CREATE INDEX idx_person_appearances_video_time 
ON person_appearances(file_uuid, start_time, end_time);

CREATE INDEX idx_person_identities_video_face 
ON person_identities(file_uuid, face_identity_id);

CREATE INDEX idx_person_identities_video_speaker 
ON person_identities(file_uuid, speaker_id);
```

### 3. 缓存策略

```rust
// 使用 Redis 缓存人物时间轴查询
pub async fn get_person_timeline_cached(
    redis: &RedisClient,
    person_id: &str,
    file_uuid: &str,
) -> Result<Vec<PersonAppearance>> {
    let cache_key = format!("person_timeline:{}:{}", file_uuid, person_id);
    
    // 尝试从缓存获取
    if let Some(cached) = redis.get(&cache_key).await? {
        return Ok(serde_json::from_str(&cached)?);
    }
    
    // 从数据库查询
    let timeline = query_person_timeline_from_db(person_id, file_uuid).await?;
    
    // 缓存结果（5分钟）
    redis.set_ex(&cache_key, &serde_json::to_string(&timeline)?, 300).await?;
    
    Ok(timeline)
}
```

## 错误处理

### 1. 匹配置信度过低

```rust
if confidence < MIN_MATCH_CONFIDENCE {
    tracing::warn!(
        "[PERSON] Low confidence match: face={}, speaker={}, confidence={}",
        face_id, speaker_id, confidence
    );
    // 记录但不创建关联
    return Ok(None);
}
```

### 2. 重复匹配

```rust
// 检查是否已存在相同关联
let existing = sqlx::query!(
    "SELECT id FROM person_identities 
     WHERE file_uuid = $1 AND face_identity_id = $2 AND speaker_id = $3",
    file_uuid, face_id, speaker_id
)
.fetch_optional(db.pool())
.await?;

if existing.is_some() {
    tracing::info!("[PERSON] Identity already exists, skipping");
    return Ok(());
}
```

### 3. 时间范围无效

```rust
if start_time >= end_time {
    anyhow::bail!(
        "Invalid time range: start={} >= end={}",
        start_time, end_time
    );
}
```

## 监控指标

```rust
// Prometheus 指标
lazy_static! {
    static ref PERSON_IDENTITIES_CREATED: Counter = 
        register_counter!("person_identities_created_total").unwrap();
    
    static ref PERSON_MATCHES_TOTAL: Counter = 
        register_counter!("person_matches_total").unwrap();
    
    static ref PERSON_MATCH_CONFIDENCE: Histogram = 
        register_histogram!("person_match_confidence").unwrap();
}
```

## 未来扩展

### 1. 多模态融合

- 结合 OCR 文字识别（字幕、名牌）
- 结合场景分类（新闻演播室、会议室）
- 结合姿态识别（站立、坐着）

### 2. 跨视频人物追踪

- 全局人物身份库
- 人脸嵌入向量相似度匹配
- 服装、配饰特征

### 3. 实时处理

- 流式视频处理
- 实时人物识别
- WebSocket 推送更新

## 参考资料

- [InsightFace Documentation](https://github.com/deepinsight/insightface)
- [WhisperX Speaker Diarization](https://github.com/m-bain/whisperX)
- [PostgreSQL pgvector](https://github.com/pgvector/pgvector)
- [DBSCAN Clustering Algorithm](https://scikit-learn.org/stable/modules/clustering.html#dbscan)
