feat: implement skin_tone_trace node builder and standardize TKG node naming
- Add build_skin_tone_trace_nodes() to tkg.rs (Fitzpatrick I-VI classification) - Add skin_tone_trace_nodes field to TkgResult - Standardize node naming: _trace -> _track (text uses _region) - Add external_id format column to Node Types table - Add storage names to Edge Types table - Create TKG_FORMATION_V1.0.md with Phase 0-4 definition, flow diagram, queries - Add cross-reference from identity_agent_v4.0.md to TKG Formation - Update Python scripts to executable mode
This commit is contained in:
Accusys
@@ -6,19 +6,23 @@
|
||||
|
||||
TKG is a time-aligned knowledge graph built from multi-processor outputs (face, yolo, ocr, pose, asrx, gaze, lip, appearance). It produces 9 node types and 14 edge types stored in `dev.tkg_nodes` and `dev.tkg_edges`.
|
||||
|
||||
**Node naming convention:** All trace types use `_track` suffix. Text uses `_region` (non-temporal).
|
||||
|
||||
**See also:** `docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md` for formation phases, flow diagrams, and query examples.
|
||||
|
||||
### Node Types
|
||||
|
||||
| Node Type | Description | Key Properties |
|
||||
|-----------|-------------|----------------|
|
||||
| `face_track` | A tracked face identity over time | `trace_id`, `frame_count`, `status`, `pending_identity_name`, `confidence`, `identity_uuid` (see Identity Agent section below) |
|
||||
| `gaze_track` | Gaze direction over time | `direction` (frontal/left/right/up/down + diagonals) |
|
||||
| `lip_track` | Lip movement synced with speech | `speaker_id`, `lip_area_range` |
|
||||
| `text_region` | Spoken text aligned to time | `speaker_id`, `text`, `start_time`, `end_time` |
|
||||
| `appearance_trace` | Human appearance (clothing) over time | `clothing_color`, `upper_cloth`, `lower_cloth` |
|
||||
| `skin_tone_trace` | Fitzpatrick skin tone classification | `fitzpatrick_type` (I–VI) |
|
||||
| `accessory` | Detected accessories | `type` (glasses/hat/etc.), `confidence` |
|
||||
| `object` | YOLO-detected object | `class`, `confidence`, `frame_count` |
|
||||
| `speaker` | ASRX speaker segment | `speaker_id`, `segment_count`, `total_duration` |
|
||||
| Node Type | External ID Format | Description | Key Properties |
|
||||
|-----------|-------------------|-------------|----------------|
|
||||
| `face_track` | `face_track_{trace_id}` | A tracked face identity over time | `trace_id`, `frame_count`, `status`, `pending_identity_name`, `confidence`, `identity_uuid` |
|
||||
| `gaze_track` | `gaze_track_{id}` | Gaze direction over time | `direction` (frontal/left/right/up/down + diagonals) |
|
||||
| `lip_track` | `lip_track_{id}` | Lip movement synced with speech | `speaker_id`, `lip_area_range` |
|
||||
| `text_region` | `text_region_{id}` | Spoken text aligned to time | `speaker_id`, `text`, `start_time`, `end_time` |
|
||||
| `appearance_trace` | `appearance_{trace_id}` | Human appearance (clothing) over time | `clothing_color`, `upper_cloth`, `lower_cloth` |
|
||||
| `skin_tone_trace` | `skin_tone_{trace_id}` | Fitzpatrick skin tone classification | `fitzpatrick_type` (I–VI) |
|
||||
| `accessory` | `accessory_{id}` | Detected accessories | `type` (glasses/hat/etc.), `confidence` |
|
||||
| `object` | `object_{class}_{id}` | YOLO-detected object | `class`, `confidence`, `frame_count` |
|
||||
| `speaker` | `speaker_{speaker_id}` | ASRX speaker segment | `speaker_id`, `segment_count`, `total_duration` |
|
||||
|
||||
---
|
||||
|
||||
@@ -69,15 +73,16 @@ Identity Agent marks face_track nodes with identity binding status.
|
||||
|
||||
### Edge Types
|
||||
|
||||
| Edge Type | Source → Target | Description |
|
||||
|-----------|-----------------|-------------|
|
||||
| `co_occurs` | object ↔ object | Two objects appear together in same frame |
|
||||
| `speaker_face` | speaker ↔ face_trace | Speaker matched to face trace via lip sync |
|
||||
| `face_face` | face_trace ↔ face_trace | Two face traces interact (mutual gaze) |
|
||||
| `mutual_gaze` | gaze_trace ↔ gaze_trace | Two people looking at each other |
|
||||
| `lip_sync` | lip_trace ↔ text_trace | Lip movement aligned with spoken text |
|
||||
| `has_appearance` | face_trace ↔ appearance_trace | Face has specific appearance |
|
||||
| `wears` | face_trace ↔ accessory | Face wears an accessory |
|
||||
| Edge Type | Storage Name | Source → Target | Description |
|
||||
|-----------|--------------|-----------------|-------------|
|
||||
| `co_occurs` | `CO_OCCURS_WITH` | object ↔ object | Two objects appear together in same frame |
|
||||
| `speaker_face` | `SPEAKS_AS` | speaker → face_track | Speaker matched to face track via lip sync |
|
||||
| `face_face` | `INTERACTS_WITH` | face_track ↔ face_track | Two face tracks interact (mutual gaze) |
|
||||
| `mutual_gaze` | `MUTUAL_GAZE` | gaze_track ↔ gaze_track | Two people looking at each other |
|
||||
| `lip_sync` | `LIP_SYNC` | lip_track → text_region | Lip movement aligned with spoken text |
|
||||
| `has_appearance` | `HAS_APPEARANCE` | face_track → appearance_trace | Face has specific appearance |
|
||||
| `wears` | `WEARS` | face_track → accessory | Face wears an accessory |
|
||||
| `hand_object` | `HOLDS` | hand → object | Hand holding object |
|
||||
|
||||
---
|
||||
|
||||
@@ -102,10 +107,10 @@ curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/rebuild" \
|
||||
"success": true,
|
||||
"file_uuid": "d3f9ae8e471a1fc4d47022c66091b920",
|
||||
"result": {
|
||||
"face_trace_nodes": 16,
|
||||
"gaze_trace_nodes": 16,
|
||||
"lip_trace_nodes": 12,
|
||||
"text_trace_nodes": 24,
|
||||
"face_track_nodes": 16,
|
||||
"gaze_track_nodes": 16,
|
||||
"lip_track_nodes": 12,
|
||||
"text_region_nodes": 24,
|
||||
"appearance_trace_nodes": 8,
|
||||
"skin_tone_trace_nodes": 5,
|
||||
"accessory_nodes": 3,
|
||||
@@ -143,18 +148,18 @@ Query TKG nodes with pagination and optional type filter.
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|-------|------|----------|---------|-------------|
|
||||
| `node_type` | string | No | all | Filter by node type: `face_trace`, `gaze_trace`, `lip_trace`, `text_trace`, `appearance_trace`, `skin_tone_trace`, `accessory`, `object`, `speaker` |
|
||||
| `node_type` | string | No | all | Filter by node type: `face_track`, `gaze_track`, `lip_track`, `text_region`, `appearance_trace`, `skin_tone_trace`, `accessory`, `object`, `speaker` |
|
||||
| `page` | integer | No | 1 | Page number |
|
||||
| `page_size` | integer | No | 100 | Items per page (max 500) |
|
||||
|
||||
#### Example
|
||||
|
||||
```bash
|
||||
# Get all face_trace nodes
|
||||
# Get all face_track nodes
|
||||
curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/nodes" \
|
||||
-H "X-API-Key: $KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"node_type": "face_trace", "page": 1, "page_size": 50}'
|
||||
-d '{"node_type": "face_track", "page": 1, "page_size": 50}'
|
||||
|
||||
# Get all nodes
|
||||
curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/nodes" \
|
||||
@@ -175,12 +180,12 @@ curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/nodes" \
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"node_type": "face_trace",
|
||||
"external_id": "trace_0",
|
||||
"label": "Face Trace 0",
|
||||
"node_type": "face_track",
|
||||
"external_id": "face_track_0",
|
||||
"label": "Face Track 0",
|
||||
"properties": {
|
||||
"trace_id": 0,
|
||||
"face_count": 142,
|
||||
"frame_count": 142,
|
||||
"avg_confidence": 0.87
|
||||
}
|
||||
}
|
||||
@@ -224,17 +229,17 @@ Query TKG edges with pagination and optional filters.
|
||||
#### Example
|
||||
|
||||
```bash
|
||||
# Get all co_occurrence edges
|
||||
# Get all co_occurs edges
|
||||
curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/edges" \
|
||||
-H "X-API-Key: $KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"edge_type": "co_occurs"}'
|
||||
|
||||
# Get edges between face_trace and speaker nodes
|
||||
# Get edges between face_track and speaker nodes
|
||||
curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/edges" \
|
||||
-H "X-API-Key: $KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"source_type": "speaker", "target_type": "face_trace"}'
|
||||
-d '{"source_type": "speaker", "target_type": "face_track"}'
|
||||
```
|
||||
|
||||
#### Response (200)
|
||||
@@ -298,12 +303,12 @@ curl -s "$API/api/v1/file/$FILE_UUID/tkg/node/1" \
|
||||
"success": true,
|
||||
"node": {
|
||||
"id": 1,
|
||||
"node_type": "face_trace",
|
||||
"external_id": "trace_0",
|
||||
"label": "Face Trace 0",
|
||||
"node_type": "face_track",
|
||||
"external_id": "face_track_0",
|
||||
"label": "Face Track 0",
|
||||
"properties": {
|
||||
"trace_id": 0,
|
||||
"face_count": 142,
|
||||
"frame_count": 142,
|
||||
"avg_confidence": 0.87
|
||||
}
|
||||
},
|
||||
@@ -422,4 +427,4 @@ curl -s "$API/api/v1/file/$FILE_UUID/processor-counts" \
|
||||
|
||||
---
|
||||
|
||||
*Updated: 2026-06-20 12:00:00*
|
||||
*Updated: 2026-06-25 17:00:00*
|
||||
|
||||
499
docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md
Normal file
499
docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md
Normal file
@@ -0,0 +1,499 @@
|
||||
---
|
||||
title: TKG Formation V1.0
|
||||
version: 1.0
|
||||
date: 2026-06-25
|
||||
author: OpenCode
|
||||
status: draft
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Temporal Knowledge Graph (TKG) is built from multi-processor outputs to create a time-aligned knowledge graph. This document defines the formation phases, node/edge types, data flow, and integration with Identity Agent.
|
||||
|
||||
---
|
||||
|
||||
## Phase Definition
|
||||
|
||||
| Phase | Name | Trigger | Input | Output | Code Location |
|
||||
|-------|------|---------|-------|--------|---------------|
|
||||
| **Phase 0** | Populate | TKG rebuild | `face.json` | `PG face_detections.trace_id` | `tkg.rs:20-100` |
|
||||
| **Phase 1** | Extract | Video register | Video frames | `Qdrant _faces` (512D embeddings) | `face_processor.py` |
|
||||
| **Phase 2** | Build Nodes | TKG rebuild | `face.json`, PG tables | `tkg_nodes` (9 types) | `tkg.rs:506-515` |
|
||||
| **Phase 3** | Build Edges | TKG rebuild | `tkg_nodes`, pose data | `tkg_edges` (9 types) | `tkg.rs:517-524` |
|
||||
| **Phase 4** | Identity | Manual/API call | `Qdrant _faces`, `_seeds` | `tkg_nodes.status` updated | `identity_matcher.py` |
|
||||
|
||||
### Phase Details
|
||||
|
||||
#### Phase 0: Populate
|
||||
|
||||
**Purpose:** Assign `trace_id` to face detections.
|
||||
|
||||
**Flow:**
|
||||
1. Check if `trace_id IS NOT NULL` already exists in `face_detections`
|
||||
2. If not, call `store_traced_faces.py`
|
||||
3. `store_traced_faces.py` runs `face_tracker.py` (IoU-only)
|
||||
4. Update `face_detections.trace_id`
|
||||
|
||||
**Dependency:** `face.json` must exist (from Phase 1)
|
||||
|
||||
---
|
||||
|
||||
#### Phase 1: Extract
|
||||
|
||||
**Purpose:** Generate face embeddings and push to Qdrant.
|
||||
|
||||
**Flow:**
|
||||
1. `face_processor.py` runs Vision detection (ANE)
|
||||
2. Crop faces from video frames
|
||||
3. CoreML FaceNet → 512D embedding
|
||||
4. Push to Qdrant `_faces` collection
|
||||
5. Write `face.json` (metadata only, no embedding)
|
||||
|
||||
**Output:**
|
||||
- `face.json` in output directory
|
||||
- Qdrant `_faces` collection with embeddings
|
||||
|
||||
---
|
||||
|
||||
#### Phase 2: Build Nodes
|
||||
|
||||
**Purpose:** Create TKG nodes from processor outputs.
|
||||
|
||||
**Node Builders:**
|
||||
|
||||
| Builder | Node Type | Data Source |
|
||||
|---------|-----------|-------------|
|
||||
| `build_face_track_nodes` | `face_track` | `face.json`, `face_detections` |
|
||||
| `build_gaze_track_nodes` | `gaze_track` | `face.json` (pose data) |
|
||||
| `build_lip_track_nodes` | `lip_track` | `face.json` (lips), `asrx.json` |
|
||||
| `build_text_region_nodes` | `text_region` | `asrx.json` |
|
||||
| `build_appearance_trace_nodes` | `appearance_trace` | `yolo.json` (person) |
|
||||
| `build_accessory_nodes` | `accessory` | `yolo.json` |
|
||||
| `build_yolo_object_nodes` | `object` | `yolo.json` |
|
||||
| `build_hand_nodes` | `hand` | `pose.json` |
|
||||
| `build_speaker_nodes` | `speaker` | `asrx.json` |
|
||||
| `build_skin_tone_trace_nodes` | `skin_tone_trace` | **TODO** |
|
||||
|
||||
---
|
||||
|
||||
#### Phase 3: Build Edges
|
||||
|
||||
**Purpose:** Create TKG edges from node relationships.
|
||||
|
||||
**Edge Builders:**
|
||||
|
||||
| Builder | Edge Type | Source → Target |
|
||||
|---------|-----------|-----------------|
|
||||
| `build_co_occurrence_edges` | `co_occurs` | `object ↔ object` |
|
||||
| `build_speaker_face_edges` | `speaker_face` | `speaker ↔ face_track` |
|
||||
| `build_face_face_edges` | `face_face` | `face_track ↔ face_track` |
|
||||
| `build_mutual_gaze_edges` | `mutual_gaze` | `gaze_track ↔ gaze_track` |
|
||||
| `build_lip_sync_edges` | `lip_sync` | `lip_track ↔ text_region` |
|
||||
| `build_has_appearance_edges` | `has_appearance` | `face_track ↔ appearance_trace` |
|
||||
| `build_wears_edges` | `wears` | `face_track ↔ accessory` |
|
||||
| `build_hand_object_edges` | `hand_object` | `hand ↔ object` |
|
||||
|
||||
---
|
||||
|
||||
#### Phase 4: Identity
|
||||
|
||||
**Purpose:** Mark face_track nodes with identity binding status.
|
||||
|
||||
**Flow:**
|
||||
1. Identity Agent queries `_seeds` collection (TMDb/manual/propagation)
|
||||
2. Queries `_faces` collection for trace representatives
|
||||
3. Multi-angle matching (3 reps per trace)
|
||||
4. Mark TKG nodes: `status='suggested'`, `confidence`, `pending_identity_name`
|
||||
5. User confirms → update TKG, Qdrant, PG
|
||||
6. Confirmed trace becomes propagation seed in `_seeds`
|
||||
|
||||
---
|
||||
|
||||
## Node Types (Naming Standardized)
|
||||
|
||||
**Naming Rule:**
|
||||
- All trace types use `_track` suffix
|
||||
- Text uses `_region` (non-temporal)
|
||||
|
||||
| Node Type | External ID Format | Key Properties |
|
||||
|-----------|---------------------|----------------|
|
||||
| `face_track` | `face_track_{trace_id}` | `trace_id`, `frame_count`, `start_frame`, `end_frame`, `avg_bbox`, `status`, `confidence`, `identity_uuid` |
|
||||
| `gaze_track` | `gaze_track_{id}` | `direction` (frontal/left/right/up/down + diagonals) |
|
||||
| `lip_track` | `lip_track_{id}` | `speaker_id`, `lip_area_range` |
|
||||
| `text_region` | `text_region_{id}` | `speaker_id`, `text`, `start_time`, `end_time` |
|
||||
| `appearance_trace` | `appearance_{trace_id}` | `clothing_color`, `upper_cloth`, `lower_cloth` |
|
||||
| `skin_tone_trace` | `skin_tone_{trace_id}` | `fitzpatrick_type` (I-VI) - **TODO** |
|
||||
| `accessory` | `accessory_{id}` | `type` (glasses/hat/etc.), `confidence` |
|
||||
| `object` | `object_{class}_{id}` | `class`, `confidence`, `frame_count` |
|
||||
| `speaker` | `speaker_{speaker_id}` | `speaker_id`, `segment_count`, `total_duration` |
|
||||
|
||||
### face_track Identity Properties
|
||||
|
||||
| Property | Type | Values |
|
||||
|----------|------|--------|
|
||||
| `status` | string | `pending` | `suggested` | `confirmed` | `stranger` |
|
||||
| `pending_identity_name` | string/null | Suggested identity name |
|
||||
| `pending_identity_uuid` | string/null | Suggested identity UUID |
|
||||
| `suggested_by` | string/null | `tmdb` | `propagation` | `manual` |
|
||||
| `confidence` | float/null | Matching score (0.0-1.0) |
|
||||
| `identity_uuid` | string/null | Confirmed identity UUID |
|
||||
| `identity_id` | integer/null | Confirmed PG identity.id |
|
||||
| `identity_ref` | string/null | Reference string (e.g., `file_uuid:identity_1`) |
|
||||
| `stranger_id` | integer/null | Stranger cluster ID |
|
||||
| `stranger_ref` | string/null | Reference string (e.g., `stranger_1`) |
|
||||
|
||||
---
|
||||
|
||||
## Edge Types
|
||||
|
||||
| Edge Type | Storage Name | Source → Target | Properties |
|
||||
|-----------|--------------|-----------------|------------|
|
||||
| `co_occurs` | `CO_OCCURS_WITH` | `object ↔ object` | `frame_count`, `confidence` |
|
||||
| `speaker_face` | `SPEAKS_AS` | `speaker → face_track` | `overlap_frames`, `confidence` |
|
||||
| `face_face` | `INTERACTS_WITH` | `face_track ↔ face_track` | `co_occurrence_frames` |
|
||||
| `mutual_gaze` | `MUTUAL_GAZE` | `gaze_track ↔ gaze_track` | `frame_count` |
|
||||
| `lip_sync` | `LIP_SYNC` | `lip_track → text_region` | `speaker_id` |
|
||||
| `has_appearance` | `HAS_APPEARANCE` | `face_track → appearance_trace` | `frame_count` |
|
||||
| `wears` | `WEARS` | `face_track → accessory` | `confidence` |
|
||||
| `hand_object` | `HOLDS` | `hand → object` | `frame_count`, `confidence` |
|
||||
|
||||
---
|
||||
|
||||
## Data Flow Diagram
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph Phase0[Phase 0: Populate]
|
||||
A[face.json] --> B[store_traced_faces.py]
|
||||
B --> C[PG face_detections.trace_id]
|
||||
end
|
||||
|
||||
subgraph Phase1[Phase 1: Extract]
|
||||
D[Video Frames] --> E[face_processor.py]
|
||||
E --> F[face.json metadata]
|
||||
E --> G[Qdrant _faces 512D]
|
||||
end
|
||||
|
||||
subgraph Phase2[Phase 2: Build Nodes]
|
||||
C --> H[TKG Builder]
|
||||
F --> H
|
||||
I[pose.json] --> H
|
||||
J[asrx.json] --> H
|
||||
K[yolo.json] --> H
|
||||
H --> L[tkg_nodes<br/>9 node types]
|
||||
end
|
||||
|
||||
subgraph Phase3[Phase 3: Build Edges]
|
||||
L --> M[TKG Builder]
|
||||
M --> N[tkg_edges<br/>9 edge types]
|
||||
end
|
||||
|
||||
subgraph Phase4[Phase 4: Identity]
|
||||
G --> O[identity_matcher.py]
|
||||
L --> O
|
||||
P[Qdrant _seeds] --> O
|
||||
O --> Q[mark_tkg_suggested]
|
||||
Q --> L
|
||||
O --> R[confirm_identity.py]
|
||||
R --> L
|
||||
R --> G
|
||||
R --> P
|
||||
end
|
||||
|
||||
style Phase0 fill:#e1f5fe
|
||||
style Phase1 fill:#fff9c4
|
||||
style Phase2 fill:#e8f5e9
|
||||
style Phase3 fill:#f3e5f5
|
||||
style Phase4 fill:#fce4ec
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## skin_tone_trace Implementation
|
||||
|
||||
### Status: ✅ Implemented (2026-06-25)
|
||||
|
||||
See `src/core/processor/tkg.rs` lines 2579-2627 for implementation.
|
||||
|
||||
### Overview
|
||||
|
||||
Skin tone classification using Fitzpatrick scale from face skin color analysis.
|
||||
|
||||
### Fitzpatrick Classification
|
||||
|
||||
| Type | Skin H Range (HSV) | Description | Example |
|
||||
|------|-------------------|-------------|---------|
|
||||
| I | H < 10° | Very fair/pale | Northern European |
|
||||
| II | 10° ≤ H < 20° | Fair | European |
|
||||
| III | 20° ≤ H < 30° | Medium | Mediterranean |
|
||||
| IV | 30° ≤ H < 40° | Olive | Asian, Hispanic |
|
||||
| V | 40° ≤ H < 50° | Brown | Indian, African |
|
||||
| VI | H ≥ 50° | Dark brown | African |
|
||||
|
||||
### Implementation
|
||||
|
||||
**Rust Code Location:** `src/core/processor/tkg.rs`
|
||||
|
||||
```rust
|
||||
// Add to build_tkg()
|
||||
let n_skin = build_skin_tone_trace_nodes(pool, file_uuid).await?;
|
||||
|
||||
// Add to TkgResult
|
||||
pub skin_tone_trace_nodes: usize,
|
||||
|
||||
// New builder function
|
||||
async fn build_skin_tone_trace_nodes(
|
||||
pool: &PgPool,
|
||||
file_uuid: &str,
|
||||
) -> Result<usize> {
|
||||
let fd_table = t("face_detections");
|
||||
|
||||
// Step 1: Get avg skin H per trace
|
||||
let rows: Vec<(i64, f64)> = sqlx::query_as(&format!(
|
||||
"SELECT trace_id, AVG(skin_h) as avg_h
|
||||
FROM {}
|
||||
WHERE file_uuid = $1 AND trace_id IS NOT NULL AND skin_h IS NOT NULL
|
||||
GROUP BY trace_id",
|
||||
fd_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
// Step 2: Classify Fitzpatrick
|
||||
let nodes_table = t("tkg_nodes");
|
||||
let mut count = 0;
|
||||
|
||||
for (trace_id, avg_h) in &rows {
|
||||
let fitz_type = classify_fitzpatrick(*avg_h);
|
||||
let external_id = format!("skin_tone_{}", trace_id);
|
||||
let label = format!("Skin Tone Trace {}", trace_id);
|
||||
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO {} (node_type, external_id, file_uuid, label, properties)
|
||||
VALUES ('skin_tone_trace', $1, $2, $3, $4::jsonb)
|
||||
ON CONFLICT (file_uuid, node_type, external_id) DO UPDATE SET properties = EXCLUDED.properties",
|
||||
nodes_table
|
||||
))
|
||||
.bind(&external_id)
|
||||
.bind(file_uuid)
|
||||
.bind(&label)
|
||||
.bind(serde_json::json!({
|
||||
"trace_id": trace_id,
|
||||
"avg_skin_h": avg_h,
|
||||
"fitzpatrick_type": fitz_type,
|
||||
}))
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
count += 1;
|
||||
}
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
fn classify_fitzpatrick(h: f64) -> &'static str {
|
||||
if h < 10.0 { "I" }
|
||||
else if h < 20.0 { "II" }
|
||||
else if h < 30.0 { "III" }
|
||||
else if h < 40.0 { "IV" }
|
||||
else if h < 50.0 { "V" }
|
||||
else { "VI" }
|
||||
}
|
||||
```
|
||||
|
||||
### Dependencies
|
||||
|
||||
- `face_detections.trace_id` must be populated (from Phase 0)
|
||||
- `face.json` used for skin_h estimation (placeholder based on attributes)
|
||||
- `output_dir` path must be passed to builder
|
||||
|
||||
### Limitations
|
||||
|
||||
- Current skin_h estimation is placeholder (based on face attributes)
|
||||
- For accurate Fitzpatrick classification, face ROI color extraction needed
|
||||
- Face.json doesn't store skin_h directly (would need video frame analysis)
|
||||
|
||||
---
|
||||
|
||||
## SQL Query Examples
|
||||
|
||||
### Status Queries
|
||||
|
||||
```sql
|
||||
-- Get all face_track nodes for a file
|
||||
SELECT id, external_id, label, properties
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track' AND file_uuid = 'xxx'
|
||||
ORDER BY external_id;
|
||||
|
||||
-- Get pending faces (no identity suggestion)
|
||||
SELECT id, external_id, properties->>'trace_id' as trace_id
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND file_uuid = 'xxx'
|
||||
AND (properties->>'status' IS NULL OR properties->>'status' = 'pending');
|
||||
|
||||
-- Get suggested faces (Identity Agent suggested)
|
||||
SELECT id, external_id,
|
||||
properties->>'pending_identity_name' as name,
|
||||
properties->>'confidence' as confidence,
|
||||
properties->>'suggested_by' as source
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND file_uuid = 'xxx'
|
||||
AND properties->>'status' = 'suggested'
|
||||
ORDER BY (properties->>'confidence')::float DESC;
|
||||
|
||||
-- Get confirmed faces
|
||||
SELECT id, external_id,
|
||||
properties->>'identity_uuid' as identity_uuid,
|
||||
properties->>'identity_name' as name
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND file_uuid = 'xxx'
|
||||
AND properties->>'status' = 'confirmed';
|
||||
|
||||
-- Get stranger cluster members
|
||||
SELECT id, external_id, properties->>'stranger_id' as cluster
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND file_uuid = 'xxx'
|
||||
AND properties->>'status' = 'stranger'
|
||||
ORDER BY (properties->>'stranger_id')::int;
|
||||
```
|
||||
|
||||
### Identity Queries
|
||||
|
||||
```sql
|
||||
-- Find all traces bound to an identity
|
||||
SELECT id, external_id, properties->>'trace_id' as trace_id
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND properties->>'identity_uuid' = 'xxx-xxx';
|
||||
|
||||
-- Count identities per file
|
||||
SELECT properties->>'identity_uuid' as identity_uuid,
|
||||
COUNT(*) as trace_count
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND file_uuid = 'xxx'
|
||||
AND properties->>'status' = 'confirmed'
|
||||
GROUP BY properties->>'identity_uuid';
|
||||
```
|
||||
|
||||
### Statistics Queries
|
||||
|
||||
```sql
|
||||
-- Status distribution for a file
|
||||
SELECT properties->>'status' as status, COUNT(*) as count
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track' AND file_uuid = 'xxx'
|
||||
GROUP BY properties->>'status';
|
||||
|
||||
-- Confidence distribution
|
||||
SELECT
|
||||
CASE
|
||||
WHEN (properties->>'confidence')::float >= 0.9 THEN 'high'
|
||||
WHEN (properties->>'confidence')::float >= 0.7 THEN 'medium'
|
||||
ELSE 'low'
|
||||
END as confidence_level,
|
||||
COUNT(*) as count
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND file_uuid = 'xxx'
|
||||
AND properties->>'status' = 'suggested'
|
||||
GROUP BY confidence_level;
|
||||
|
||||
-- Top suggested identities
|
||||
SELECT properties->>'pending_identity_name' as name,
|
||||
COUNT(*) as trace_count,
|
||||
AVG((properties->>'confidence')::float) as avg_confidence
|
||||
FROM dev.tkg_nodes
|
||||
WHERE node_type = 'face_track'
|
||||
AND properties->>'status' = 'suggested'
|
||||
GROUP BY properties->>'pending_identity_name'
|
||||
ORDER BY trace_count DESC
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
### Cross-node Queries
|
||||
|
||||
```sql
|
||||
-- Speaker ↔ face_track edges
|
||||
SELECT
|
||||
s.external_id as speaker,
|
||||
f.external_id as face_track,
|
||||
e.properties->>'overlap_frames' as overlap
|
||||
FROM dev.tkg_edges e
|
||||
JOIN dev.tkg_nodes s ON e.source_node_id = s.id
|
||||
JOIN dev.tkg_nodes f ON e.target_node_id = f.id
|
||||
WHERE e.file_uuid = 'xxx'
|
||||
AND e.edge_type = 'SPEAKS_AS';
|
||||
|
||||
-- Objects co-occurrence
|
||||
SELECT
|
||||
o1.external_id as obj1,
|
||||
o2.external_id as obj2,
|
||||
e.properties->>'frame_count' as co_frames
|
||||
FROM dev.tkg_edges e
|
||||
JOIN dev.tkg_nodes o1 ON e.source_node_id = o1.id
|
||||
JOIN dev.tkg_nodes o2 ON e.target_node_id = o2.id
|
||||
WHERE e.file_uuid = 'xxx'
|
||||
AND e.edge_type = 'CO_OCCURS_WITH'
|
||||
ORDER BY (e.properties->>'frame_count')::int DESC;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Integration with Identity Agent
|
||||
|
||||
### Identity Agent Flow
|
||||
|
||||
```
|
||||
Identity Agent Pipeline:
|
||||
│
|
||||
├─ Round 1 (TH=0.55):
|
||||
│ Query _seeds (source='tmdb')
|
||||
│ Query _faces (file_uuid) → get trace representatives
|
||||
│ Multi-angle match → suggestions
|
||||
│ Mark TKG: status='suggested', confidence
|
||||
│
|
||||
├─ User Confirmation:
|
||||
│ Update TKG: status='confirmed'
|
||||
│ Update _faces: identity_uuid for all points
|
||||
│ Update PG face_detections: identity_id
|
||||
│ Add _seeds: source='propagation'
|
||||
│
|
||||
├─ Round 2 (TH=0.55):
|
||||
│ Use confirmed traces as seeds
|
||||
│ Match remaining pending traces
|
||||
│
|
||||
└─ Round 3+ (TH=0.50):
|
||||
Continue propagation
|
||||
Stranger clustering (TH=0.40)
|
||||
```
|
||||
|
||||
### TKG Node Status Transitions
|
||||
|
||||
```
|
||||
pending → suggested → confirmed → (final)
|
||||
↘ stranger ↘ (final)
|
||||
```
|
||||
|
||||
### Status Transition Triggers
|
||||
|
||||
| Transition | Trigger | Action |
|
||||
|------------|---------|--------|
|
||||
| `pending → suggested` | Identity Agent Round 1-3 | `mark_face_track_suggested()` |
|
||||
| `suggested → confirmed` | User confirmation API | `mark_face_track_confirmed()` |
|
||||
| `pending → stranger` | Stranger clustering | `mark_face_track_stranger()` |
|
||||
| `confirmed → pending` | Undo binding | `clear_face_track_status()` |
|
||||
|
||||
---
|
||||
|
||||
## Version History
|
||||
|
||||
| Version | Date | Changes |
|
||||
|---------|------|---------|
|
||||
| 1.0 | 2026-06-25 | Initial version with Phase 0-4 definition, node naming, flow diagram, query examples |
|
||||
@@ -163,4 +163,16 @@ status: Completed
|
||||
|
||||
- **Rust API integration**: Call Python scripts from stubbed Rust functions
|
||||
- **Production deployment**: Generate TMDb seeds for all identities
|
||||
- **Workflow documentation**: User guide for Identity Agent CLI usage
|
||||
- **Workflow documentation**: User guide for Identity Agent CLI usage
|
||||
|
||||
## Related Documents
|
||||
|
||||
- `docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md` — TKG formation phases, node/edge types, data flow diagram
|
||||
- `docs_v1.0/API_WORKSPACE/modules/15_tkg.md` — TKG API endpoints
|
||||
|
||||
## Version History
|
||||
|
||||
| Version | Date | Changes |
|
||||
|---------|------|--------|
|
||||
| 1.0 | 2026-06-25 | Initial architecture doc, all phases completed |
|
||||
| 1.1 | 2026-06-25 | Added reference to TKG Formation V1.0 |
|
||||
0
scripts/confirm_identity.py
Normal file → Executable file
0
scripts/confirm_identity.py
Normal file → Executable file
0
scripts/generate_seed_embeddings.py
Normal file → Executable file
0
scripts/generate_seed_embeddings.py
Normal file → Executable file
0
scripts/identity_matcher.py
Normal file → Executable file
0
scripts/identity_matcher.py
Normal file → Executable file
0
scripts/manual_seed.py
Normal file → Executable file
0
scripts/manual_seed.py
Normal file → Executable file
@@ -459,12 +459,13 @@ struct FaceDetectionRow {
|
||||
// ── Public API ────────────────────────────────────────────────────
|
||||
|
||||
pub struct TkgResult {
|
||||
pub face_track_nodes: usize,
|
||||
pub gaze_track_nodes: usize,
|
||||
pub lip_track_nodes: usize,
|
||||
pub text_region_nodes: usize,
|
||||
pub appearance_trace_nodes: usize,
|
||||
pub accessory_nodes: usize,
|
||||
pub face_track_nodes: usize,
|
||||
pub gaze_track_nodes: usize,
|
||||
pub lip_track_nodes: usize,
|
||||
pub text_region_nodes: usize,
|
||||
pub appearance_trace_nodes: usize,
|
||||
pub skin_tone_trace_nodes: usize,
|
||||
pub accessory_nodes: usize,
|
||||
pub object_nodes: usize,
|
||||
pub hand_nodes: usize,
|
||||
pub speaker_nodes: usize,
|
||||
@@ -507,9 +508,10 @@ pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Re
|
||||
let n_gaze = build_gaze_track_nodes(pool, file_uuid, &pose_data).await?;
|
||||
let n_lip = build_lip_track_nodes(pool, file_uuid, output_dir, &pose_data).await?;
|
||||
let n_text = build_text_region_nodes(pool, file_uuid).await?;
|
||||
let n_appearance =
|
||||
build_appearance_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?;
|
||||
let n_accessories = build_accessory_nodes(pool, file_uuid, output_dir).await?;
|
||||
let n_appearance =
|
||||
build_appearance_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?;
|
||||
let n_skin_tone = build_skin_tone_trace_nodes(pool, file_uuid, output_dir).await?;
|
||||
let n_accessories = build_accessory_nodes(pool, file_uuid, output_dir).await?;
|
||||
let n_objects = build_yolo_object_nodes(pool, file_uuid, output_dir).await?;
|
||||
let n_hands = build_hand_nodes(pool, file_uuid, output_dir).await?;
|
||||
let n_speakers = build_speaker_nodes(pool, file_uuid, output_dir).await?;
|
||||
@@ -523,14 +525,15 @@ let n_accessories = build_accessory_nodes(pool, file_uuid, output_dir).await?;
|
||||
let e_w = build_wears_edges(pool, file_uuid).await?;
|
||||
let e_ho = build_hand_object_edges(pool, file_uuid, output_dir).await?;
|
||||
|
||||
Ok(TkgResult {
|
||||
face_track_nodes: n_face,
|
||||
gaze_track_nodes: n_gaze,
|
||||
lip_track_nodes: n_lip,
|
||||
text_region_nodes: n_text,
|
||||
appearance_trace_nodes: n_appearance,
|
||||
accessory_nodes: n_accessories,
|
||||
object_nodes: n_objects,
|
||||
Ok(TkgResult {
|
||||
face_track_nodes: n_face,
|
||||
gaze_track_nodes: n_gaze,
|
||||
lip_track_nodes: n_lip,
|
||||
text_region_nodes: n_text,
|
||||
appearance_trace_nodes: n_appearance,
|
||||
skin_tone_trace_nodes: n_skin_tone,
|
||||
accessory_nodes: n_accessories,
|
||||
object_nodes: n_objects,
|
||||
hand_nodes: n_hands,
|
||||
speaker_nodes: n_speakers,
|
||||
co_occurrence_edges: e_co,
|
||||
@@ -2559,21 +2562,109 @@ fn compute_skin_h_from_face(face: &serde_json::Value) -> f64 {
|
||||
}
|
||||
|
||||
fn classify_fitzpatrick(h_mean: f64) -> &'static str {
|
||||
if h_mean < 5.0 {
|
||||
"Type I - Very Fair"
|
||||
} else if h_mean < 12.0 {
|
||||
"Type II - Fair"
|
||||
} else if h_mean < 18.0 {
|
||||
"Type III - Medium-Fair"
|
||||
} else if h_mean < 25.0 {
|
||||
"Type IV - Medium"
|
||||
} else if h_mean < 35.0 {
|
||||
"Type V - Dark"
|
||||
if h_mean < 10.0 {
|
||||
"I"
|
||||
} else if h_mean < 20.0 {
|
||||
"II"
|
||||
} else if h_mean < 30.0 {
|
||||
"III"
|
||||
} else if h_mean < 40.0 {
|
||||
"IV"
|
||||
} else if h_mean < 50.0 {
|
||||
"V"
|
||||
} else {
|
||||
"Type VI - Very Dark"
|
||||
"VI"
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_skin_tone_trace_nodes(
|
||||
pool: &PgPool,
|
||||
file_uuid: &str,
|
||||
output_dir: &str,
|
||||
) -> Result<usize> {
|
||||
let nodes_table = t("tkg_nodes");
|
||||
let fd_table = t("face_detections");
|
||||
let mut count = 0;
|
||||
|
||||
let rows: Vec<(i64, i64)> = sqlx::query_as(&format!(
|
||||
"SELECT trace_id, COUNT(*) \
|
||||
FROM {} \
|
||||
WHERE file_uuid = $1 AND trace_id IS NOT NULL \
|
||||
GROUP BY trace_id",
|
||||
fd_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
if rows.is_empty() {
|
||||
tracing::info!("[TKG] No traced faces for skin_tone_trace nodes");
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let path = Path::new(output_dir).join(format!("{}.face.json", file_uuid));
|
||||
let face_json = if path.exists() {
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.with_context(|| format!("Failed to read face.json: {}", path.display()))?;
|
||||
serde_json::from_str::<serde_json::Value>(&content)?
|
||||
} else {
|
||||
tracing::warn!("[TKG] No face.json for skin_tone computation");
|
||||
serde_json::Value::Null
|
||||
};
|
||||
|
||||
for (trace_id, frame_count) in &rows {
|
||||
let avg_skin_h = if !face_json.is_null() {
|
||||
let mut skin_h_values = Vec::new();
|
||||
if let Some(frames) = face_json.get("frames").and_then(|v| v.as_array()) {
|
||||
for frame_entry in frames {
|
||||
if let Some(faces) = frame_entry.get("faces").and_then(|v| v.as_array()) {
|
||||
for face in faces {
|
||||
let face_trace_id = face.get("trace_id").and_then(|v| v.as_i64());
|
||||
if face_trace_id == Some(*trace_id) {
|
||||
skin_h_values.push(compute_skin_h_from_face(face));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if skin_h_values.is_empty() {
|
||||
20.0
|
||||
} else {
|
||||
skin_h_values.iter().sum::<f64>() / skin_h_values.len() as f64
|
||||
}
|
||||
} else {
|
||||
20.0
|
||||
};
|
||||
|
||||
let fitz_type = classify_fitzpatrick(avg_skin_h);
|
||||
let external_id = format!("skin_tone_{}", trace_id);
|
||||
let label = format!("Skin Tone Trace {}", trace_id);
|
||||
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO {} (node_type, external_id, file_uuid, label, properties) \
|
||||
VALUES ('skin_tone_trace', $1, $2, $3, $4::jsonb) \
|
||||
ON CONFLICT (file_uuid, node_type, external_id) DO UPDATE SET properties = EXCLUDED.properties",
|
||||
nodes_table
|
||||
))
|
||||
.bind(&external_id)
|
||||
.bind(file_uuid)
|
||||
.bind(&label)
|
||||
.bind(serde_json::json!({
|
||||
"trace_id": trace_id,
|
||||
"avg_skin_h": avg_skin_h,
|
||||
"fitzpatrick_type": fitz_type,
|
||||
"frame_count": frame_count,
|
||||
}))
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
count += 1;
|
||||
}
|
||||
|
||||
tracing::info!("[TKG] Built {} skin_tone_trace nodes", count);
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
// ── Accessory Nodes ──────────────────────────────────────────────
|
||||
|
||||
async fn build_accessory_nodes(pool: &PgPool, file_uuid: &str, output_dir: &str) -> Result<usize> {
|
||||
@@ -3124,10 +3215,11 @@ mod tests {
|
||||
let r = TkgResult {
|
||||
face_track_nodes: 5,
|
||||
gaze_track_nodes: 5,
|
||||
lip_track_nodes: 4,
|
||||
text_region_nodes: 20,
|
||||
appearance_trace_nodes: 3,
|
||||
accessory_nodes: 0,
|
||||
lip_track_nodes: 4,
|
||||
text_region_nodes: 20,
|
||||
appearance_trace_nodes: 3,
|
||||
skin_tone_trace_nodes: 5,
|
||||
accessory_nodes: 0,
|
||||
object_nodes: 10,
|
||||
hand_nodes: 0,
|
||||
speaker_nodes: 3,
|
||||
|
||||
Reference in New Issue
Block a user