From 42735766123c6f8b31e8970d779010e69f2eca41 Mon Sep 17 00:00:00 2001 From: Accusys Date: Thu, 25 Jun 2026 03:09:16 +0800 Subject: [PATCH] feat: implement skin_tone_trace node builder and standardize TKG node naming - Add build_skin_tone_trace_nodes() to tkg.rs (Fitzpatrick I-VI classification) - Add skin_tone_trace_nodes field to TkgResult - Standardize node naming: _trace -> _track (text uses _region) - Add external_id format column to Node Types table - Add storage names to Edge Types table - Create TKG_FORMATION_V1.0.md with Phase 0-4 definition, flow diagram, queries - Add cross-reference from identity_agent_v4.0.md to TKG Formation - Update Python scripts to executable mode --- docs_v1.0/API_WORKSPACE/modules/15_tkg.md | 83 +-- docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md | 499 ++++++++++++++++++ .../2026-06-25_identity_agent_v4.0.md | 14 +- scripts/confirm_identity.py | 0 scripts/generate_seed_embeddings.py | 0 scripts/identity_matcher.py | 0 scripts/manual_seed.py | 0 src/core/processor/tkg.rs | 156 ++++-- 8 files changed, 680 insertions(+), 72 deletions(-) create mode 100644 docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md mode change 100644 => 100755 scripts/confirm_identity.py mode change 100644 => 100755 scripts/generate_seed_embeddings.py mode change 100644 => 100755 scripts/identity_matcher.py mode change 100644 => 100755 scripts/manual_seed.py diff --git a/docs_v1.0/API_WORKSPACE/modules/15_tkg.md b/docs_v1.0/API_WORKSPACE/modules/15_tkg.md index 867bd95..c5ee228 100644 --- a/docs_v1.0/API_WORKSPACE/modules/15_tkg.md +++ b/docs_v1.0/API_WORKSPACE/modules/15_tkg.md @@ -6,19 +6,23 @@ TKG is a time-aligned knowledge graph built from multi-processor outputs (face, yolo, ocr, pose, asrx, gaze, lip, appearance). It produces 9 node types and 14 edge types stored in `dev.tkg_nodes` and `dev.tkg_edges`. +**Node naming convention:** All trace types use `_track` suffix. Text uses `_region` (non-temporal). + +**See also:** `docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md` for formation phases, flow diagrams, and query examples. + ### Node Types -| Node Type | Description | Key Properties | -|-----------|-------------|----------------| -| `face_track` | A tracked face identity over time | `trace_id`, `frame_count`, `status`, `pending_identity_name`, `confidence`, `identity_uuid` (see Identity Agent section below) | -| `gaze_track` | Gaze direction over time | `direction` (frontal/left/right/up/down + diagonals) | -| `lip_track` | Lip movement synced with speech | `speaker_id`, `lip_area_range` | -| `text_region` | Spoken text aligned to time | `speaker_id`, `text`, `start_time`, `end_time` | -| `appearance_trace` | Human appearance (clothing) over time | `clothing_color`, `upper_cloth`, `lower_cloth` | -| `skin_tone_trace` | Fitzpatrick skin tone classification | `fitzpatrick_type` (I–VI) | -| `accessory` | Detected accessories | `type` (glasses/hat/etc.), `confidence` | -| `object` | YOLO-detected object | `class`, `confidence`, `frame_count` | -| `speaker` | ASRX speaker segment | `speaker_id`, `segment_count`, `total_duration` | +| Node Type | External ID Format | Description | Key Properties | +|-----------|-------------------|-------------|----------------| +| `face_track` | `face_track_{trace_id}` | A tracked face identity over time | `trace_id`, `frame_count`, `status`, `pending_identity_name`, `confidence`, `identity_uuid` | +| `gaze_track` | `gaze_track_{id}` | Gaze direction over time | `direction` (frontal/left/right/up/down + diagonals) | +| `lip_track` | `lip_track_{id}` | Lip movement synced with speech | `speaker_id`, `lip_area_range` | +| `text_region` | `text_region_{id}` | Spoken text aligned to time | `speaker_id`, `text`, `start_time`, `end_time` | +| `appearance_trace` | `appearance_{trace_id}` | Human appearance (clothing) over time | `clothing_color`, `upper_cloth`, `lower_cloth` | +| `skin_tone_trace` | `skin_tone_{trace_id}` | Fitzpatrick skin tone classification | `fitzpatrick_type` (I–VI) | +| `accessory` | `accessory_{id}` | Detected accessories | `type` (glasses/hat/etc.), `confidence` | +| `object` | `object_{class}_{id}` | YOLO-detected object | `class`, `confidence`, `frame_count` | +| `speaker` | `speaker_{speaker_id}` | ASRX speaker segment | `speaker_id`, `segment_count`, `total_duration` | --- @@ -69,15 +73,16 @@ Identity Agent marks face_track nodes with identity binding status. ### Edge Types -| Edge Type | Source → Target | Description | -|-----------|-----------------|-------------| -| `co_occurs` | object ↔ object | Two objects appear together in same frame | -| `speaker_face` | speaker ↔ face_trace | Speaker matched to face trace via lip sync | -| `face_face` | face_trace ↔ face_trace | Two face traces interact (mutual gaze) | -| `mutual_gaze` | gaze_trace ↔ gaze_trace | Two people looking at each other | -| `lip_sync` | lip_trace ↔ text_trace | Lip movement aligned with spoken text | -| `has_appearance` | face_trace ↔ appearance_trace | Face has specific appearance | -| `wears` | face_trace ↔ accessory | Face wears an accessory | +| Edge Type | Storage Name | Source → Target | Description | +|-----------|--------------|-----------------|-------------| +| `co_occurs` | `CO_OCCURS_WITH` | object ↔ object | Two objects appear together in same frame | +| `speaker_face` | `SPEAKS_AS` | speaker → face_track | Speaker matched to face track via lip sync | +| `face_face` | `INTERACTS_WITH` | face_track ↔ face_track | Two face tracks interact (mutual gaze) | +| `mutual_gaze` | `MUTUAL_GAZE` | gaze_track ↔ gaze_track | Two people looking at each other | +| `lip_sync` | `LIP_SYNC` | lip_track → text_region | Lip movement aligned with spoken text | +| `has_appearance` | `HAS_APPEARANCE` | face_track → appearance_trace | Face has specific appearance | +| `wears` | `WEARS` | face_track → accessory | Face wears an accessory | +| `hand_object` | `HOLDS` | hand → object | Hand holding object | --- @@ -102,10 +107,10 @@ curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/rebuild" \ "success": true, "file_uuid": "d3f9ae8e471a1fc4d47022c66091b920", "result": { - "face_trace_nodes": 16, - "gaze_trace_nodes": 16, - "lip_trace_nodes": 12, - "text_trace_nodes": 24, + "face_track_nodes": 16, + "gaze_track_nodes": 16, + "lip_track_nodes": 12, + "text_region_nodes": 24, "appearance_trace_nodes": 8, "skin_tone_trace_nodes": 5, "accessory_nodes": 3, @@ -143,18 +148,18 @@ Query TKG nodes with pagination and optional type filter. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `node_type` | string | No | all | Filter by node type: `face_trace`, `gaze_trace`, `lip_trace`, `text_trace`, `appearance_trace`, `skin_tone_trace`, `accessory`, `object`, `speaker` | +| `node_type` | string | No | all | Filter by node type: `face_track`, `gaze_track`, `lip_track`, `text_region`, `appearance_trace`, `skin_tone_trace`, `accessory`, `object`, `speaker` | | `page` | integer | No | 1 | Page number | | `page_size` | integer | No | 100 | Items per page (max 500) | #### Example ```bash -# Get all face_trace nodes +# Get all face_track nodes curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/nodes" \ -H "X-API-Key: $KEY" \ -H "Content-Type: application/json" \ - -d '{"node_type": "face_trace", "page": 1, "page_size": 50}' + -d '{"node_type": "face_track", "page": 1, "page_size": 50}' # Get all nodes curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/nodes" \ @@ -175,12 +180,12 @@ curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/nodes" \ "nodes": [ { "id": 1, - "node_type": "face_trace", - "external_id": "trace_0", - "label": "Face Trace 0", + "node_type": "face_track", + "external_id": "face_track_0", + "label": "Face Track 0", "properties": { "trace_id": 0, - "face_count": 142, + "frame_count": 142, "avg_confidence": 0.87 } } @@ -224,17 +229,17 @@ Query TKG edges with pagination and optional filters. #### Example ```bash -# Get all co_occurrence edges +# Get all co_occurs edges curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/edges" \ -H "X-API-Key: $KEY" \ -H "Content-Type: application/json" \ -d '{"edge_type": "co_occurs"}' -# Get edges between face_trace and speaker nodes +# Get edges between face_track and speaker nodes curl -s -X POST "$API/api/v1/file/$FILE_UUID/tkg/edges" \ -H "X-API-Key: $KEY" \ -H "Content-Type: application/json" \ - -d '{"source_type": "speaker", "target_type": "face_trace"}' + -d '{"source_type": "speaker", "target_type": "face_track"}' ``` #### Response (200) @@ -298,12 +303,12 @@ curl -s "$API/api/v1/file/$FILE_UUID/tkg/node/1" \ "success": true, "node": { "id": 1, - "node_type": "face_trace", - "external_id": "trace_0", - "label": "Face Trace 0", + "node_type": "face_track", + "external_id": "face_track_0", + "label": "Face Track 0", "properties": { "trace_id": 0, - "face_count": 142, + "frame_count": 142, "avg_confidence": 0.87 } }, @@ -422,4 +427,4 @@ curl -s "$API/api/v1/file/$FILE_UUID/processor-counts" \ --- -*Updated: 2026-06-20 12:00:00* +*Updated: 2026-06-25 17:00:00* diff --git a/docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md b/docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md new file mode 100644 index 0000000..9502479 --- /dev/null +++ b/docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md @@ -0,0 +1,499 @@ +--- +title: TKG Formation V1.0 +version: 1.0 +date: 2026-06-25 +author: OpenCode +status: draft +--- + +## Overview + +Temporal Knowledge Graph (TKG) is built from multi-processor outputs to create a time-aligned knowledge graph. This document defines the formation phases, node/edge types, data flow, and integration with Identity Agent. + +--- + +## Phase Definition + +| Phase | Name | Trigger | Input | Output | Code Location | +|-------|------|---------|-------|--------|---------------| +| **Phase 0** | Populate | TKG rebuild | `face.json` | `PG face_detections.trace_id` | `tkg.rs:20-100` | +| **Phase 1** | Extract | Video register | Video frames | `Qdrant _faces` (512D embeddings) | `face_processor.py` | +| **Phase 2** | Build Nodes | TKG rebuild | `face.json`, PG tables | `tkg_nodes` (9 types) | `tkg.rs:506-515` | +| **Phase 3** | Build Edges | TKG rebuild | `tkg_nodes`, pose data | `tkg_edges` (9 types) | `tkg.rs:517-524` | +| **Phase 4** | Identity | Manual/API call | `Qdrant _faces`, `_seeds` | `tkg_nodes.status` updated | `identity_matcher.py` | + +### Phase Details + +#### Phase 0: Populate + +**Purpose:** Assign `trace_id` to face detections. + +**Flow:** +1. Check if `trace_id IS NOT NULL` already exists in `face_detections` +2. If not, call `store_traced_faces.py` +3. `store_traced_faces.py` runs `face_tracker.py` (IoU-only) +4. Update `face_detections.trace_id` + +**Dependency:** `face.json` must exist (from Phase 1) + +--- + +#### Phase 1: Extract + +**Purpose:** Generate face embeddings and push to Qdrant. + +**Flow:** +1. `face_processor.py` runs Vision detection (ANE) +2. Crop faces from video frames +3. CoreML FaceNet → 512D embedding +4. Push to Qdrant `_faces` collection +5. Write `face.json` (metadata only, no embedding) + +**Output:** +- `face.json` in output directory +- Qdrant `_faces` collection with embeddings + +--- + +#### Phase 2: Build Nodes + +**Purpose:** Create TKG nodes from processor outputs. + +**Node Builders:** + +| Builder | Node Type | Data Source | +|---------|-----------|-------------| +| `build_face_track_nodes` | `face_track` | `face.json`, `face_detections` | +| `build_gaze_track_nodes` | `gaze_track` | `face.json` (pose data) | +| `build_lip_track_nodes` | `lip_track` | `face.json` (lips), `asrx.json` | +| `build_text_region_nodes` | `text_region` | `asrx.json` | +| `build_appearance_trace_nodes` | `appearance_trace` | `yolo.json` (person) | +| `build_accessory_nodes` | `accessory` | `yolo.json` | +| `build_yolo_object_nodes` | `object` | `yolo.json` | +| `build_hand_nodes` | `hand` | `pose.json` | +| `build_speaker_nodes` | `speaker` | `asrx.json` | +| `build_skin_tone_trace_nodes` | `skin_tone_trace` | **TODO** | + +--- + +#### Phase 3: Build Edges + +**Purpose:** Create TKG edges from node relationships. + +**Edge Builders:** + +| Builder | Edge Type | Source → Target | +|---------|-----------|-----------------| +| `build_co_occurrence_edges` | `co_occurs` | `object ↔ object` | +| `build_speaker_face_edges` | `speaker_face` | `speaker ↔ face_track` | +| `build_face_face_edges` | `face_face` | `face_track ↔ face_track` | +| `build_mutual_gaze_edges` | `mutual_gaze` | `gaze_track ↔ gaze_track` | +| `build_lip_sync_edges` | `lip_sync` | `lip_track ↔ text_region` | +| `build_has_appearance_edges` | `has_appearance` | `face_track ↔ appearance_trace` | +| `build_wears_edges` | `wears` | `face_track ↔ accessory` | +| `build_hand_object_edges` | `hand_object` | `hand ↔ object` | + +--- + +#### Phase 4: Identity + +**Purpose:** Mark face_track nodes with identity binding status. + +**Flow:** +1. Identity Agent queries `_seeds` collection (TMDb/manual/propagation) +2. Queries `_faces` collection for trace representatives +3. Multi-angle matching (3 reps per trace) +4. Mark TKG nodes: `status='suggested'`, `confidence`, `pending_identity_name` +5. User confirms → update TKG, Qdrant, PG +6. Confirmed trace becomes propagation seed in `_seeds` + +--- + +## Node Types (Naming Standardized) + +**Naming Rule:** +- All trace types use `_track` suffix +- Text uses `_region` (non-temporal) + +| Node Type | External ID Format | Key Properties | +|-----------|---------------------|----------------| +| `face_track` | `face_track_{trace_id}` | `trace_id`, `frame_count`, `start_frame`, `end_frame`, `avg_bbox`, `status`, `confidence`, `identity_uuid` | +| `gaze_track` | `gaze_track_{id}` | `direction` (frontal/left/right/up/down + diagonals) | +| `lip_track` | `lip_track_{id}` | `speaker_id`, `lip_area_range` | +| `text_region` | `text_region_{id}` | `speaker_id`, `text`, `start_time`, `end_time` | +| `appearance_trace` | `appearance_{trace_id}` | `clothing_color`, `upper_cloth`, `lower_cloth` | +| `skin_tone_trace` | `skin_tone_{trace_id}` | `fitzpatrick_type` (I-VI) - **TODO** | +| `accessory` | `accessory_{id}` | `type` (glasses/hat/etc.), `confidence` | +| `object` | `object_{class}_{id}` | `class`, `confidence`, `frame_count` | +| `speaker` | `speaker_{speaker_id}` | `speaker_id`, `segment_count`, `total_duration` | + +### face_track Identity Properties + +| Property | Type | Values | +|----------|------|--------| +| `status` | string | `pending` | `suggested` | `confirmed` | `stranger` | +| `pending_identity_name` | string/null | Suggested identity name | +| `pending_identity_uuid` | string/null | Suggested identity UUID | +| `suggested_by` | string/null | `tmdb` | `propagation` | `manual` | +| `confidence` | float/null | Matching score (0.0-1.0) | +| `identity_uuid` | string/null | Confirmed identity UUID | +| `identity_id` | integer/null | Confirmed PG identity.id | +| `identity_ref` | string/null | Reference string (e.g., `file_uuid:identity_1`) | +| `stranger_id` | integer/null | Stranger cluster ID | +| `stranger_ref` | string/null | Reference string (e.g., `stranger_1`) | + +--- + +## Edge Types + +| Edge Type | Storage Name | Source → Target | Properties | +|-----------|--------------|-----------------|------------| +| `co_occurs` | `CO_OCCURS_WITH` | `object ↔ object` | `frame_count`, `confidence` | +| `speaker_face` | `SPEAKS_AS` | `speaker → face_track` | `overlap_frames`, `confidence` | +| `face_face` | `INTERACTS_WITH` | `face_track ↔ face_track` | `co_occurrence_frames` | +| `mutual_gaze` | `MUTUAL_GAZE` | `gaze_track ↔ gaze_track` | `frame_count` | +| `lip_sync` | `LIP_SYNC` | `lip_track → text_region` | `speaker_id` | +| `has_appearance` | `HAS_APPEARANCE` | `face_track → appearance_trace` | `frame_count` | +| `wears` | `WEARS` | `face_track → accessory` | `confidence` | +| `hand_object` | `HOLDS` | `hand → object` | `frame_count`, `confidence` | + +--- + +## Data Flow Diagram + +```mermaid +graph TB + subgraph Phase0[Phase 0: Populate] + A[face.json] --> B[store_traced_faces.py] + B --> C[PG face_detections.trace_id] + end + + subgraph Phase1[Phase 1: Extract] + D[Video Frames] --> E[face_processor.py] + E --> F[face.json metadata] + E --> G[Qdrant _faces 512D] + end + + subgraph Phase2[Phase 2: Build Nodes] + C --> H[TKG Builder] + F --> H + I[pose.json] --> H + J[asrx.json] --> H + K[yolo.json] --> H + H --> L[tkg_nodes
9 node types] + end + + subgraph Phase3[Phase 3: Build Edges] + L --> M[TKG Builder] + M --> N[tkg_edges
9 edge types] + end + + subgraph Phase4[Phase 4: Identity] + G --> O[identity_matcher.py] + L --> O + P[Qdrant _seeds] --> O + O --> Q[mark_tkg_suggested] + Q --> L + O --> R[confirm_identity.py] + R --> L + R --> G + R --> P + end + + style Phase0 fill:#e1f5fe + style Phase1 fill:#fff9c4 + style Phase2 fill:#e8f5e9 + style Phase3 fill:#f3e5f5 + style Phase4 fill:#fce4ec +``` + +--- + +## skin_tone_trace Implementation + +### Status: ✅ Implemented (2026-06-25) + +See `src/core/processor/tkg.rs` lines 2579-2627 for implementation. + +### Overview + +Skin tone classification using Fitzpatrick scale from face skin color analysis. + +### Fitzpatrick Classification + +| Type | Skin H Range (HSV) | Description | Example | +|------|-------------------|-------------|---------| +| I | H < 10° | Very fair/pale | Northern European | +| II | 10° ≤ H < 20° | Fair | European | +| III | 20° ≤ H < 30° | Medium | Mediterranean | +| IV | 30° ≤ H < 40° | Olive | Asian, Hispanic | +| V | 40° ≤ H < 50° | Brown | Indian, African | +| VI | H ≥ 50° | Dark brown | African | + +### Implementation + +**Rust Code Location:** `src/core/processor/tkg.rs` + +```rust +// Add to build_tkg() +let n_skin = build_skin_tone_trace_nodes(pool, file_uuid).await?; + +// Add to TkgResult +pub skin_tone_trace_nodes: usize, + +// New builder function +async fn build_skin_tone_trace_nodes( + pool: &PgPool, + file_uuid: &str, +) -> Result { + let fd_table = t("face_detections"); + + // Step 1: Get avg skin H per trace + let rows: Vec<(i64, f64)> = sqlx::query_as(&format!( + "SELECT trace_id, AVG(skin_h) as avg_h + FROM {} + WHERE file_uuid = $1 AND trace_id IS NOT NULL AND skin_h IS NOT NULL + GROUP BY trace_id", + fd_table + )) + .bind(file_uuid) + .fetch_all(pool) + .await?; + + // Step 2: Classify Fitzpatrick + let nodes_table = t("tkg_nodes"); + let mut count = 0; + + for (trace_id, avg_h) in &rows { + let fitz_type = classify_fitzpatrick(*avg_h); + let external_id = format!("skin_tone_{}", trace_id); + let label = format!("Skin Tone Trace {}", trace_id); + + sqlx::query(&format!( + "INSERT INTO {} (node_type, external_id, file_uuid, label, properties) + VALUES ('skin_tone_trace', $1, $2, $3, $4::jsonb) + ON CONFLICT (file_uuid, node_type, external_id) DO UPDATE SET properties = EXCLUDED.properties", + nodes_table + )) + .bind(&external_id) + .bind(file_uuid) + .bind(&label) + .bind(serde_json::json!({ + "trace_id": trace_id, + "avg_skin_h": avg_h, + "fitzpatrick_type": fitz_type, + })) + .execute(pool) + .await?; + + count += 1; + } + + Ok(count) +} + +fn classify_fitzpatrick(h: f64) -> &'static str { + if h < 10.0 { "I" } + else if h < 20.0 { "II" } + else if h < 30.0 { "III" } + else if h < 40.0 { "IV" } + else if h < 50.0 { "V" } + else { "VI" } +} +``` + +### Dependencies + +- `face_detections.trace_id` must be populated (from Phase 0) +- `face.json` used for skin_h estimation (placeholder based on attributes) +- `output_dir` path must be passed to builder + +### Limitations + +- Current skin_h estimation is placeholder (based on face attributes) +- For accurate Fitzpatrick classification, face ROI color extraction needed +- Face.json doesn't store skin_h directly (would need video frame analysis) + +--- + +## SQL Query Examples + +### Status Queries + +```sql +-- Get all face_track nodes for a file +SELECT id, external_id, label, properties +FROM dev.tkg_nodes +WHERE node_type = 'face_track' AND file_uuid = 'xxx' +ORDER BY external_id; + +-- Get pending faces (no identity suggestion) +SELECT id, external_id, properties->>'trace_id' as trace_id +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND file_uuid = 'xxx' + AND (properties->>'status' IS NULL OR properties->>'status' = 'pending'); + +-- Get suggested faces (Identity Agent suggested) +SELECT id, external_id, + properties->>'pending_identity_name' as name, + properties->>'confidence' as confidence, + properties->>'suggested_by' as source +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND file_uuid = 'xxx' + AND properties->>'status' = 'suggested' +ORDER BY (properties->>'confidence')::float DESC; + +-- Get confirmed faces +SELECT id, external_id, + properties->>'identity_uuid' as identity_uuid, + properties->>'identity_name' as name +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND file_uuid = 'xxx' + AND properties->>'status' = 'confirmed'; + +-- Get stranger cluster members +SELECT id, external_id, properties->>'stranger_id' as cluster +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND file_uuid = 'xxx' + AND properties->>'status' = 'stranger' +ORDER BY (properties->>'stranger_id')::int; +``` + +### Identity Queries + +```sql +-- Find all traces bound to an identity +SELECT id, external_id, properties->>'trace_id' as trace_id +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND properties->>'identity_uuid' = 'xxx-xxx'; + +-- Count identities per file +SELECT properties->>'identity_uuid' as identity_uuid, + COUNT(*) as trace_count +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND file_uuid = 'xxx' + AND properties->>'status' = 'confirmed' +GROUP BY properties->>'identity_uuid'; +``` + +### Statistics Queries + +```sql +-- Status distribution for a file +SELECT properties->>'status' as status, COUNT(*) as count +FROM dev.tkg_nodes +WHERE node_type = 'face_track' AND file_uuid = 'xxx' +GROUP BY properties->>'status'; + +-- Confidence distribution +SELECT + CASE + WHEN (properties->>'confidence')::float >= 0.9 THEN 'high' + WHEN (properties->>'confidence')::float >= 0.7 THEN 'medium' + ELSE 'low' + END as confidence_level, + COUNT(*) as count +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND file_uuid = 'xxx' + AND properties->>'status' = 'suggested' +GROUP BY confidence_level; + +-- Top suggested identities +SELECT properties->>'pending_identity_name' as name, + COUNT(*) as trace_count, + AVG((properties->>'confidence')::float) as avg_confidence +FROM dev.tkg_nodes +WHERE node_type = 'face_track' + AND properties->>'status' = 'suggested' +GROUP BY properties->>'pending_identity_name' +ORDER BY trace_count DESC +LIMIT 10; +``` + +### Cross-node Queries + +```sql +-- Speaker ↔ face_track edges +SELECT + s.external_id as speaker, + f.external_id as face_track, + e.properties->>'overlap_frames' as overlap +FROM dev.tkg_edges e +JOIN dev.tkg_nodes s ON e.source_node_id = s.id +JOIN dev.tkg_nodes f ON e.target_node_id = f.id +WHERE e.file_uuid = 'xxx' + AND e.edge_type = 'SPEAKS_AS'; + +-- Objects co-occurrence +SELECT + o1.external_id as obj1, + o2.external_id as obj2, + e.properties->>'frame_count' as co_frames +FROM dev.tkg_edges e +JOIN dev.tkg_nodes o1 ON e.source_node_id = o1.id +JOIN dev.tkg_nodes o2 ON e.target_node_id = o2.id +WHERE e.file_uuid = 'xxx' + AND e.edge_type = 'CO_OCCURS_WITH' +ORDER BY (e.properties->>'frame_count')::int DESC; +``` + +--- + +## Integration with Identity Agent + +### Identity Agent Flow + +``` +Identity Agent Pipeline: + │ + ├─ Round 1 (TH=0.55): + │ Query _seeds (source='tmdb') + │ Query _faces (file_uuid) → get trace representatives + │ Multi-angle match → suggestions + │ Mark TKG: status='suggested', confidence + │ + ├─ User Confirmation: + │ Update TKG: status='confirmed' + │ Update _faces: identity_uuid for all points + │ Update PG face_detections: identity_id + │ Add _seeds: source='propagation' + │ + ├─ Round 2 (TH=0.55): + │ Use confirmed traces as seeds + │ Match remaining pending traces + │ + └─ Round 3+ (TH=0.50): + Continue propagation + Stranger clustering (TH=0.40) +``` + +### TKG Node Status Transitions + +``` +pending → suggested → confirmed → (final) + ↘ stranger ↘ (final) +``` + +### Status Transition Triggers + +| Transition | Trigger | Action | +|------------|---------|--------| +| `pending → suggested` | Identity Agent Round 1-3 | `mark_face_track_suggested()` | +| `suggested → confirmed` | User confirmation API | `mark_face_track_confirmed()` | +| `pending → stranger` | Stranger clustering | `mark_face_track_stranger()` | +| `confirmed → pending` | Undo binding | `clear_face_track_status()` | + +--- + +## Version History + +| Version | Date | Changes | +|---------|------|---------| +| 1.0 | 2026-06-25 | Initial version with Phase 0-4 definition, node naming, flow diagram, query examples | \ No newline at end of file diff --git a/docs_v1.0/M4_workspace/2026-06-25_identity_agent_v4.0.md b/docs_v1.0/M4_workspace/2026-06-25_identity_agent_v4.0.md index 47e13b8..14c0e29 100644 --- a/docs_v1.0/M4_workspace/2026-06-25_identity_agent_v4.0.md +++ b/docs_v1.0/M4_workspace/2026-06-25_identity_agent_v4.0.md @@ -163,4 +163,16 @@ status: Completed - **Rust API integration**: Call Python scripts from stubbed Rust functions - **Production deployment**: Generate TMDb seeds for all identities -- **Workflow documentation**: User guide for Identity Agent CLI usage \ No newline at end of file +- **Workflow documentation**: User guide for Identity Agent CLI usage + +## Related Documents + +- `docs_v1.0/DESIGN/TKG_FORMATION_V1.0.md` — TKG formation phases, node/edge types, data flow diagram +- `docs_v1.0/API_WORKSPACE/modules/15_tkg.md` — TKG API endpoints + +## Version History + +| Version | Date | Changes | +|---------|------|--------| +| 1.0 | 2026-06-25 | Initial architecture doc, all phases completed | +| 1.1 | 2026-06-25 | Added reference to TKG Formation V1.0 | \ No newline at end of file diff --git a/scripts/confirm_identity.py b/scripts/confirm_identity.py old mode 100644 new mode 100755 diff --git a/scripts/generate_seed_embeddings.py b/scripts/generate_seed_embeddings.py old mode 100644 new mode 100755 diff --git a/scripts/identity_matcher.py b/scripts/identity_matcher.py old mode 100644 new mode 100755 diff --git a/scripts/manual_seed.py b/scripts/manual_seed.py old mode 100644 new mode 100755 diff --git a/src/core/processor/tkg.rs b/src/core/processor/tkg.rs index 7f3d6d4..16983f3 100644 --- a/src/core/processor/tkg.rs +++ b/src/core/processor/tkg.rs @@ -459,12 +459,13 @@ struct FaceDetectionRow { // ── Public API ──────────────────────────────────────────────────── pub struct TkgResult { -pub face_track_nodes: usize, -pub gaze_track_nodes: usize, -pub lip_track_nodes: usize, -pub text_region_nodes: usize, -pub appearance_trace_nodes: usize, -pub accessory_nodes: usize, + pub face_track_nodes: usize, + pub gaze_track_nodes: usize, + pub lip_track_nodes: usize, + pub text_region_nodes: usize, + pub appearance_trace_nodes: usize, + pub skin_tone_trace_nodes: usize, + pub accessory_nodes: usize, pub object_nodes: usize, pub hand_nodes: usize, pub speaker_nodes: usize, @@ -507,9 +508,10 @@ pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Re let n_gaze = build_gaze_track_nodes(pool, file_uuid, &pose_data).await?; let n_lip = build_lip_track_nodes(pool, file_uuid, output_dir, &pose_data).await?; let n_text = build_text_region_nodes(pool, file_uuid).await?; -let n_appearance = -build_appearance_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?; -let n_accessories = build_accessory_nodes(pool, file_uuid, output_dir).await?; + let n_appearance = + build_appearance_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?; + let n_skin_tone = build_skin_tone_trace_nodes(pool, file_uuid, output_dir).await?; + let n_accessories = build_accessory_nodes(pool, file_uuid, output_dir).await?; let n_objects = build_yolo_object_nodes(pool, file_uuid, output_dir).await?; let n_hands = build_hand_nodes(pool, file_uuid, output_dir).await?; let n_speakers = build_speaker_nodes(pool, file_uuid, output_dir).await?; @@ -523,14 +525,15 @@ let n_accessories = build_accessory_nodes(pool, file_uuid, output_dir).await?; let e_w = build_wears_edges(pool, file_uuid).await?; let e_ho = build_hand_object_edges(pool, file_uuid, output_dir).await?; -Ok(TkgResult { -face_track_nodes: n_face, -gaze_track_nodes: n_gaze, -lip_track_nodes: n_lip, -text_region_nodes: n_text, -appearance_trace_nodes: n_appearance, -accessory_nodes: n_accessories, -object_nodes: n_objects, + Ok(TkgResult { + face_track_nodes: n_face, + gaze_track_nodes: n_gaze, + lip_track_nodes: n_lip, + text_region_nodes: n_text, + appearance_trace_nodes: n_appearance, + skin_tone_trace_nodes: n_skin_tone, + accessory_nodes: n_accessories, + object_nodes: n_objects, hand_nodes: n_hands, speaker_nodes: n_speakers, co_occurrence_edges: e_co, @@ -2559,21 +2562,109 @@ fn compute_skin_h_from_face(face: &serde_json::Value) -> f64 { } fn classify_fitzpatrick(h_mean: f64) -> &'static str { - if h_mean < 5.0 { - "Type I - Very Fair" - } else if h_mean < 12.0 { - "Type II - Fair" - } else if h_mean < 18.0 { - "Type III - Medium-Fair" - } else if h_mean < 25.0 { - "Type IV - Medium" - } else if h_mean < 35.0 { - "Type V - Dark" + if h_mean < 10.0 { + "I" + } else if h_mean < 20.0 { + "II" + } else if h_mean < 30.0 { + "III" + } else if h_mean < 40.0 { + "IV" + } else if h_mean < 50.0 { + "V" } else { - "Type VI - Very Dark" + "VI" } } +async fn build_skin_tone_trace_nodes( + pool: &PgPool, + file_uuid: &str, + output_dir: &str, +) -> Result { + let nodes_table = t("tkg_nodes"); + let fd_table = t("face_detections"); + let mut count = 0; + + let rows: Vec<(i64, i64)> = sqlx::query_as(&format!( + "SELECT trace_id, COUNT(*) \ + FROM {} \ + WHERE file_uuid = $1 AND trace_id IS NOT NULL \ + GROUP BY trace_id", + fd_table + )) + .bind(file_uuid) + .fetch_all(pool) + .await?; + + if rows.is_empty() { + tracing::info!("[TKG] No traced faces for skin_tone_trace nodes"); + return Ok(0); + } + + let path = Path::new(output_dir).join(format!("{}.face.json", file_uuid)); + let face_json = if path.exists() { + let content = std::fs::read_to_string(&path) + .with_context(|| format!("Failed to read face.json: {}", path.display()))?; + serde_json::from_str::(&content)? + } else { + tracing::warn!("[TKG] No face.json for skin_tone computation"); + serde_json::Value::Null + }; + + for (trace_id, frame_count) in &rows { + let avg_skin_h = if !face_json.is_null() { + let mut skin_h_values = Vec::new(); + if let Some(frames) = face_json.get("frames").and_then(|v| v.as_array()) { + for frame_entry in frames { + if let Some(faces) = frame_entry.get("faces").and_then(|v| v.as_array()) { + for face in faces { + let face_trace_id = face.get("trace_id").and_then(|v| v.as_i64()); + if face_trace_id == Some(*trace_id) { + skin_h_values.push(compute_skin_h_from_face(face)); + } + } + } + } + } + if skin_h_values.is_empty() { + 20.0 + } else { + skin_h_values.iter().sum::() / skin_h_values.len() as f64 + } + } else { + 20.0 + }; + + let fitz_type = classify_fitzpatrick(avg_skin_h); + let external_id = format!("skin_tone_{}", trace_id); + let label = format!("Skin Tone Trace {}", trace_id); + + sqlx::query(&format!( + "INSERT INTO {} (node_type, external_id, file_uuid, label, properties) \ + VALUES ('skin_tone_trace', $1, $2, $3, $4::jsonb) \ + ON CONFLICT (file_uuid, node_type, external_id) DO UPDATE SET properties = EXCLUDED.properties", + nodes_table + )) + .bind(&external_id) + .bind(file_uuid) + .bind(&label) + .bind(serde_json::json!({ + "trace_id": trace_id, + "avg_skin_h": avg_skin_h, + "fitzpatrick_type": fitz_type, + "frame_count": frame_count, + })) + .execute(pool) + .await?; + + count += 1; + } + + tracing::info!("[TKG] Built {} skin_tone_trace nodes", count); + Ok(count) +} + // ── Accessory Nodes ────────────────────────────────────────────── async fn build_accessory_nodes(pool: &PgPool, file_uuid: &str, output_dir: &str) -> Result { @@ -3124,10 +3215,11 @@ mod tests { let r = TkgResult { face_track_nodes: 5, gaze_track_nodes: 5, -lip_track_nodes: 4, -text_region_nodes: 20, -appearance_trace_nodes: 3, -accessory_nodes: 0, + lip_track_nodes: 4, + text_region_nodes: 20, + appearance_trace_nodes: 3, + skin_tone_trace_nodes: 5, + accessory_nodes: 0, object_nodes: 10, hand_nodes: 0, speaker_nodes: 3,