admin/momentry_core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ASRX hybrid pipeline, identity history, worker fixes, checkpoint system

Browse Source
This commit is contained in:
Accusys
2026-06-02 07:13:23 +08:00
parent e3066c3f49
commit e1572907ae
198 changed files with 43705 additions and 8910 deletions
Download Patch File Download Diff File Expand all files Collapse all files

518
src/api/agent_search.rs
View File

@@ -1,10 +1,4 @@
use axum::{
extract::State,
http::StatusCode,
response::Json,
routing::post,
Router,
};
use axum::{extract::State, http::StatusCode, response::Json, routing::post, Router};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -13,7 +7,10 @@ use std::time::Instant;
use crate::api::types::AppState;
use crate::core::db::schema;
use crate::core::llm::function_calling::{self, ChatMessage, LlmResponse, ToolCall, ToolDef};
use crate::core::llm::function_calling::{
self, call_llm_vision, ChatMessage, LlmResponse, ToolCall, ToolDef,
};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
// ── Conversation Manager ─────────────────────────────────────────
@@ -43,11 +40,14 @@ fn get_or_create_conv(conv_id: Option<&str>) -> (String, Vec<ChatMessage>) {
}
}
let id = uuid::Uuid::new_v4().to_string().replace('-', "")[..16].to_string();
map.insert(id.clone(), Conversation {
messages: Vec::new(),
created_at: Instant::now(),
last_active: Instant::now(),
});
map.insert(
id.clone(),
Conversation {
messages: Vec::new(),
created_at: Instant::now(),
last_active: Instant::now(),
},
);
(id, Vec::new())
}
@@ -85,8 +85,13 @@ const SYSTEM_PROMPT: &str = r#"你是 Momentry 影片分析助手。回答用戶
## 工具使用規則
1. 先確認用戶在問哪部影片 — 使用 find_file 或 list_files
2. 人物問題優先使用 tkg_query
3. 語意/內容問題使用 smart_search 或 universal_search
4. 可以同時呼叫多個工具
3. 人物台詞/發言問題使用 identities_search（輸入人名→回傳台詞片段）
4. 人物對話互動（誰跟誰說話）使用 tkg_query 的 speaker_interaction
5. 人物台詞內容使用 tkg_query 的 speaker_dialogue
6. 用文字反查人物使用 identity_text（輸入關鍵字→找出誰說/提到這段話）
7. 語意/內容問題使用 smart_search 或 universal_search
8. 畫面分析使用 analyze_frame — 可以分析影片中的任何畫面內容（場景、人物表情、動作、物件等）
9. 可以同時呼叫多個工具
## 引導規則
- 如果用戶沒說片名 → 用 find_file 搜尋，如果名稱不明確就反問
@@ -120,16 +125,16 @@ fn make_tools(pool: &sqlx::PgPool) -> Vec<ToolDef> {
),
function_calling::make_tool(
"tkg_query",
"查詢影片的人物互動、配對、同框資料。query_type 包括：top_identities（人物排名）、first_cooccurrence（第一次同框）、identity_details（人物詳細）、mutual_gaze（互看）、interaction_network（互動網絡）、identity_traces（出場片段）、file_info（影片資訊）。",
"查詢影片的人物互動、配對、同框、台詞資料。query_type 包括：top_identities（人物排名）、first_cooccurrence（第一次同框）、identity_details（人物詳細）、mutual_gaze（互看）、interaction_network（互動網絡）、identity_traces（出場片段）、file_info（影片資訊）、speaker_dialogue（人物台詞）、speaker_interaction（兩人對話互動）。",
serde_json::json!({
"file_uuid": {"type": "string", "description": "影片 UUID"},
"query_type": {
"type": "string",
"enum": ["top_identities", "first_cooccurrence", "identity_details", "mutual_gaze", "interaction_network", "identity_traces", "file_info"],
"enum": ["top_identities", "first_cooccurrence", "identity_details", "mutual_gaze", "interaction_network", "identity_traces", "file_info", "speaker_dialogue", "speaker_interaction"],
"description": "查詢類型"
},
"identity_name": {"type": "string", "description": "人物名稱（配合 identity_details / identity_traces）"},
"identity_b": {"type": "string", "description": "第二人物名稱（配合 first_cooccurrence / mutual_gaze）"},
"identity_name": {"type": "string", "description": "人物名稱（配合 identity_details / identity_traces / speaker_dialogue / speaker_interaction）"},
"identity_b": {"type": "string", "description": "第二人物名稱（配合 first_cooccurrence / mutual_gaze / speaker_interaction）"},
"limit": {"type": "integer", "default": 5}
}),
vec!["file_uuid", "query_type"],
@@ -144,6 +149,26 @@ fn make_tools(pool: &sqlx::PgPool) -> Vec<ToolDef> {
}),
vec!["query"],
),
function_calling::make_tool(
"identity_text",
"搜尋文字關鍵字，找出有提及該內容的影片人物。適合回答「誰說了OOO」、「誰跟OOO有關」。不是查詢人物的台詞，而是用文字反查人物。",
serde_json::json!({
"q": {"type": "string", "description": "搜尋關鍵字（台詞片段、主題等）"},
"file_uuid": {"type": "string", "description": "限制搜尋範圍（可選）"},
"limit": {"type": "integer", "default": 10}
}),
vec!["q"],
),
function_calling::make_tool(
"identities_search",
"查詢特定人物的台詞/發言內容。輸入人物名稱，回傳該人物在影片中說過的話。適合回答「某某人說了什麼」、「某某人的台詞」。",
serde_json::json!({
"q": {"type": "string", "description": "人物名稱關鍵字（姓名、角色名、別名）"},
"file_uuid": {"type": "string", "description": "限制搜尋範圍（可選）"},
"limit": {"type": "integer", "default": 10}
}),
vec!["q"],
),
function_calling::make_tool(
"get_identity_detail",
"查詢單一身份的詳細資料（名字、角色、TMDb 資訊）。",
@@ -168,6 +193,16 @@ fn make_tools(pool: &sqlx::PgPool) -> Vec<ToolDef> {
}),
vec!["file_uuid"],
),
function_calling::make_tool(
"analyze_frame",
"分析影片中指定畫面的視覺內容（場景、人物表情、動作、物件等）。若不指定 frame_number，會使用代表性畫面。問題會傳給視覺 LLM 分析。",
serde_json::json!({
"file_uuid": {"type": "string", "description": "影片 UUID"},
"question": {"type": "string", "description": "關於畫面的問題，例如「這個場景發生什麼事？」"},
"frame_number": {"type": "integer", "description": "指定的 frame 編號（可選）"}
}),
vec!["file_uuid"],
),
]
}
@@ -193,9 +228,10 @@ async fn exec_find_file(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result
if rows.is_empty() {
return Ok(serde_json::json!({"found": false, "message": "No files match the query. Try different keywords."}).to_string());
}
let files: Vec<serde_json::Value> = rows.into_iter().map(|(u, n, hd)| {
serde_json::json!({"file_uuid": u, "file_name": n, "has_data": hd})
}).collect();
let files: Vec<serde_json::Value> = rows
.into_iter()
.map(|(u, n, hd)| serde_json::json!({"file_uuid": u, "file_name": n, "has_data": hd}))
.collect();
Ok(serde_json::json!({"found": true, "files": files}).to_string())
}
@@ -214,15 +250,19 @@ async fn exec_list_files(pool: &sqlx::PgPool, args: &serde_json::Value) -> Resul
.await
.map_err(|e| e.to_string())?;
let files: Vec<serde_json::Value> = rows.into_iter().map(|(u, n, hd)| {
serde_json::json!({"file_uuid": u, "file_name": n, "has_data": hd})
}).collect();
let files: Vec<serde_json::Value> = rows
.into_iter()
.map(|(u, n, hd)| serde_json::json!({"file_uuid": u, "file_name": n, "has_data": hd}))
.collect();
Ok(serde_json::json!({"files": files}).to_string())
}
async fn exec_tkg_query(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
let query_type = args.get("query_type").and_then(|v| v.as_str()).unwrap_or("");
let query_type = args
.get("query_type")
.and_then(|v| v.as_str())
.unwrap_or("");
let identity_name = args.get("identity_name").and_then(|v| v.as_str());
let identity_b = args.get("identity_b").and_then(|v| v.as_str());
let limit = args.get("limit").and_then(|v| v.as_i64()).unwrap_or(5);
@@ -242,9 +282,11 @@ async fn exec_tkg_query(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result
GROUP BY i.uuid, i.name ORDER BY face_count DESC LIMIT $2",
fd_table, id_table
))
.bind(file_uuid).bind(limit)
.bind(file_uuid)
.bind(limit)
.fetch_all(pool)
.await.map_err(|e| e.to_string())?;
.await
.map_err(|e| e.to_string())?;
Ok(serde_json::json!({"identities": rows}).to_string())
}
"first_cooccurrence" => {
@@ -325,8 +367,9 @@ async fn exec_tkg_query(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result
}
"identity_traces" => {
let name = identity_name.unwrap_or("");
let rows: Vec<(i32, i64, i32, i32)> = sqlx::query_as(&format!(
"SELECT fd.trace_id, COUNT(*)::bigint, MIN(fd.frame_number)::int, MAX(fd.frame_number)::int \
// MIN/MAX frame_number should be bigint (i64), not int
let rows: Vec<(i32, i64, i64, i64)> = sqlx::query_as(&format!(
"SELECT fd.trace_id, COUNT(*)::bigint, MIN(fd.frame_number)::bigint, MAX(fd.frame_number)::bigint \
FROM {} fd JOIN {} i ON i.id = fd.identity_id \
WHERE fd.file_uuid = $1 AND i.name ILIKE $2 \
GROUP BY fd.trace_id ORDER BY COUNT(*) DESC LIMIT $3",
@@ -344,14 +387,133 @@ async fn exec_tkg_query(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result
))
.bind(file_uuid)
.fetch_optional(pool)
.await.map_err(|e| e.to_string())?;
.await
.map_err(|e| e.to_string())?;
Ok(serde_json::json!({"file_info": row.map(|(n, d, w, h, f)| serde_json::json!({"file_name": n, "duration_sec": d, "width": w, "height": h, "fps": f}))}).to_string())
}
_ => Ok(serde_json::json!({"error": format!("Unknown query_type: {}", query_type)}).to_string()),
"speaker_dialogue" => {
let name = identity_name.unwrap_or("");
let rows: Vec<(String, Option<String>)> = sqlx::query_as(&format!(
"SELECT DISTINCT sn.external_id, sn.properties->>'full_text' AS full_text \
FROM {} i \
JOIN {} fd ON fd.identity_id = i.id AND ($2::text IS NULL OR fd.file_uuid = $2) \
JOIN {} fn ON fn.file_uuid = fd.file_uuid \
AND fn.node_type = 'face_trace' \
AND fn.external_id = CONCAT('trace_', fd.trace_id) \
JOIN {} e ON e.source_node_id = fn.id \
AND e.edge_type = 'SPEAKS_AS' \
AND ($2::text IS NULL OR e.file_uuid = $2) \
JOIN {} sn ON sn.id = e.target_node_id \
WHERE i.name ILIKE $1 \
LIMIT $3",
id_table, fd_table, nodes, edges, nodes
))
.bind(name)
.bind(file_uuid)
.bind(limit)
.fetch_all(pool)
.await
.map_err(|e| e.to_string())?;
Ok(
serde_json::json!({"speakers": rows.iter().map(|(sid, text)| {
serde_json::json!({"speaker_id": sid, "dialogue": text})
}).collect::<Vec<_>>()})
.to_string(),
)
}
"speaker_interaction" => {
let name_a = identity_name.unwrap_or("");
let name_b = identity_b.unwrap_or("");
if name_a.is_empty() || name_b.is_empty() {
return Ok(
serde_json::json!({"error": "identity_name and identity_b are required"})
.to_string(),
);
}
// Get both speakers' segments from TKG
let rows: Vec<(String, String, serde_json::Value)> = sqlx::query_as(&format!(
"SELECT sn.external_id, sn.properties->>'full_text' AS full_text, sn.properties->'segments' AS segments \
FROM {} i \
JOIN {} fd ON fd.identity_id = i.id AND ($3::text IS NULL OR fd.file_uuid = $3) \
JOIN {} fn ON fn.file_uuid = fd.file_uuid \
AND fn.node_type = 'face_trace' \
AND fn.external_id = CONCAT('trace_', fd.trace_id) \
JOIN {} e ON e.source_node_id = fn.id \
AND e.edge_type = 'SPEAKS_AS' \
AND ($3::text IS NULL OR e.file_uuid = $3) \
JOIN {} sn ON sn.id = e.target_node_id \
WHERE (i.name ILIKE $1 OR i.name ILIKE $2) \
ORDER BY sn.external_id",
id_table, fd_table, nodes, edges, nodes
))
.bind(name_a)
.bind(name_b)
.bind(file_uuid)
.fetch_all(pool)
.await
.map_err(|e| e.to_string())?;
let mut interactions = Vec::new();
for i in 0..rows.len() {
for j in i + 1..rows.len() {
let (sid_a, text_a, segs_a_val) = &rows[i];
let (sid_b, text_b, segs_b_val) = &rows[j];
let segs_a = segs_a_val.as_array();
let segs_b = segs_b_val.as_array();
if let (Some(a_list), Some(b_list)) = (segs_a, segs_b) {
for sa in a_list {
let sa_start = sa.get("start").and_then(|v| v.as_f64()).unwrap_or(0.0);
let sa_end = sa.get("end").and_then(|v| v.as_f64()).unwrap_or(0.0);
let sa_text = sa.get("text").and_then(|v| v.as_str()).unwrap_or("");
if sa_text.is_empty() {
continue;
}
for sb in b_list {
let sb_start =
sb.get("start").and_then(|v| v.as_f64()).unwrap_or(0.0);
let sb_end = sb.get("end").and_then(|v| v.as_f64()).unwrap_or(0.0);
let sb_text = sb.get("text").and_then(|v| v.as_str()).unwrap_or("");
if sb_text.is_empty() {
continue;
}
// Check temporal overlap
let overlap_start = sa_start.max(sb_start);
let overlap_end = sa_end.min(sb_end);
if overlap_start < overlap_end {
interactions.push(serde_json::json!({
"speaker_a": sid_a,
"speaker_b": sid_b,
"time_range_s": [overlap_start, overlap_end],
"dialogue_a": sa_text,
"dialogue_b": sb_text,
}));
}
}
}
}
}
}
interactions.sort_by(|a, b| {
let a_start = a["time_range_s"][0].as_f64().unwrap_or(0.0);
let b_start = b["time_range_s"][0].as_f64().unwrap_or(0.0);
a_start.partial_cmp(&b_start).unwrap()
});
interactions.truncate(limit as usize);
Ok(serde_json::json!({"interactions": interactions, "speaker_a_text": rows.first().map(|r| r.1.clone()), "speaker_b_text": rows.get(1).map(|r| r.1.clone())}).to_string())
}
_ => Ok(
serde_json::json!({"error": format!("Unknown query_type: {}", query_type)}).to_string(),
),
}
}
async fn exec_smart_search(_pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
async fn exec_smart_search(
_pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let query = args.get("query").and_then(|v| v.as_str()).unwrap_or("");
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str());
let limit = args.get("limit").and_then(|v| v.as_i64()).unwrap_or(5);
@@ -359,7 +521,8 @@ async fn exec_smart_search(_pool: &sqlx::PgPool, args: &serde_json::Value) -> Re
let chunk_table = schema::table_name("chunk");
let mut sql = format!(
"SELECT chunk_id, text_content, start_frame, end_frame, chunk_type \
FROM {} WHERE text_content ILIKE $1", chunk_table
FROM {} WHERE text_content ILIKE $1",
chunk_table
);
if file_uuid.is_some() {
sql.push_str(" AND file_uuid = $2");
@@ -369,21 +532,147 @@ async fn exec_smart_search(_pool: &sqlx::PgPool, args: &serde_json::Value) -> Re
if let Some(fuid) = file_uuid {
let like = format!("%{}%", query);
let rows: Vec<(String, Option<String>, i64, i64, String)> = sqlx::query_as(&sql)
.bind(&like).bind(fuid)
.bind(&like)
.bind(fuid)
.fetch_all(_pool)
.await.map_err(|e| e.to_string())?;
.await
.map_err(|e| e.to_string())?;
Ok(serde_json::json!({"results": rows}).to_string())
} else {
let like = format!("%{}%", query);
let rows: Vec<(String, Option<String>, i64, i64, String)> = sqlx::query_as(&sql)
.bind(&like)
.fetch_all(_pool)
.await.map_err(|e| e.to_string())?;
.await
.map_err(|e| e.to_string())?;
Ok(serde_json::json!({"results": rows}).to_string())
}
}
async fn exec_get_identity_detail(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
async fn exec_identity_text(
pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let q = args.get("q").and_then(|v| v.as_str()).unwrap_or("");
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str());
let limit = args
.get("limit")
.and_then(|v| v.as_i64())
.unwrap_or(10)
.min(50);
let chunk_table = schema::table_name("chunk");
let fd_table = schema::table_name("face_detections");
let id_table = schema::table_name("identities");
let like_q = format!("%{}%", q.replace('%', "%%"));
let sql = format!(
"SELECT c.chunk_id, c.start_time, c.end_time, c.text_content, \
i.name AS identity_name, fd.trace_id, i.source AS identity_source \
FROM {} c \
JOIN {} fd ON fd.file_uuid = c.file_uuid \
AND fd.frame_number BETWEEN c.start_frame AND c.end_frame \
AND fd.identity_id IS NOT NULL \
JOIN {} i ON i.id = fd.identity_id \
WHERE ($1::text IS NULL OR c.file_uuid = $1) \
AND (LOWER(c.text_content) LIKE LOWER($2) OR LOWER(c.content::text) LIKE LOWER($2)) \
ORDER BY c.start_time \
LIMIT $3",
chunk_table, fd_table, id_table
);
let rows: Vec<(
String,
f64,
f64,
Option<String>,
String,
Option<i32>,
String,
)> = sqlx::query_as(&sql)
.bind(file_uuid)
.bind(&like_q)
.bind(limit)
.fetch_all(pool)
.await
.map_err(|e| e.to_string())?;
Ok(
serde_json::json!({"results": rows.iter().map(|(chunk_id, st, et, txt, name, tid, src)| {
serde_json::json!({
"chunk_id": chunk_id,
"start_time": st,
"end_time": et,
"text": txt,
"identity_name": name,
"trace_id": tid,
"source": src
})
} ).collect::<Vec<_>>()})
.to_string(),
)
}
async fn exec_identities_search(
pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let q = args.get("q").and_then(|v| v.as_str()).unwrap_or("");
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str());
let limit = args
.get("limit")
.and_then(|v| v.as_i64())
.unwrap_or(10)
.min(50);
let id_table = schema::table_name("identities");
let fd_table = schema::table_name("face_detections");
let chunk_table = schema::table_name("chunk");
let like_q = format!("%{}%", q.replace('%', "%%"));
let sql = format!(
"SELECT DISTINCT ON (i.name, c.chunk_id) \
i.name, c.chunk_id, c.start_time, c.end_time, c.text_content, fd.trace_id \
FROM {} i \
JOIN {} fd ON fd.identity_id = i.id \
JOIN {} c ON c.file_uuid = fd.file_uuid \
AND c.start_time <= fd.frame_number / COALESCE(c.fps, 25.0) \
AND c.end_time >= fd.frame_number / COALESCE(c.fps, 25.0) \
WHERE (i.name ILIKE $1 \
OR EXISTS (SELECT 1 FROM jsonb_array_elements(i.metadata->'aliases') AS a WHERE a->>'name' ILIKE $1)) \
AND ($2::text IS NULL OR fd.file_uuid = $2) \
ORDER BY i.name, c.chunk_id, c.start_time \
LIMIT $3",
id_table, fd_table, chunk_table
);
let rows: Vec<(String, String, f64, f64, Option<String>, Option<i32>)> = sqlx::query_as(&sql)
.bind(&like_q)
.bind(file_uuid)
.bind(limit)
.fetch_all(pool)
.await
.map_err(|e| e.to_string())?;
Ok(
serde_json::json!({"results": rows.iter().map(|(name, chunk_id, st, et, txt, tid)| {
serde_json::json!({
"identity_name": name,
"chunk_id": chunk_id,
"start_time": st,
"end_time": et,
"text": txt,
"trace_id": tid,
})
}).collect::<Vec<_>>()})
.to_string(),
)
}
async fn exec_get_identity_detail(
pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let name = args.get("name").and_then(|v| v.as_str()).unwrap_or("");
let id_table = schema::table_name("identities");
let row: Option<(String, String, Option<String>, Option<i32>, Option<String>)> = sqlx::query_as(&format!(
@@ -396,7 +685,10 @@ async fn exec_get_identity_detail(pool: &sqlx::PgPool, args: &serde_json::Value)
Ok(serde_json::json!({"identity": row.map(|(u, n, s, t, c)| serde_json::json!({"uuid": u, "name": n, "source": s, "tmdb_id": t, "character": c}))}).to_string())
}
async fn exec_get_file_info(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
async fn exec_get_file_info(
pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
let videos = schema::table_name("videos");
let row: Option<(String, f64, i32, i32, f64)> = sqlx::query_as(&format!(
@@ -405,11 +697,15 @@ async fn exec_get_file_info(pool: &sqlx::PgPool, args: &serde_json::Value) -> Re
))
.bind(file_uuid)
.fetch_optional(pool)
.await.map_err(|e| e.to_string())?;
.await
.map_err(|e| e.to_string())?;
Ok(serde_json::json!({"file_info": row.map(|(n, d, w, h, f)| serde_json::json!({"file_name": n, "duration_sec": d, "width": w, "height": h, "fps": f}))}).to_string())
}
async fn exec_get_representative_frame(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
async fn exec_get_representative_frame(
pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
match crate::core::processor::tkg::query_auto_representative_frame(pool, file_uuid).await {
Ok(r) => Ok(serde_json::json!({
@@ -417,24 +713,131 @@ async fn exec_get_representative_frame(pool: &sqlx::PgPool, args: &serde_json::V
"face_quality": r.face_quality,
"main_identities": r.main_identities,
"traces": r.traces,
}).to_string()),
})
.to_string()),
Err(e) => Ok(serde_json::json!({"error": e.to_string()}).to_string()),
}
}
async fn exec_analyze_frame(
pool: &sqlx::PgPool,
args: &serde_json::Value,
) -> Result<String, String> {
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
let question = args
.get("question")
.and_then(|v| v.as_str())
.unwrap_or("請描述這個畫面中的內容");
if file_uuid.is_empty() {
return Ok(serde_json::json!({"error": "file_uuid is required"}).to_string());
}
let videos = schema::table_name("videos");
let (video_path, fps): (String, f64) = sqlx::query_as(&format!(
"SELECT file_path, COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
videos
))
.bind(file_uuid)
.fetch_optional(pool)
.await
.map_err(|e| e.to_string())?
.ok_or_else(|| "Video not found".to_string())?;
let frame_number = match args.get("frame_number").and_then(|v| v.as_i64()) {
Some(f) => f,
None => {
match crate::core::processor::tkg::query_auto_representative_frame(pool, file_uuid)
.await
{
Ok(r) => r.frame_number,
Err(_) => {
let duration: f64 = sqlx::query_scalar(&format!(
"SELECT COALESCE(duration, 0) FROM {} WHERE file_uuid = $1",
videos
))
.bind(file_uuid)
.fetch_optional(pool)
.await
.map_err(|e| e.to_string())?
.unwrap_or(0.0);
if duration > 0.0 {
((duration / 2.0) * fps) as i64
} else {
0
}
}
}
}
};
let timestamp_secs = frame_number as f64 / fps;
let ffmpeg_path = std::env::var("MOMENTRY_FFMPEG").unwrap_or_else(|_| {
let full = "/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg";
if std::path::Path::new(full).exists() {
full.to_string()
} else {
"ffmpeg".to_string()
}
});
let output = tokio::process::Command::new(&ffmpeg_path)
.args([
"-ss",
&format!("{:.3}", timestamp_secs),
"-i",
&video_path,
"-vframes",
"1",
"-f",
"image2pipe",
"-vcodec",
"mjpeg",
"-",
])
.output()
.await
.map_err(|e| format!("ffmpeg execution error: {}", e))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Ok(serde_json::json!({"error": format!("ffmpeg failed: {}", stderr)}).to_string());
}
let base64_img = BASE64.encode(&output.stdout);
let system_prompt =
"你是一個專業的影片畫面分析助手。請根據提供的畫面以及用戶的問題，詳細描述畫面中的內容，包括場景、人物、動作、表情、物件等。請用繁體中文回答。";
let vision_result = call_llm_vision(system_prompt, question, vec![base64_img], 1024, 120)
.await
.map_err(|e| e.to_string())?;
Ok(serde_json::json!({
"frame_number": frame_number,
"timestamp_secs": timestamp_secs,
"analysis": vision_result,
})
.to_string())
}
// ── Tool Router ───────────────────────────────────────────────────
async fn execute_tool(pool: &sqlx::PgPool, tool_call: &ToolCall) -> (String, String, String) {
let name = tool_call.function.name.clone();
let args: serde_json::Value = serde_json::from_str(&tool_call.function.arguments).unwrap_or_default();
let args: serde_json::Value =
serde_json::from_str(&tool_call.function.arguments).unwrap_or_default();
let result = match name.as_str() {
"find_file" => exec_find_file(pool, &args).await,
"list_files" => exec_list_files(pool, &args).await,
"tkg_query" => exec_tkg_query(pool, &args).await,
"smart_search" => exec_smart_search(pool, &args).await,
"identity_text" => exec_identity_text(pool, &args).await,
"identities_search" => exec_identities_search(pool, &args).await,
"get_identity_detail" => exec_get_identity_detail(pool, &args).await,
"get_file_info" => exec_get_file_info(pool, &args).await,
"get_representative_frame" => exec_get_representative_frame(pool, &args).await,
"analyze_frame" => exec_analyze_frame(pool, &args).await,
_ => Err(format!("Unknown tool: {}", name)),
};
let content = match result {
@@ -476,7 +879,11 @@ async fn run_tool_loop(
for call in &calls {
let (tool_call_id, name, content) = execute_tool(pool, call).await;
sources.push(serde_json::json!({"tool": name, "result": content}));
messages.push(function_calling::make_tool_result(&tool_call_id, &name, &content));
messages.push(function_calling::make_tool_result(
&tool_call_id,
&name,
&content,
));
}
}
Err(e) => {
@@ -484,7 +891,10 @@ async fn run_tool_loop(
}
}
}
("已達到最大查詢次數，請縮小問題範圍後重新詢問。".to_string(), sources)
(
"已達到最大查詢次數，請縮小問題範圍後重新詢問。".to_string(),
sources,
)
}
// ── Handler ───────────────────────────────────────────────────────
@@ -495,13 +905,8 @@ async fn agent_search(
) -> Result<Json<AgentSearchResponse>, (StatusCode, Json<serde_json::Value>)> {
let (conv_id, history) = get_or_create_conv(req.conversation_id.as_deref());
let (answer, sources) = run_tool_loop(
state.db.pool(),
SYSTEM_PROMPT,
&req.query,
history,
)
.await;
let (answer, sources) =
run_tool_loop(state.db.pool(), SYSTEM_PROMPT, &req.query, history).await;
// Save updated messages for conversation continuation
let new_msgs = function_calling::build_conversation(SYSTEM_PROMPT, &req.query, vec![]);
@@ -509,7 +914,11 @@ async fn agent_search(
let needs_input = answer.contains('？') || answer.contains('?');
let suggestions = if needs_input {
Some(vec!["演員名".to_string(), "電影片名".to_string(), "年份".to_string()])
Some(vec![
"演員名".to_string(),
"電影片名".to_string(),
"年份".to_string(),
])
} else {
None
};
@@ -526,6 +935,5 @@ async fn agent_search(
// ── Routes ─────────────────────────────────────────────────────────
pub fn agent_search_routes() -> Router<AppState> {
Router::new()
.route("/api/v1/agents/search", post(agent_search))
Router::new().route("/api/v1/agents/search", post(agent_search))
}

3
src/api/docs.rs
View File

@@ -8,8 +8,7 @@ async fn doc_redirect() -> axum::response::Redirect {
async fn wasm_doc_handler() -> Result<impl axum::response::IntoResponse, (StatusCode, &'static str)>
{
let path =
std::path::Path::new("/Users/accusys/momentry_core/docs_v1.0/doc_wasm/index.html");
let path = std::path::Path::new("/Users/accusys/momentry_core/docs_v1.0/doc_wasm/index.html");
match tokio::fs::read_to_string(path).await {
Ok(html) => Ok(([("content-type", "text/html; charset=utf-8")], html)),
Err(_) => Err((StatusCode::NOT_FOUND, "Doc not found")),

198
src/api/files.rs
View File

@@ -12,7 +12,7 @@ use std::collections::HashMap;
use super::types::AppState;
use crate::core::config;
use crate::core::db::schema;
use crate::core::db::{Database, PostgresDb};
use crate::core::db::{Database, PostgresDb, QdrantDb, RedisClient};
use crate::core::storage::content_hash;
use crate::FileManager;
@@ -767,17 +767,7 @@ async fn register_file(
if let Some(ref vp) = video_path {
if let Ok(job) = auto_state.db.create_monitor_job(&auto_uuid, Some(vp)).await {
tracing::info!("[AUTO-PIPELINE] Job {} created for {}", job.id, auto_uuid);
let all_procs: Vec<&str> = vec![
"asr",
"cut",
"yolo",
"ocr",
"face",
"pose",
"asrx",
"visual_chunk",
"5w1h",
];
let all_procs: Vec<&str> = vec!["cut", "yolo", "ocr", "face", "pose", "asrx"];
let total = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COALESCE(total_frames, 0) FROM {} WHERE file_uuid = $1",
schema::table_name("videos")
@@ -986,6 +976,10 @@ struct UnregisterResponse {
deleted_face_detections: u64,
deleted_processor_results: u64,
deleted_chunks: u64,
deleted_tkg_nodes: u64,
deleted_qdrant_vectors: Option<u64>,
deleted_redis_keys: Option<u64>,
deleted_output_files: u64,
}
#[derive(Debug, Deserialize)]
@@ -994,18 +988,30 @@ struct UnregisterRequest {
file_path: Option<String>,
}
fn delete_output_files(uuid: &str) {
let output_dir = config::OUTPUT_DIR.to_string();
if let Ok(entries) = std::fs::read_dir(&output_dir) {
for entry in entries.flatten() {
let path = entry.path();
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if name.starts_with(uuid) {
let _ = std::fs::remove_file(&path);
fn delete_output_files(uuid: &str) -> u64 {
let mut deleted_count = 0u64;
let output_dirs = [
config::OUTPUT_DIR.to_string(),
"/Users/accusys/momentry/output_dev".to_string(),
"/Users/accusys/momentry/output".to_string(),
];
for output_dir in &output_dirs {
if let Ok(entries) = std::fs::read_dir(output_dir) {
for entry in entries.flatten() {
let path = entry.path();
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if name.starts_with(uuid) && name.ends_with(".json") {
if std::fs::remove_file(&path).is_ok() {
deleted_count += 1;
tracing::info!("[UNREGISTER] Deleted output file: {}", name);
}
}
}
}
}
}
deleted_count
}
async fn unregister(
@@ -1024,65 +1030,54 @@ async fn unregister(
let processor_table = schema::table_name("processor_results");
let chunks_table = schema::table_name("chunk");
let parent_chunks_table = schema::table_name("parent_chunks");
let deleted_faces: i64 =
sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", face_table))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete faces: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
let deleted_processors: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE file_uuid = $1",
processor_table
))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete processors: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
let deleted_parent_chunks: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE uuid = $1",
parent_chunks_table
))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete parent chunks: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
let deleted_chunks: i64 = sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", chunks_table))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete chunks: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
// Delete pre_chunks
let pre_chunks_table = schema::table_name("pre_chunks");
let deleted_pre_chunks: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE file_uuid = $1",
pre_chunks_table
let tkg_nodes_table = schema::table_name("tkg_nodes");
let cuts_table = schema::table_name("cuts");
let strangers_table = schema::table_name("strangers");
let chunk_vectors_table = schema::table_name("chunk_vectors");
let monitor_jobs_table = schema::table_name("monitor_jobs");
let frames_table = schema::table_name("frames");
let mut tx = state.db.pool().begin().await.map_err(|e| {
tracing::error!("[unregister] Failed to start transaction: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
macro_rules! delete_safe {
($table:expr, $where:expr, $bind:expr, $label:expr) => {{
sqlx::query(&format!("DELETE FROM {} WHERE {}", $table, $where))
.bind($bind)
.execute(&mut *tx)
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete {}: {}", $label, e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64
}};
}
let deleted_faces = delete_safe!(face_table, "file_uuid = $1", &uuid, "faces");
let deleted_processors = delete_safe!(processor_table, "file_uuid = $1", &uuid, "processors");
let deleted_parent_chunks =
delete_safe!(parent_chunks_table, "uuid = $1", &uuid, "parent chunks");
let deleted_chunks = delete_safe!(chunks_table, "file_uuid = $1", &uuid, "chunks");
let deleted_pre_chunks = delete_safe!(pre_chunks_table, "file_uuid = $1", &uuid, "pre_chunks");
let deleted_tkg_nodes = delete_safe!(tkg_nodes_table, "file_uuid = $1", &uuid, "TKG nodes");
let deleted_cuts = delete_safe!(cuts_table, "file_uuid = $1", &uuid, "cuts");
let deleted_strangers = delete_safe!(strangers_table, "file_uuid = $1", &uuid, "strangers");
let deleted_chunk_vectors =
delete_safe!(chunk_vectors_table, "uuid = $1", &uuid, "chunk vectors");
let deleted_monitor_jobs = delete_safe!(monitor_jobs_table, "uuid = $1", &uuid, "monitor jobs");
let deleted_frames: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE file_id = (SELECT id FROM {} WHERE file_uuid = $1)",
frames_table, videos_table
))
.bind(&uuid)
.execute(state.db.pool())
.execute(&mut *tx)
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete pre_chunks: {}", e);
tracing::error!("[unregister] Failed to delete frames: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
@@ -1092,14 +1087,59 @@ async fn unregister(
videos_table
))
.bind(&uuid)
.execute(state.db.pool())
.execute(&mut *tx)
.await
.map_err(|e| {
tracing::error!("[unregister] Failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
delete_output_files(&uuid);
tx.commit().await.map_err(|e| {
tracing::error!("[unregister] Failed to commit transaction: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
tracing::info!(
"[UNREGISTER] Deleted: {} faces, {} processors, {} parent_chunks, {} chunks, {} pre_chunks, {} tkg_nodes, {} cuts, {} strangers, {} chunk_vectors, {} monitor_jobs, {} frames",
deleted_faces, deleted_processors, deleted_parent_chunks, deleted_chunks,
deleted_pre_chunks, deleted_tkg_nodes, deleted_cuts, deleted_strangers,
deleted_chunk_vectors, deleted_monitor_jobs, deleted_frames
);
let deleted_output_files = delete_output_files(&uuid);
let deleted_qdrant_vectors = {
let qdrant = QdrantDb::new();
match qdrant.delete_by_uuid(&uuid).await {
Ok(_) => {
tracing::info!("[UNREGISTER] Deleted Qdrant vectors for {}", uuid);
Some(1)
}
Err(e) => {
tracing::warn!("[UNREGISTER] Failed to delete Qdrant vectors: {}", e);
None
}
}
};
let deleted_redis_keys = {
match RedisClient::new() {
Ok(redis) => match redis.delete_worker_job(&uuid).await {
Ok(_) => {
tracing::info!("[UNREGISTER] Deleted Redis keys for {}", uuid);
Some(1)
}
Err(e) => {
tracing::warn!("[UNREGISTER] Failed to delete Redis keys: {}", e);
None
}
},
Err(e) => {
tracing::warn!("[UNREGISTER] Failed to create Redis client: {}", e);
None
}
}
};
Ok(Json(UnregisterResponse {
success: true,
@@ -1107,7 +1147,11 @@ async fn unregister(
file_uuid: uuid,
deleted_face_detections: deleted_faces as u64,
deleted_processor_results: deleted_processors as u64,
deleted_chunks: (deleted_chunks + deleted_parent_chunks) as u64,
deleted_chunks: (deleted_chunks + deleted_parent_chunks + deleted_pre_chunks) as u64,
deleted_tkg_nodes: deleted_tkg_nodes as u64,
deleted_qdrant_vectors,
deleted_redis_keys,
deleted_output_files,
}))
}

4
src/api/five_w1h_agent_api.rs
View File

@@ -471,7 +471,7 @@ async fn store_parent_summary(
"sentence_count": sentences.len(),
});
sqlx::query(&format!(
r#"UPDATE {} SET summary_text = $1, metadata = metadata || $2::jsonb
r#"UPDATE {} SET summary_text = $1, metadata = jsonb_deep_merge(COALESCE(metadata, '{{}}'::jsonb), $2::jsonb)
WHERE chunk_id = $3 AND file_uuid = $4"#,
table
))
@@ -743,7 +743,7 @@ pub async fn run_5w1h_agent(db: &PostgresDb, file_uuid: &str) -> anyhow::Result<
// Auto-vectorize sentences with EmbeddingGemma (768D)
tracing::info!("[5W1H] Starting vectorize for sentence chunks...");
let embedder = Embedder::new("embeddinggemma-300M-Q8_0.gguf".to_string());
let embedder = Embedder::new("embeddinggemma-300m".to_string());
let qdrant = QdrantDb::new();
qdrant.init_collection(768).await?;

10
src/api/health.rs
View File

@@ -388,10 +388,18 @@ async fn health_detailed(State(state): State<AppState>) -> Json<DetailedHealthRe
let directory_exists = identities_root.is_dir();
let files_count = crate::core::identity::storage::count_identity_files();
let index_ok = crate::core::identity::storage::read_index().is_ok();
let db_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM identities")
let id_cnt: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM identities")
.fetch_one(state.db.pool())
.await
.unwrap_or(0);
let st_cnt: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid IS NOT NULL",
crate::core::db::schema::table_name("strangers")
))
.fetch_one(state.db.pool())
.await
.unwrap_or(0);
let db_count = id_cnt + st_cnt;
IdentityHealth {
directory_exists,
files_count,

14
src/api/identities.rs
View File

@@ -220,8 +220,8 @@ async fn list_identities(
.await
.unwrap_or(0);
let auto_identities: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE source = 'auto'",
identities_table
"SELECT COUNT(*) FROM {} WHERE file_uuid IS NOT NULL",
crate::core::db::schema::table_name("strangers")
))
.fetch_one(db.pool())
.await
@@ -258,7 +258,7 @@ pub struct FaceCandidate {
pub id: i32,
pub face_id: Option<String>,
pub file_uuid: String,
pub frame_number: i32,
pub frame_number: i64,
pub confidence: f32,
pub bbox: Option<serde_json::Value>,
pub attributes: Option<serde_json::Value>,
@@ -352,7 +352,7 @@ async fn list_face_candidates(
let rows = if let Some(file_uuid) = &query.file_uuid {
let sql = format!(
"SELECT id, face_id, file_uuid, frame_number::int, confidence::float4,
"SELECT id, face_id, file_uuid, frame_number::bigint, confidence::float4,
jsonb_build_object('x', x, 'y', y, 'width', width, 'height', height) as bbox,
NULL::jsonb as attributes
FROM {}
@@ -367,7 +367,7 @@ async fn list_face_candidates(
i32,
Option<String>,
String,
i32,
i64,
f32,
Option<serde_json::Value>,
Option<serde_json::Value>,
@@ -390,7 +390,7 @@ async fn list_face_candidates(
}
} else {
let sql = format!(
"SELECT id, face_id, file_uuid, frame_number::int, confidence::float4,
"SELECT id, face_id, file_uuid, frame_number::bigint, confidence::float4,
jsonb_build_object('x', x, 'y', y, 'width', width, 'height', height) as bbox,
NULL::jsonb as attributes
FROM {}
@@ -405,7 +405,7 @@ async fn list_face_candidates(
i32,
Option<String>,
String,
i32,
i64,
f32,
Option<serde_json::Value>,
Option<serde_json::Value>,

83
src/api/identity_agent_api.rs
View File

@@ -640,8 +640,9 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
);
// Step 2: 載入所有 face_detections（含 frame_number），按 trace_id 分組
// frame_number is BIGINT (i64) in database
let fd_table = schema::table_name("face_detections");
let fd_rows = sqlx::query_as::<_, (i32, i32, Vec<f32>)>(&format!(
let fd_rows = sqlx::query_as::<_, (i32, i64, Vec<f32>)>(&format!(
"SELECT trace_id, frame_number, embedding FROM {} \
WHERE file_uuid=$1 AND trace_id IS NOT NULL AND embedding IS NOT NULL \
ORDER BY trace_id, frame_number",
@@ -658,7 +659,7 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
// 分組：trace_id → (frame_number, embedding)
use std::collections::HashMap;
let mut trace_faces_raw: HashMap<i32, Vec<(i32, Vec<f32>)>> = HashMap::new();
let mut trace_faces_raw: HashMap<i32, Vec<(i64, Vec<f32>)>> = HashMap::new();
for (tid, frame, emb) in &fd_rows {
trace_faces_raw
.entry(*tid)
@@ -723,6 +724,7 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
// Step 5: 寫入 DB — Round 1 結果先存
let identities_table = schema::table_name("identities");
let strangers_table = schema::table_name("strangers");
let fd_table = schema::table_name("face_detections");
let mut updated = 0usize;
for (tid, name) in &matched {
@@ -805,13 +807,28 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
}
}
// Step 6: 未匹配的 trace 設 stranger_id = trace_id
// trace_id 在同一個 file 內是 sequential integer，直接複用為 stranger_id
// Step 6: 未匹配的 trace 設 stranger_id = strangers.id (FK)
// First: ensure strangers records exist
let _ = sqlx::query(&format!(
"INSERT INTO {} (file_uuid, trace_id) \
SELECT $1, fd.trace_id FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.trace_id IS NOT NULL \
AND fd.identity_id IS NULL \
ON CONFLICT (file_uuid, trace_id) DO NOTHING",
strangers_table, fd_table
))
.bind(file_uuid)
.execute(pool)
.await?;
// Then: update face_detections.stranger_id = strangers.id
let stranger_update = sqlx::query(&format!(
"UPDATE {} SET stranger_id = trace_id \
WHERE file_uuid = $1 AND trace_id IS NOT NULL AND identity_id IS NULL \
AND (stranger_id IS NULL OR stranger_id != trace_id)",
fd_table
"UPDATE {} fd SET stranger_id = s.id \
FROM {} s \
WHERE s.file_uuid = fd.file_uuid AND s.trace_id = fd.trace_id \
AND fd.file_uuid = $1 AND fd.identity_id IS NULL \
AND fd.trace_id IS NOT NULL AND fd.stranger_id IS NULL",
fd_table, strangers_table
))
.bind(file_uuid)
.execute(pool)
@@ -971,16 +988,30 @@ pub async fn bind_speakers(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Resu
let ib_table = schema::table_name("identity_bindings");
let _ = sqlx::query(
&format!("INSERT INTO {} (identity_id, identity_type, identity_value, confidence, metadata) \
VALUES ($1, 'speaker', $2, $3, $4::jsonb) \
ON CONFLICT (identity_id, identity_type, identity_value) DO UPDATE SET confidence = EXCLUDED.confidence, metadata = EXCLUDED.metadata", ib_table)
&format!("INSERT INTO {} (identity_id, identity_type, identity_value, file_uuid, confidence, metadata) \
VALUES ($1, 'speaker', $2, $3, $4, $5::jsonb) \
ON CONFLICT (identity_id, identity_type, identity_value, file_uuid) \
DO UPDATE SET confidence = EXCLUDED.confidence, metadata = EXCLUDED.metadata", ib_table)
)
.bind(identity_id)
.bind(&best_speaker)
.bind(file_uuid)
.bind(overlap_ratio)
.bind(&metadata)
.execute(pool).await;
// Also update speaker_detections with the identity_id
let sd_table = schema::table_name("speaker_detections");
let _ = sqlx::query(
&format!("UPDATE {} SET identity_id = $1, confidence = $2 \
WHERE file_uuid = $3 AND speaker_id = $4 AND identity_id IS NULL", sd_table)
)
.bind(identity_id)
.bind(overlap_ratio)
.bind(file_uuid)
.bind(&best_speaker)
.execute(pool).await;
bindings += 1;
}
}
@@ -1028,31 +1059,31 @@ pub async fn run_identity_agent(db: &PostgresDb, file_uuid: &str) -> anyhow::Res
let speakers = extract_speakers_from_asrx_data(&asrx_data);
let identities = analyze_person_speaker_overlap(&persons, &speakers);
for (idx, id_result) in identities.iter().enumerate() {
let identity_name = format!("stranger_{}", idx);
let _ = identities.len();
if !identities.is_empty() {
let metadata = serde_json::json!({
"source": "identity_agent",
"trace_ids": id_result.person_ids,
"speaker_ids": id_result.speaker_ids,
"confidence": id_result.confidence,
"speaker_ids": identities[0].speaker_ids,
"confidence": identities[0].confidence,
"evidence": {
"speaker_overlap": id_result.evidence.speaker_overlap,
"frame_ratio": id_result.evidence.frame_ratio,
"speaker_overlap": identities[0].evidence.speaker_overlap,
"frame_ratio": identities[0].evidence.frame_ratio,
},
"reasoning": id_result.reasoning,
"reasoning": identities[0].reasoning,
});
let _ = sqlx::query(
&format!("INSERT INTO {} (name, identity_type, source, metadata, status) VALUES ($1, 'people', 'auto', $2::jsonb, 'pending') ON CONFLICT DO NOTHING", schema::table_name("identities"))
)
.bind(&identity_name)
let _ = sqlx::query(&format!(
"INSERT INTO {} (file_uuid, trace_id, metadata) \
VALUES ($1, NULL, $2::jsonb) ON CONFLICT DO NOTHING",
schema::table_name("strangers")
))
.bind(file_uuid)
.bind(&metadata)
.execute(pool)
.await;
}
let _created = identities.len();
tracing::info!(
"[IdentityAgent] Created {} auto identities from face_clustered for {}",
_created,
"[IdentityAgent] Analyzed {} face clusters from face_clustered for {}",
identities.len(),
file_uuid
);
} else {

1331
src/api/identity_api.rs
View File

File diff suppressed because it is too large Load Diff

1402
src/api/identity_binding.rs
View File

File diff suppressed because it is too large Load Diff

224
src/api/media_api.rs
View File

@@ -57,6 +57,10 @@ pub fn bbox_routes() -> Router<crate::api::types::AppState> {
"/api/v1/file/:file_uuid/trace/:trace_id/video",
get(trace_video),
)
.route(
"/api/v1/file/:file_uuid/stranger/:stranger_id/video",
get(stranger_video),
)
.route("/api/v1/file/:file_uuid/video", get(stream_video))
.route("/api/v1/file/:file_uuid/thumbnail", get(face_thumbnail))
.route("/api/v1/file/:file_uuid/clip", get(video_clip))
@@ -210,8 +214,9 @@ async fn bbox_overlay_video(
let start_sec = start_f as f64 / fps;
// Get face bboxes
// frame_number is BIGINT (i64) in database
let face_table = schema::table_name("face_detections");
let rows: Vec<(i32, i32, i32, i32, i32, Option<i32>, Option<String>)> = sqlx::query_as(
let rows: Vec<(i64, i32, i32, i32, i32, Option<i32>, Option<String>)> = sqlx::query_as(
&format!("SELECT frame_number, x, y, width, height, trace_id, face_id FROM {} WHERE file_uuid = $1 AND frame_number BETWEEN $2 AND $3 ORDER BY frame_number", face_table)
)
.bind(face_fuid).bind(start_f).bind(end_f)
@@ -222,7 +227,7 @@ async fn bbox_overlay_video(
let mut parts: Vec<String> = Vec::new();
for (frame, x, y, w, h, trace_id, _) in &rows {
let text = format!("t{}", trace_id.unwrap_or(0));
let offset = frame - start_f;
let offset = (*frame as i32) - start_f;
parts.push(format!(
"drawbox=x={}:y={}:w={}:h={}:color=red@0.8:thickness=4:enable='eq(n,{})'",
x, y, w, h, offset
@@ -300,6 +305,15 @@ async fn trace_video(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
Query(params): Query<std::collections::HashMap<String, String>>,
) -> Result<impl IntoResponse, StatusCode> {
trace_video_inner(&state, &file_uuid, trace_id, &params).await
}
async fn trace_video_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
trace_id: i32,
params: &std::collections::HashMap<String, String>,
) -> Result<impl IntoResponse, StatusCode> {
use axum::http::header;
@@ -317,8 +331,9 @@ async fn trace_video(
let (video_path, fps, _width, _height) = row.ok_or(StatusCode::NOT_FOUND)?;
// Query face detections to find frame range for target trace
// frame_number is BIGINT (i64) in database
let face_table = schema::table_name("face_detections");
let rows: Vec<(i32, i32, i32, i32, i32)> = sqlx::query_as(&format!(
let rows: Vec<(i64, i32, i32, i32, i32)> = sqlx::query_as(&format!(
"SELECT frame_number, x, y, width, height FROM {} WHERE file_uuid = $1 AND trace_id = $2 ORDER BY frame_number",
face_table
))
@@ -371,11 +386,12 @@ async fn trace_video(
// === DEBUG MODE: text overlay, list all traces in frame range ===
let start_fn = (start_sec * fps) as i32;
let end_fn = ((start_sec + duration) * fps) as i32;
let end_fn = ((start_sec + duration) * fps) as i64;
// Query all traces with identity names and bbox positions in the visible frame range
// frame_number is BIGINT (i64) in database
let identities_table = schema::table_name("identities");
let all_rows: Vec<(i32, i32, i32, i32, i32, i32, Option<String>)> = sqlx::query_as(&format!(
let all_rows: Vec<(i32, i64, i32, i32, i32, i32, Option<String>)> = sqlx::query_as(&format!(
"SELECT fd.trace_id, fd.frame_number, fd.x, fd.y, fd.width, fd.height, i.name \
FROM {} fd \
LEFT JOIN {} i ON fd.identity_id = i.id \
@@ -391,9 +407,10 @@ async fn trace_video(
.unwrap_or_default();
// Group frames by trace_id, compute start_frame per trace; collect bbox per frame
let mut trace_frames: HashMap<i32, Vec<i32>> = HashMap::new();
// frame_number is i64 (BIGINT), so HashMaps need i64 for frame values
let mut trace_frames: HashMap<i32, Vec<i64>> = HashMap::new();
let mut trace_identity: HashMap<i32, String> = HashMap::new();
let mut bbox_per_frame: HashMap<(i32, i32), (i32, i32, i32, i32)> = HashMap::new(); // (tid, fn) -> (x, y, w, h)
let mut bbox_per_frame: HashMap<(i32, i64), (i32, i32, i32, i32)> = HashMap::new(); // (tid, fn) -> (x, y, w, h)
for (tid, fn_, x, y, w, h, name_opt) in &all_rows {
trace_frames.entry(*tid).or_default().push(*fn_);
bbox_per_frame.insert((*tid, *fn_), (*x, *y, *w, *h));
@@ -417,7 +434,7 @@ async fn trace_video(
.unwrap_or_else(|| "-".to_string());
// Sort traces for consistent ordering
let mut sorted_traces: Vec<(i32, &Vec<i32>)> =
let mut sorted_traces: Vec<(i32, &Vec<i64>)> =
trace_frames.iter().map(|(k, v)| (*k, v)).collect();
sorted_traces.sort_by_key(|(tid, _)| *tid);
@@ -695,6 +712,7 @@ struct ThumbQuery {
y: Option<i32>,
w: Option<i32>,
h: Option<i32>,
trace_id: Option<i32>,
}
async fn face_thumbnail(
@@ -717,15 +735,70 @@ async fn face_thumbnail(
}
};
let row: Option<(String,)> = sqlx::query_as(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1",
// Step 1: Check for pre-stored face crop if trace_id is provided
if let Some(trace_id) = q.trace_id {
let output_dir = crate::core::config::OUTPUT_DIR.as_str();
let cached_path = std::path::PathBuf::from(output_dir)
.join(".faces")
.join(&file_uuid)
.join(trace_id.to_string())
.join(format!("{}.jpg", frame));
if cached_path.exists() {
tracing::debug!("[thumbnail] Using cached face crop: {}", cached_path.display());
let bytes = tokio::fs::read(&cached_path)
.await
.map_err(|e| {
tracing::warn!("[thumbnail] Failed to read cached file: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
// Validate cached JPEG
crate::core::thumbnail::validator::validate_jpeg(&bytes).map_err(|e| {
tracing::warn!("[thumbnail] Cached JPEG validation failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
return Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(bytes))
.unwrap());
}
// Cached file not found, fallback to ffmpeg
tracing::debug!("[thumbnail] Cached file not found, falling back to ffmpeg");
}
// Step 2: Fallback to ffmpeg on-demand extraction
let row: Option<(String, Option<i64>, Option<i32>, Option<i32>)> = sqlx::query_as(&format!(
"SELECT file_path, total_frames, width, height FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let (file_path,) = row.ok_or(StatusCode::NOT_FOUND)?;
let (file_path, total_frames, video_width, video_height) = row.ok_or(StatusCode::NOT_FOUND)?;
if let Some(total) = total_frames {
if total > 0 {
crate::core::thumbnail::validator::validate_frame(frame, total).map_err(|e| {
tracing::warn!("[thumbnail] Frame validation failed: {}", e);
StatusCode::BAD_REQUEST
})?;
}
}
if let (Some(x), Some(y), Some(w), Some(h)) = (q.x, q.y, q.w, q.h) {
if let (Some(vw), Some(vh)) = (video_width, video_height) {
crate::core::thumbnail::validator::validate_crop(x, y, w, h, vw, vh).map_err(|e| {
tracing::warn!("[thumbnail] Crop validation failed: {}", e);
StatusCode::BAD_REQUEST
})?;
}
}
let select = format!("select=eq(n\\,{})", frame);
let vf = if let (Some(x), Some(y), Some(w), Some(h)) = (q.x, q.y, q.w, q.h) {
@@ -755,6 +828,11 @@ async fn face_thumbnail(
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
crate::core::thumbnail::validator::validate_jpeg(&output.stdout).map_err(|e| {
tracing::warn!("[thumbnail] JPEG validation failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
@@ -849,3 +927,127 @@ async fn video_clip(
.body(Body::from(output.stdout))
.unwrap())
}
async fn stranger_video(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, stranger_id)): Path<(String, i32)>,
Query(params): Query<std::collections::HashMap<String, String>>,
) -> Result<impl IntoResponse, StatusCode> {
stranger_video_inner(&state, &file_uuid, stranger_id, &params).await
}
async fn stranger_video_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
stranger_id: i32,
params: &std::collections::HashMap<String, String>,
) -> Result<impl IntoResponse, StatusCode> {
use axum::http::header;
use uuid::Uuid;
tracing::info!("[stranger_video] Starting for file={}, stranger={}", file_uuid, stranger_id);
let (mode, audio) = parse_video_params(&params);
let videos_table = schema::table_name("videos");
tracing::debug!("[stranger_video] videos_table: {}", videos_table);
let row: Option<(String, f64, i32, i32)> = sqlx::query_as(&format!(
"SELECT file_path, COALESCE(fps, 24.0), COALESCE(width, 0), COALESCE(height, 0) FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[stranger_video] Video query error: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let (video_path, fps, _width, _height) = row.ok_or_else(|| {
tracing::error!("[stranger_video] Video not found for uuid={}", file_uuid);
StatusCode::NOT_FOUND
})?;
tracing::info!("[stranger_video] Found video: path={}, fps={}", video_path, fps);
// Query face detections by stranger_id directly
let face_table = schema::table_name("face_detections");
tracing::debug!("[stranger_video] face_table: {}", face_table);
// frame_number is BIGINT (i64) in database
let rows: Vec<(i64, i32, i32, i32, i32)> = sqlx::query_as(&format!(
"SELECT frame_number, x, y, width, height FROM {} WHERE file_uuid = $1 AND stranger_id = $2 ORDER BY frame_number",
face_table
))
.bind(&file_uuid).bind(stranger_id)
.fetch_all(state.db.pool()).await
.unwrap_or_else(|e| {
tracing::error!("[stranger_video] Face query error: {}", e);
vec![]
});
tracing::info!("[stranger_video] Found {} faces", rows.len());
if rows.is_empty() {
tracing::error!("[stranger_video] No faces found for stranger_id={}", stranger_id);
return Err(StatusCode::NOT_FOUND);
}
let first_frame = rows[0].0;
let last_frame = rows[rows.len() - 1].0;
let start_sec = first_frame as f64 / fps;
let padding = params
.get("padding")
.and_then(|s| s.parse().ok())
.unwrap_or(2.0);
let duration = (last_frame - first_frame) as f64 / fps + padding * 2.0;
let seek = (start_sec - padding).max(0.0);
tracing::info!("[stranger_video] Frame range: {} - {}, time: {:.2}s - {:.2}s",
first_frame, last_frame, seek, seek + duration);
// Only support normal mode for stranger video
let tmp = std::env::temp_dir().join(format!("stranger_{}.mp4", Uuid::new_v4()));
let tmp_str = tmp.to_str().unwrap_or("").to_string();
let sk = seek.to_string();
let du = duration.to_string();
let mut cmd_args = vec!["-ss", &sk, "-i", &video_path, "-t", &du, "-c", "copy"];
if audio == "off" {
cmd_args.push("-an");
}
cmd_args.extend_from_slice(&["-y", &tmp_str]);
tracing::debug!("[stranger_video] ffmpeg args: {:?}", cmd_args);
let result = ffmpeg_cmd()
.args(&cmd_args)
.output()
.map_err(|e| {
tracing::error!("[stranger_video] ffmpeg spawn error: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
if !result.status.success() {
tracing::error!("[stranger_video] ffmpeg failed: {}", String::from_utf8_lossy(&result.stderr));
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
tracing::info!("[stranger_video] ffmpeg success, output size: {} bytes", result.stdout.len());
let data = tokio::fs::read(&tmp)
.await
.map_err(|e| {
tracing::error!("[stranger_video] Read output error: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let _ = std::fs::remove_file(&tmp);
tracing::info!("[stranger_video] Returning video, size: {} bytes", data.len());
Ok(Response::builder()
.header(header::CONTENT_TYPE, "video/mp4")
.header(header::CONTENT_LENGTH, data.len())
.body(Body::from(data))
.unwrap())
}

4
src/api/mod.rs
View File

@@ -4,7 +4,6 @@ pub mod auth;
pub mod docs;
pub mod files;
pub mod five_w1h_agent_api;
pub mod processing;
pub mod health;
pub mod identities;
pub mod identity_agent_api;
@@ -12,6 +11,7 @@ pub mod identity_api;
pub mod identity_binding;
pub mod media_api;
pub mod middleware;
pub mod processing;
pub mod scan;
pub mod search;
pub mod server;
@@ -19,7 +19,5 @@ pub mod tmdb_api;
pub mod trace_agent_api;
pub mod types;
pub mod universal_search;
pub mod visual_chunk_search;
pub mod visual_search;
pub use server::start_server;

88
src/api/processing.rs
View File

@@ -233,50 +233,54 @@ async fn trigger_processing(
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let processors_to_run: Vec<&str> = if let Some(procs) = &req.processors {
// 檢查 job 是否存在，不存在則 INSERT（state machine entry）
let existing_id: Option<i32> = sqlx::query_scalar(&format!(
"SELECT id FROM {monitor_jobs_table} WHERE uuid = $1"
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if existing_id.is_none() {
state
.db
.create_monitor_job(&file_uuid, Some(&file_path))
.await
.map_err(|e| {
tracing::error!(
"[TRIGGER] Failed to create monitor job for {}: {}",
file_uuid,
e
);
StatusCode::INTERNAL_SERVER_ERROR
})?;
}
// UPDATE processors + reset 狀態讓 worker 可 pickup
let procs_db: Vec<String> = procs.iter().map(|s| s.to_string()).collect();
sqlx::query(&format!(
"UPDATE {monitor_jobs_table} SET processors = $1::text[], status = 'pending' WHERE uuid = $2"
))
.bind(&procs_db)
.bind(&file_uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[TRIGGER] Failed to update monitor job for {}: {}", file_uuid, e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
procs.iter().map(|s| s.as_str()).collect()
let processors_to_run: Vec<String> = if let Some(procs) = &req.processors {
procs.iter().map(|s| s.to_string()).collect()
} else {
vec![]
crate::core::db::ProcessorType::all()
.iter()
.map(|p| p.as_str().to_string())
.collect()
};
// 確保 monitor_job 存在
let existing_id: Option<i32> = sqlx::query_scalar(&format!(
"SELECT id FROM {monitor_jobs_table} WHERE uuid = $1"
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if existing_id.is_none() {
state
.db
.create_monitor_job(&file_uuid, Some(&file_path))
.await
.map_err(|e| {
tracing::error!(
"[TRIGGER] Failed to create monitor job for {}: {}",
file_uuid,
e
);
StatusCode::INTERNAL_SERVER_ERROR
})?;
}
// UPDATE processors + reset 狀態讓 worker 可 pickup
sqlx::query(&format!(
"UPDATE {monitor_jobs_table} SET processors = $1::text[], status = 'pending' WHERE uuid = $2"
))
.bind(&processors_to_run)
.bind(&file_uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[TRIGGER] Failed to update monitor job for {}: {}", file_uuid, e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let processors_to_run_refs: Vec<&str> = processors_to_run.iter().map(|s| s.as_str()).collect();
let notification = serde_json::json!({
"action": "process",
"file_uuid": file_uuid,
@@ -285,7 +289,7 @@ async fn trigger_processing(
"file_type": file_type,
"content_hash": content_hash,
"output_dir": output_dir,
"processors": processors_to_run,
"processors": processors_to_run_refs,
});
let notification_key = format!("{}notifications", REDIS_KEY_PREFIX.as_str());

7
src/api/scan.rs
View File

@@ -414,8 +414,6 @@ async fn get_ingestion_status(
"SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
schema::table_name("tkg_edges")
));
let scene_5w1h = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut' AND summary_text IS NOT NULL AND summary_text != ''"));
let related_identities: Vec<IdentityRef> =
match sqlx::query_as::<_, (String, String)>(&format!(
"SELECT DISTINCT i.uuid::text, i.name FROM {identities} i \
@@ -491,11 +489,6 @@ async fn get_ingestion_status(
Some(format!("{identity_count} identities matched"))
),
step!("scene_metadata", scene_meta_ok, None),
step!(
"5w1h",
scene_5w1h > 0,
Some(format!("{scene_5w1h} scenes with 5W1H"))
),
];
Ok(Json(IngestionStatusResponse {

13
src/api/server.rs
View File

@@ -5,7 +5,7 @@ use tokio::time::timeout;
use tower_http::cors::{Any, CorsLayer};
use crate::core::cache::{MongoCache, RedisCache};
use crate::core::db::{Database, PostgresDb};
use crate::core::db::{Database, PostgresDb, QdrantDb};
use crate::Embedder;
use super::agent_api;
@@ -14,7 +14,6 @@ use super::auth;
use super::docs;
use super::files;
use super::five_w1h_agent_api;
use super::processing;
use super::health;
use super::identities;
use super::identity_agent_api;
@@ -22,18 +21,18 @@ use super::identity_api;
use super::identity_binding;
use super::media_api;
use super::middleware::unified_auth;
use super::processing;
use super::scan;
use super::search::search_routes;
use super::tmdb_api;
use super::trace_agent_api;
use super::types::AppState;
use super::universal_search::universal_search_routes;
use super::visual_search;
pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
health::init_server_state(host, port);
let embedder = std::sync::Arc::new(Embedder::new("nomic-embed-text-v2-moe:latest".to_string()));
let embedder = std::sync::Arc::new(Embedder::new("embeddinggemma-300m".to_string()));
// ── ⚠️ WARNING: DO NOT move MongoCache::init() back to critical path ──
//
@@ -57,6 +56,9 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
let redis_cache = RedisCache::new()?;
let db = PostgresDb::init().await?;
// Run migrations (create identity_history table if not exists)
PostgresDb::run_migrations(db.pool()).await?;
let schema_health = health::check_schema_migrations(db.pool()).await;
if schema_health.ok {
tracing::info!(
@@ -89,8 +91,10 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
let db = std::sync::Arc::new(db);
let api_state = super::middleware::ApiState { db: db.clone() };
let qdrant = std::sync::Arc::new(QdrantDb::new());
let state = AppState {
db,
qdrant,
embedder,
embedder_model: "nomic-embed-text-v2-moe:latest".to_string(),
mongo_cache,
@@ -129,7 +133,6 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
.merge(auth::auth_routes())
.merge(health::health_routes())
.merge(docs::doc_routes())
.merge(visual_search::visual_search_routes())
.merge(protected_routes)
.layer(cors)
.with_state(state);

419
src/api/trace_agent_api.rs
View File

@@ -25,14 +25,19 @@ pub fn trace_agent_routes() -> Router<crate::api::types::AppState> {
"/api/v1/file/:file_uuid/trace/:trace_id/thumbnail",
get(get_trace_thumbnail),
)
.route(
"/api/v1/file/:file_uuid/stranger/:stranger_id/representative-face",
get(get_stranger_representative_face),
)
.route(
"/api/v1/file/:file_uuid/stranger/:stranger_id/thumbnail",
get(get_stranger_thumbnail),
)
.route(
"/api/v1/file/:file_uuid/identities/:identity_uuid_a/co-occur-with/:identity_uuid_b",
get(get_cooccurrence),
)
.route(
"/api/v1/file/:file_uuid/tkg/rebuild",
post(rebuild_tkg),
)
.route("/api/v1/file/:file_uuid/tkg/rebuild", post(rebuild_tkg))
.route(
"/api/v1/file/:file_uuid/representative-frame",
get(get_representative_frame),
@@ -54,8 +59,8 @@ struct TracesRequest {
struct TraceInfo {
trace_id: i32,
face_count: i64,
start_frame: i32,
end_frame: i32,
start_frame: i64,
end_frame: i64,
start_time: f64,
end_time: f64,
duration_sec: f64,
@@ -110,8 +115,8 @@ async fn list_traces_sorted(
"SELECT tt.*, fd.id AS sample_face_id FROM (
SELECT trace_id::int AS trace_id,
COUNT(*) AS face_count,
MIN(frame_number)::int AS start_frame,
MAX(frame_number)::int AS end_frame,
MIN(frame_number)::bigint AS start_frame,
MAX(frame_number)::bigint AS end_frame,
(MAX(frame_number) - MIN(frame_number))::float8 AS duration_sec,
AVG(confidence)::float8 AS avg_confidence
FROM {}
@@ -132,7 +137,7 @@ async fn list_traces_sorted(
crate::core::db::schema::table_name("face_detections"),
);
let rows: Vec<(i32, i64, i32, i32, f64, f64, Option<i32>)> = sqlx::query_as(&query)
let rows: Vec<(i32, i64, i64, i64, f64, f64, Option<i32>)> = sqlx::query_as(&query)
.bind(&file_uuid)
.bind(min_faces)
.bind(effective_limit)
@@ -193,8 +198,8 @@ struct TraceFacesQuery {
#[derive(Debug, Serialize)]
struct TraceFaceItem {
id: i32,
start_frame: i32,
end_frame: i32,
start_frame: i64,
end_frame: i64,
start_time: f64,
end_time: f64,
x: Option<i32>,
@@ -260,14 +265,14 @@ async fn list_trace_faces(
let rows: Vec<(
i32,
i32,
i64,
Option<i32>,
Option<i32>,
Option<i32>,
Option<i32>,
f32,
)> = sqlx::query_as(&format!(
"SELECT id, frame_number::int, x, y, width, height, confidence::float4 \
"SELECT id, frame_number, x, y, width, height, confidence::float4 \
FROM {} WHERE file_uuid = $1 AND trace_id = $2 \
ORDER BY frame_number ASC LIMIT $3 OFFSET $4",
crate::core::db::schema::table_name("face_detections")
@@ -405,7 +410,8 @@ where
let video_table = schema::table_name("videos");
let fps: f64 = sqlx::query_scalar(&format!(
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1", video_table
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
video_table
))
.bind(file_uuid)
.fetch_optional(pool)
@@ -414,7 +420,8 @@ where
.unwrap_or(25.0);
let face_count: (i64,) = sqlx::query_as(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND trace_id = $2", fd_table
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND trace_id = $2",
fd_table
))
.bind(file_uuid)
.bind(trace_id)
@@ -422,7 +429,15 @@ where
.await
.map_err(|e| err_fn(anyhow::anyhow!("{}", e)))?;
struct Candidate { frame: i64, x: i32, y: i32, w: i32, h: i32, conf: f64, score: f64 }
struct Candidate {
frame: i64,
x: i32,
y: i32,
w: i32,
h: i32,
conf: f64,
score: f64,
}
let rows = sqlx::query_as::<_, (i64, i32, i32, i32, i32, f64)>(&format!(
"SELECT frame_number::bigint, x, y, width, height, confidence::float8 \
@@ -431,7 +446,8 @@ where
ORDER BY (width::float8 * height::float8) * confidence::float8 DESC LIMIT 10",
fd_table
))
.bind(file_uuid).bind(trace_id)
.bind(file_uuid)
.bind(trace_id)
.fetch_all(pool)
.await
.map_err(|e| err_fn(anyhow::anyhow!("{}", e)))?;
@@ -440,15 +456,25 @@ where
return Err(err_fn(anyhow::anyhow!("No suitable face found")));
}
let candidates: Vec<Candidate> = rows.into_iter()
let candidates: Vec<Candidate> = rows
.into_iter()
.map(|(frame, x, y, w, h, conf)| {
let score = (w as f64 * h as f64) * conf;
Candidate { frame, x, y, w, h, conf, score }
Candidate {
frame,
x,
y,
w,
h,
conf,
score,
}
})
.collect();
let video_path: String = sqlx::query_scalar(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1", video_table
"SELECT file_path FROM {} WHERE file_uuid = $1",
video_table
))
.bind(file_uuid)
.fetch_optional(pool)
@@ -463,16 +489,31 @@ where
for (i, c) in candidates.iter().enumerate() {
let seek = c.frame as f64 / fps;
if let Ok(output) = tokio::process::Command::new("ffmpeg")
.args(["-ss", &format!("{:.2}", seek), "-i", &video_path,
"-vframes", "1", "-vf", &format!("crop={}:{}:{}:{},blurdetect", c.w, c.h, c.x, c.y),
"-f", "null", "-"])
.output().await
.args([
"-ss",
&format!("{:.2}", seek),
"-i",
&video_path,
"-vframes",
"1",
"-vf",
&format!("crop={}:{}:{}:{},blurdetect", c.w, c.h, c.x, c.y),
"-f",
"null",
"-",
])
.output()
.await
{
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stderr.lines() {
if let Some(blur_str) = line.split("blur mean: ").nth(1) {
if let Ok(blur) = blur_str.trim().parse::<f64>() {
if blur < best_blur { best_blur = blur; best = c.frame; best_idx = i; }
if blur < best_blur {
best_blur = blur;
best = c.frame;
best_idx = i;
}
}
}
}
@@ -481,9 +522,17 @@ where
let chosen = &candidates[best_idx];
Ok(RepFaceSelection {
frame: chosen.frame, x: chosen.x, y: chosen.y, w: chosen.w, h: chosen.h,
conf: chosen.conf, blur: best_blur, score: chosen.score,
video_path, fps, face_count: face_count.0,
frame: chosen.frame,
x: chosen.x,
y: chosen.y,
w: chosen.w,
h: chosen.h,
conf: chosen.conf,
blur: best_blur,
score: chosen.score,
video_path,
fps,
face_count: face_count.0,
})
}
@@ -491,19 +540,36 @@ async fn get_representative_face(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
) -> Result<Json<RepFaceResponse>, (StatusCode, Json<serde_json::Value>)> {
let sel = select_rep_face(state.db.pool(), &file_uuid, trace_id, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await?;
get_representative_face_inner(&state, &file_uuid, trace_id).await
}
async fn get_representative_face_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
trace_id: i32,
) -> Result<Json<RepFaceResponse>, (StatusCode, Json<serde_json::Value>)> {
let sel = select_rep_face(state.db.pool(), file_uuid, trace_id, |e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await?;
Ok(Json(RepFaceResponse {
success: true,
file_uuid,
file_uuid: file_uuid.to_string(),
trace_id,
face_count: sel.face_count,
representative: RepFaceResult {
frame_number: sel.frame,
timestamp_secs: sel.frame as f64 / sel.fps,
bbox: RepFaceBbox { x: sel.x, y: sel.y, width: sel.w, height: sel.h },
bbox: RepFaceBbox {
x: sel.x,
y: sel.y,
width: sel.w,
height: sel.h,
},
confidence: sel.conf,
quality_score: sel.score,
blur_score: sel.blur,
@@ -515,34 +581,118 @@ async fn get_trace_thumbnail(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
get_trace_thumbnail_inner(&state, &file_uuid, trace_id).await
}
async fn get_trace_thumbnail_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
trace_id: i32,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
// Step 1: Check for pre-stored face crops in .faces/{file_uuid}/{trace_id}/
// For trace_id=0 (untracked/stranger), check unbound directory instead
let output_dir = crate::core::config::OUTPUT_DIR.as_str();
let trace_id_str = trace_id.to_string();
let trace_dir_name = if trace_id == 0 { "unbound" } else { &trace_id_str };
let trace_dir = std::path::PathBuf::from(output_dir)
.join(".faces")
.join(&file_uuid)
.join(trace_dir_name);
if trace_dir.exists() {
// Find any cached face crop in this trace directory
if let Ok(mut entries) = std::fs::read_dir(&trace_dir) {
while let Some(Ok(entry)) = entries.next() {
let path = entry.path();
if path.extension().map_or(false, |e| e == "jpg") {
tracing::info!("[trace_thumbnail] Using cached face crop: {}", path.display());
let bytes = tokio::fs::read(&path)
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
// Validate cached JPEG
crate::core::thumbnail::validator::validate_jpeg(&bytes).map_err(|e| {
tracing::warn!("[trace_thumbnail] Cached JPEG validation failed: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "Invalid cached JPEG"})),
)
})?;
return Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(bytes))
.unwrap());
}
}
}
}
// Step 2: Fallback to ffmpeg on-demand extraction
let sel = select_rep_face(state.db.pool(), &file_uuid, trace_id, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await?;
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await?;
let seek = sel.frame as f64 / sel.fps;
let tmp = std::env::temp_dir().join(format!("trace_{}_{}.jpg", file_uuid, trace_id));
tracing::debug!("[trace_thumbnail] Fallback to ffmpeg for trace {} frame {}", trace_id, sel.frame);
let status = tokio::process::Command::new("ffmpeg")
.args([
"-ss", &format!("{:.2}", seek),
"-i", &sel.video_path,
"-vframes", "1",
"-vf", &format!("crop={}:{}:{}:{},scale=320:320", sel.w, sel.h, sel.x, sel.y),
"-q:v", "2",
"-y", &tmp.to_string_lossy().to_string(),
"-ss",
&format!("{:.2}", seek),
"-i",
&sel.video_path,
"-vframes",
"1",
"-vf",
&format!("crop={}:{}:{}:{},scale=320:320", sel.w, sel.h, sel.x, sel.y),
"-q:v",
"2",
"-y",
&tmp.to_string_lossy().to_string(),
])
.output()
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
if !status.status.success() {
return Err((StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": "FFmpeg failed"}))));
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "FFmpeg failed"})),
));
}
let bytes = tokio::fs::read(&tmp).await.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
crate::core::thumbnail::validator::validate_jpeg(&bytes).map_err(|e| {
tracing::warn!("[trace_thumbnail] JPEG validation failed: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "Invalid JPEG output"})),
)
})?;
let _ = tokio::fs::remove_file(&tmp).await;
@@ -605,10 +755,16 @@ async fn get_cooccurrence(
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or_else(|| {
(StatusCode::NOT_FOUND, Json(serde_json::json!({"error": "Identity A not found"})))
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Identity A not found"})),
)
})?;
let id_b = sqlx::query_as::<_, (i32, String)>(&format!(
@@ -619,31 +775,38 @@ async fn get_cooccurrence(
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or_else(|| {
(StatusCode::NOT_FOUND, Json(serde_json::json!({"error": "Identity B not found"})))
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Identity B not found"})),
)
})?;
// Stage 2: Find first frame where both identity_ids appear
let cooccur: Option<(i64,)> = sqlx::query_as(
&format!(
"SELECT MIN(fd.frame_number)::bigint FROM {} fd \
let cooccur: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT MIN(fd.frame_number)::bigint FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.identity_id = $2 \
AND fd.frame_number IN ( \
SELECT frame_number FROM {} \
WHERE file_uuid = $1 AND identity_id = $3 \
)",
fd_table, fd_table
)
)
fd_table, fd_table
))
.bind(&file_uuid)
.bind(id_a.0)
.bind(id_b.0)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
let (first_frame,) = cooccur.ok_or_else(|| {
@@ -653,13 +816,17 @@ async fn get_cooccurrence(
// Get fps for timestamp
let video_table = schema::table_name("videos");
let fps: f64 = sqlx::query_scalar(&format!(
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1", video_table
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
video_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.unwrap_or(25.0);
@@ -685,40 +852,67 @@ async fn get_cooccurrence(
// Stage 4: Get representative faces for both traces (reusing select_rep_face)
let rep_a = if let Some((tid,)) = trace_a {
select_rep_face(state.db.pool(), &file_uuid, tid, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await.ok().map(|sel| CoOccurRepFace {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await
.ok()
.map(|sel| CoOccurRepFace {
frame_number: sel.frame,
bbox: RepFaceBbox { x: sel.x, y: sel.y, width: sel.w, height: sel.h },
bbox: RepFaceBbox {
x: sel.x,
y: sel.y,
width: sel.w,
height: sel.h,
},
confidence: sel.conf,
thumbnail_url: format!("/api/v1/file/{}/trace/{}/thumbnail", file_uuid, tid),
})
} else { None };
} else {
None
};
let rep_b = if let Some((tid,)) = trace_b {
select_rep_face(state.db.pool(), &file_uuid, tid, |e| {
(StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({"error": e.to_string()})))
}).await.ok().map(|sel| CoOccurRepFace {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})
.await
.ok()
.map(|sel| CoOccurRepFace {
frame_number: sel.frame,
bbox: RepFaceBbox { x: sel.x, y: sel.y, width: sel.w, height: sel.h },
bbox: RepFaceBbox {
x: sel.x,
y: sel.y,
width: sel.w,
height: sel.h,
},
confidence: sel.conf,
thumbnail_url: format!("/api/v1/file/{}/trace/{}/thumbnail", file_uuid, tid),
})
} else { None };
} else {
None
};
// Total co-occurrence frames (from TKG if available, otherwise from face_detections)
let total_cooccurrence_frames: i64 = sqlx::query_scalar(
&format!(
"SELECT COUNT(DISTINCT fd.frame_number)::bigint FROM {} fd \
let total_cooccurrence_frames: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(DISTINCT fd.frame_number)::bigint FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.identity_id = $2 \
AND fd.frame_number IN ( \
SELECT frame_number FROM {} \
WHERE file_uuid = $1 AND identity_id = $3 \
)",
fd_table, fd_table
)
)
.bind(&file_uuid).bind(id_a.0).bind(id_b.0)
.fetch_one(state.db.pool()).await
fd_table, fd_table
))
.bind(&file_uuid)
.bind(id_a.0)
.bind(id_b.0)
.fetch_one(state.db.pool())
.await
.unwrap_or(0);
Ok(Json(CoOccurResponse {
@@ -758,12 +952,7 @@ async fn rebuild_tkg(
State(state): State<crate::api::types::AppState>,
Path(file_uuid): Path<String>,
) -> Json<TkgRebuildResponse> {
let result = crate::core::processor::tkg::build_tkg(
&state.db,
&file_uuid,
&OUTPUT_DIR,
)
.await;
let result = crate::core::processor::tkg::build_tkg(&state.db, &file_uuid, &OUTPUT_DIR).await;
match result {
Ok(r) => Json(TkgRebuildResponse {
@@ -807,14 +996,14 @@ async fn get_representative_frame(
State(state): State<crate::api::types::AppState>,
Path(file_uuid): Path<String>,
) -> Result<Json<RepFrameResponse>, (StatusCode, Json<serde_json::Value>)> {
let result = tkg::query_auto_representative_frame(
state.db.pool(),
&file_uuid,
)
.await
.map_err(|e| {
(StatusCode::NOT_FOUND, Json(serde_json::json!({"error": e.to_string()})))
})?;
let result = tkg::query_auto_representative_frame(state.db.pool(), &file_uuid)
.await
.map_err(|e| {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
let fps = query_fps(state.db.pool(), &file_uuid).await;
@@ -843,3 +1032,59 @@ async fn query_fps(pool: &sqlx::PgPool, file_uuid: &str) -> f64 {
.flatten()
.unwrap_or(25.0)
}
async fn get_stranger_representative_face(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, stranger_id)): Path<(String, i32)>,
) -> Result<Json<RepFaceResponse>, (StatusCode, Json<serde_json::Value>)> {
let faces_table = crate::core::db::schema::table_name("face_detections");
let trace_id: i32 = sqlx::query_scalar(&format!(
"SELECT trace_id FROM {} WHERE file_uuid = $1 AND stranger_id = $2 LIMIT 1",
faces_table
))
.bind(&file_uuid)
.bind(stranger_id)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or((
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Stranger not found"})),
))?;
get_representative_face_inner(&state, &file_uuid, trace_id).await
}
async fn get_stranger_thumbnail(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, stranger_id)): Path<(String, i32)>,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
let faces_table = crate::core::db::schema::table_name("face_detections");
let trace_id: i32 = sqlx::query_scalar(&format!(
"SELECT trace_id FROM {} WHERE file_uuid = $1 AND stranger_id = $2 LIMIT 1",
faces_table
))
.bind(&file_uuid)
.bind(stranger_id)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?
.ok_or((
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "Stranger not found"})),
))?;
get_trace_thumbnail_inner(&state, &file_uuid, trace_id).await
}

1
src/api/types.rs
View File

@@ -1,6 +1,7 @@
#[derive(Clone)]
pub struct AppState {
pub db: std::sync::Arc<crate::core::db::PostgresDb>,
pub qdrant: std::sync::Arc<crate::core::db::QdrantDb>,
pub embedder: std::sync::Arc<crate::Embedder>,
pub embedder_model: String,
pub mongo_cache: crate::core::cache::MongoCache,

72
src/api/universal_search.rs
View File

@@ -60,13 +60,12 @@ pub struct UniversalSearchResponse {
pub enum SearchResult {
#[serde(rename = "chunk")]
Chunk {
file_uuid: String,
chunk_id: String,
chunk_type: String,
// Primary: frame-accurate position
start_frame: i64,
end_frame: i64,
fps: f64,
// Reference: time derived from frames (subject to FPS variation)
start_time: f64,
end_time: f64,
score: f64,
@@ -76,9 +75,8 @@ pub enum SearchResult {
},
#[serde(rename = "frame")]
Frame {
// Primary: exact frame number
file_uuid: String,
frame_number: i64,
// Reference: time derived from frame (subject to FPS variation)
timestamp: f64,
score: f64,
objects: Option<Vec<serde_json::Value>>,
@@ -88,6 +86,7 @@ pub enum SearchResult {
},
#[serde(rename = "person")]
Person {
file_uuid: Option<String>,
identity_id: i32,
identity_uuid: String,
name: Option<String>,
@@ -328,17 +327,15 @@ async fn search_chunks(
db: &PostgresDb,
req: &UniversalSearchRequest,
) -> Result<Vec<SearchResult>, anyhow::Error> {
// uuid is required for chunk search - chunk_id is only unique within a video
let uuid = match &req.file_uuid {
Some(u) => u.replace('\'', "''"),
None => return Err(anyhow::anyhow!("file_uuid is required for chunk search")),
};
let chunk_table = schema::table_name("chunk");
let mut sql = format!(
"SELECT chunk_id, chunk_type, start_time, end_time, (start_time * fps)::bigint as start_frame, (end_time * fps)::bigint as end_frame, fps, text_content, content FROM {} WHERE file_uuid = '{}'",
chunk_table, uuid
"SELECT file_uuid, chunk_id, chunk_type, start_time, end_time, (start_time * fps)::bigint as start_frame, (end_time * fps)::bigint as end_frame, fps, text_content, content FROM {} WHERE 1=1",
chunk_table
);
if let Some(uuid) = &req.file_uuid {
sql.push_str(&format!(" AND file_uuid = '{}'", uuid.replace('\'', "''")));
}
if let Some(tr) = &req.time_range {
sql.push_str(&format!(
" AND start_time >= {} AND end_time <= {}",
@@ -422,6 +419,7 @@ async fn search_chunks(
sql.push_str(&format!(" LIMIT {}", req.page_size.unwrap_or(20)));
let rows: Vec<(
String,
String,
String,
f64,
@@ -437,6 +435,7 @@ async fn search_chunks(
.into_iter()
.map(
|(
file_uuid,
chunk_id,
chunk_type,
start_time,
@@ -457,7 +456,6 @@ async fn search_chunks(
.and_then(|v| v.as_str())
.map(String::from)
});
// Simple scoring: if query matches, score 0.8
let score = if !req.query.is_empty()
&& text.as_ref().map_or(false, |t| {
t.to_lowercase().contains(&req.query.to_lowercase())
@@ -468,6 +466,7 @@ async fn search_chunks(
};
SearchResult::Chunk {
file_uuid,
chunk_id,
chunk_type,
start_time,
@@ -549,7 +548,7 @@ async fn search_frames_internal(
let results: Vec<SearchResult> = rows
.into_iter()
.map(|(frame_number, timestamp, yolo, ocr, face, _uuid)| {
.map(|(frame_number, timestamp, yolo, ocr, face, file_uuid)| {
let objects = yolo.as_ref().and_then(|v| {
v.get("objects")
.map(|o| o.as_array().cloned().unwrap_or_default())
@@ -571,6 +570,7 @@ async fn search_frames_internal(
});
SearchResult::Frame {
file_uuid,
frame_number,
timestamp,
score: 0.7,
@@ -589,37 +589,54 @@ async fn search_persons_internal(
db: &PostgresDb,
req: &UniversalSearchRequest,
) -> Result<Vec<SearchResult>, anyhow::Error> {
let uuid = match &req.file_uuid {
Some(u) => u.replace('\'', "''"),
None => return Err(anyhow::anyhow!("file_uuid is required for person search")),
};
let id_table = schema::table_name("identities");
let fd_table = schema::table_name("face_detections");
let mut sql = format!(
"SELECT i.id, i.uuid::text, i.name, COUNT(fd.id) AS appearance_count, \
MIN(fd.timestamp_secs) AS first_time, MAX(fd.timestamp_secs) AS last_time \
FROM {} i JOIN {} fd ON fd.identity_id = i.id \
WHERE fd.file_uuid = '{}'",
id_table, fd_table, uuid
MIN(fd.timestamp_secs) AS first_time, MAX(fd.timestamp_secs) AS last_time, \
fd.file_uuid \
FROM {} i JOIN {} fd ON fd.identity_id = i.id WHERE 1=1",
id_table, fd_table
);
if let Some(uuid) = &req.file_uuid {
sql.push_str(&format!(
" AND fd.file_uuid = '{}'",
uuid.replace('\'', "''")
));
}
if !req.query.is_empty() {
let q = req.query.replace('\'', "''");
sql.push_str(&format!(" AND i.name ILIKE '%{}%'", q));
}
sql.push_str(" GROUP BY i.id, i.uuid, i.name");
sql.push_str(" GROUP BY i.id, i.uuid, i.name, fd.file_uuid");
sql.push_str(" ORDER BY appearance_count DESC");
sql.push_str(&format!(" LIMIT {}", req.page_size.unwrap_or(20)));
let rows: Vec<(i32, String, Option<String>, i64, Option<f64>, Option<f64>)> =
sqlx::query_as(&sql).fetch_all(db.pool()).await?;
let rows: Vec<(
i32,
String,
Option<String>,
i64,
Option<f64>,
Option<f64>,
String,
)> = sqlx::query_as(&sql).fetch_all(db.pool()).await?;
let results: Vec<SearchResult> = rows
.into_iter()
.map(
|(identity_id, identity_uuid, name, appearance_count, first_time, last_time)| {
|(
identity_id,
identity_uuid,
name,
appearance_count,
first_time,
last_time,
file_uuid,
)| {
let score = if !req.query.is_empty()
&& name.as_ref().map_or(false, |n| {
n.to_lowercase().contains(&req.query.to_lowercase())
@@ -630,6 +647,7 @@ async fn search_persons_internal(
};
SearchResult::Person {
file_uuid: Some(file_uuid),
identity_id,
identity_uuid,
name,

513
src/api/visual_chunk_search.rs
View File

@@ -1,513 +0,0 @@
//! Visual chunk search functionality.
//!
//! This module provides search capabilities for visual chunks based on:
//! - Object classes (e.g., "person", "car", "envelope")
//! - Confidence thresholds
//! - Object counts
//! - Spatial density
//! - Object relationships
use crate::core::chunk::types::{Chunk, ChunkRule, ChunkType};
use crate::core::db::{schema, PostgresDb};
use anyhow::Result;
use serde_json::Value;
use std::collections::HashMap;
/// Criteria for searching visual chunks
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct VisualChunkSearchCriteria {
/// Minimum average confidence across frames
pub min_avg_confidence: Option<f32>,
/// Minimum number of frames with objects
pub min_frames_with_objects: Option<u32>,
/// Minimum number of unique object classes
pub min_unique_classes: Option<u32>,
/// Specific object classes to include (empty means all)
#[serde(default)]
pub required_classes: Vec<String>,
/// Object class counts to filter by
#[serde(default)]
pub class_counts: HashMap<String, (u32, u32)>,
/// Time range (optional)
pub time_range: Option<(f64, f64)>,
}
impl Default for VisualChunkSearchCriteria {
fn default() -> Self {
Self {
min_avg_confidence: None,
min_frames_with_objects: None,
min_unique_classes: None,
required_classes: Vec::new(),
class_counts: HashMap::new(),
time_range: None,
}
}
}
/// Search visual chunks based on criteria
pub async fn search_visual_chunks(
db: &PostgresDb,
uuid: &str,
criteria: &VisualChunkSearchCriteria,
) -> Result<Vec<Chunk>> {
// First, get all visual chunks for this video
let all_chunks = get_visual_chunks_by_uuid(db, uuid).await?;
// Apply filters
let filtered_chunks: Vec<Chunk> = all_chunks
.into_iter()
.filter(|chunk| {
// Check min avg confidence
if let Some(min_avg_confidence) = criteria.min_avg_confidence {
if let Some(content) = &chunk.content.as_object() {
if let Some(metadata) = content.get("metadata") {
if let Some(avg_confidence) = metadata.get("avg_confidence") {
if let Some(conf) = avg_confidence.as_f64() {
if conf < min_avg_confidence as f64 {
return false;
}
}
}
}
}
}
// Check min frames with objects
if let Some(min_frames) = criteria.min_frames_with_objects {
if let Some(stats) = &chunk.visual_stats {
if let Some(frames_with_objects) = stats.get("frames_with_objects") {
if let Some(count) = frames_with_objects.as_u64() {
if count < min_frames as u64 {
return false;
}
}
}
}
}
// Check min unique classes
if let Some(min_unique_classes) = criteria.min_unique_classes {
if let Some(content) = &chunk.content.as_object() {
if let Some(metadata) = content.get("metadata") {
if let Some(unique_classes) = metadata.get("unique_classes") {
if let Some(classes) = unique_classes.as_array() {
if (classes.len() as u32) < min_unique_classes {
return false;
}
}
}
}
}
}
// Check required classes
if !criteria.required_classes.is_empty() {
if let Some(content) = &chunk.content.as_object() {
if let Some(keyframe_objects) = content.get("keyframe_objects") {
if let Some(objects) = keyframe_objects.as_array() {
let mut found_all = true;
for required_class in &criteria.required_classes {
let mut found = false;
for obj in objects {
if let Some(class_name) = obj.get("class_name") {
if let Some(class_str) = class_name.as_str() {
if class_str == required_class {
found = true;
break;
}
}
}
}
if !found {
found_all = false;
break;
}
}
if !found_all {
return false;
}
}
}
}
}
// Check class counts
if !criteria.class_counts.is_empty() {
if let Some(content) = &chunk.content.as_object() {
if let Some(metadata) = content.get("metadata") {
if let Some(object_counts) = metadata.get("object_counts") {
for (class, (min, max)) in &criteria.class_counts {
if let Some(count_value) = object_counts.get(class) {
if let Some(count) = count_value.as_u64() {
if *min > 0 && count < *min as u64 {
return false;
}
if *max < u32::MAX && count > *max as u64 {
return false;
}
}
} else if *min > 0 {
return false;
}
}
} else if criteria.class_counts.values().any(|(min, _)| *min > 0) {
return false;
}
}
}
}
// Check time range
if let Some((start_time, end_time)) = criteria.time_range {
// Calculate chunk time from frames
let chunk_start_time = chunk.start_frame as f64 / chunk.fps;
let chunk_end_time = chunk.end_frame as f64 / chunk.fps;
if chunk_start_time < start_time || chunk_end_time > end_time {
return false;
}
}
true
})
.collect();
Ok(filtered_chunks)
}
/// Get all visual chunks for a video UUID
async fn get_visual_chunks_by_uuid(db: &PostgresDb, uuid: &str) -> Result<Vec<Chunk>> {
let chunk_table = schema::table_name("chunk");
let sql = format!(
"SELECT file_id, file_uuid, chunk_id, chunk_type, fps, start_frame, end_frame, text_content, content, metadata, vector_id, visual_stats FROM {} WHERE file_uuid = '{}' AND chunk_type = 'visual' ORDER BY start_frame ASC",
chunk_table, uuid.replace('\'', "''")
);
let rows: Vec<(
i32, // file_id
String, // uuid
String, // chunk_id
String, // chunk_type
f64, // fps
i64, // start_frame
i64, // end_frame
Option<String>, // text_content
Value, // content
Option<Value>, // metadata
Option<String>, // vector_id
Option<Value>, // visual_stats
)> = sqlx::query_as(&sql).fetch_all(db.pool()).await?;
let mut chunks = Vec::new();
for row in rows {
let chunk_type = match row.3.as_str() {
"visual" => ChunkType::Visual,
"sentence" => ChunkType::Sentence,
"time_based" => ChunkType::TimeBased,
"cut" => ChunkType::Cut,
"trace" => ChunkType::Trace,
"story" => ChunkType::Story,
_ => ChunkType::TimeBased,
};
// Calculate frame_count
let frame_count = (row.6 - row.5) as i32;
chunks.push(Chunk {
file_id: row.0,
uuid: row.1,
chunk_id: row.2,
chunk_type,
rule: ChunkRule::Rule2, // Visual chunks use Rule2
fps: row.4,
start_frame: row.5,
end_frame: row.6,
text_content: row.7,
content: row.8,
metadata: row.9,
vector_id: row.10,
frame_count,
pre_chunk_ids: Vec::new(),
parent_chunk_id: None,
child_chunk_ids: Vec::new(),
visual_stats: row.11,
});
}
Ok(chunks)
}
/// Search visual chunks by object class
pub async fn search_visual_chunks_by_class(
db: &PostgresDb,
uuid: &str,
object_class: &str,
min_count: Option<u32>,
max_count: Option<u32>,
) -> Result<Vec<Chunk>> {
let all_chunks = get_visual_chunks_by_uuid(db, uuid).await?;
let filtered_chunks: Vec<Chunk> = all_chunks
.into_iter()
.filter(|chunk| {
// Check if chunk contains the object class
let mut contains_class = false;
if let Some(content) = &chunk.content.as_object() {
if let Some(keyframe_objects) = content.get("keyframe_objects") {
if let Some(objects) = keyframe_objects.as_array() {
for obj in objects {
if let Some(class_name) = obj.get("class_name") {
if let Some(class_str) = class_name.as_str() {
if class_str == object_class {
contains_class = true;
break;
}
}
}
}
}
}
}
if !contains_class {
return false;
}
// Check count in visual_stats
if let Some(stats) = &chunk.visual_stats {
if let Some(count) = stats.get(object_class) {
if let Some(c) = count.as_u64() {
if let Some(min) = min_count {
if c < min as u64 {
return false;
}
}
if let Some(max) = max_count {
if c > max as u64 {
return false;
}
}
}
}
}
true
})
.collect();
Ok(filtered_chunks)
}
/// Search visual chunks by spatial density
pub async fn search_visual_chunks_by_density(
db: &PostgresDb,
uuid: &str,
min_density: f32,
max_density: Option<f32>,
) -> Result<Vec<Chunk>> {
let all_chunks = get_visual_chunks_by_uuid(db, uuid).await?;
let filtered_chunks: Vec<Chunk> = all_chunks
.into_iter()
.filter(|chunk| {
if let Some(content) = &chunk.content.as_object() {
if let Some(metadata) = content.get("metadata") {
if let Some(density_value) = metadata.get("spatial_density") {
if let Some(density) = density_value.as_f64() {
if density < min_density as f64 {
return false;
}
if let Some(max_dens) = max_density {
if density > max_dens as f64 {
return false;
}
}
return true;
}
}
}
}
false
})
.collect();
Ok(filtered_chunks)
}
/// Find chunks containing specific object combinations
pub async fn search_visual_chunks_by_combination(
db: &PostgresDb,
uuid: &str,
combination: &[(&str, u32)],
) -> Result<Vec<Chunk>> {
let all_chunks = get_visual_chunks_by_uuid(db, uuid).await?;
let filtered_chunks: Vec<Chunk> = all_chunks
.into_iter()
.filter(|chunk| {
// Check if all required combinations are present
for (object_class, min_count) in combination {
let mut found = false;
if let Some(stats) = &chunk.visual_stats {
if let Some(object_counts) = stats.get("object_counts") {
if let Some(count_value) = object_counts.get(*object_class) {
if let Some(count) = count_value.as_u64() {
if count >= *min_count as u64 {
found = true;
}
}
}
}
}
if !found {
return false;
}
}
true
})
.collect();
Ok(filtered_chunks)
}
/// Get visual chunk statistics
pub async fn get_visual_chunk_statistics(
db: &PostgresDb,
uuid: &str,
) -> Result<HashMap<String, Value>> {
let chunk_table = schema::table_name("chunk");
let sql = format!(
"SELECT
COUNT(*) as total_chunks,
AVG((content->'metadata'->>'avg_confidence')::float) as avg_confidence,
MIN((content->'metadata'->>'avg_confidence')::float) as min_confidence,
MAX((content->'metadata'->>'avg_confidence')::float) as max_confidence,
SUM((content->'metadata'->>'object_count')::int) as total_objects,
AVG((content->'metadata'->>'spatial_density')::float) as avg_density
FROM {}
WHERE file_uuid = '{}'
AND chunk_type = 'visual'",
chunk_table,
uuid.replace('\'', "''")
);
let row: (
i64,
Option<f64>,
Option<f64>,
Option<f64>,
Option<i64>,
Option<f64>,
) = sqlx::query_as(&sql).fetch_one(db.pool()).await?;
let mut stats = HashMap::new();
stats.insert("total_chunks".to_string(), Value::from(row.0));
stats.insert(
"avg_confidence".to_string(),
Value::from(row.1.unwrap_or(0.0)),
);
stats.insert(
"min_confidence".to_string(),
Value::from(row.2.unwrap_or(0.0)),
);
stats.insert(
"max_confidence".to_string(),
Value::from(row.3.unwrap_or(0.0)),
);
stats.insert("total_objects".to_string(), Value::from(row.4.unwrap_or(0)));
stats.insert("avg_density".to_string(), Value::from(row.5.unwrap_or(0.0)));
Ok(stats)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_visual_chunk_search_criteria_default() {
let criteria = VisualChunkSearchCriteria::default();
assert_eq!(criteria.min_avg_confidence, None);
assert_eq!(criteria.min_frames_with_objects, None);
assert_eq!(criteria.min_unique_classes, None);
assert!(criteria.required_classes.is_empty());
assert!(criteria.class_counts.is_empty());
assert_eq!(criteria.time_range, None);
}
#[test]
fn test_visual_chunk_search_criteria_with_values() {
let mut criteria = VisualChunkSearchCriteria::default();
criteria.min_avg_confidence = Some(0.8);
criteria.min_frames_with_objects = Some(10);
criteria.min_unique_classes = Some(3);
criteria.required_classes = vec!["person".to_string(), "car".to_string()];
criteria.time_range = Some((0.0, 60.0));
assert_eq!(criteria.min_avg_confidence, Some(0.8));
assert_eq!(criteria.min_frames_with_objects, Some(10));
assert_eq!(criteria.min_unique_classes, Some(3));
assert_eq!(criteria.required_classes.len(), 2);
assert_eq!(criteria.time_range, Some((0.0, 60.0)));
}
#[test]
fn test_visual_chunk_search_criteria_serialization() {
let criteria = VisualChunkSearchCriteria {
min_avg_confidence: Some(0.85),
min_frames_with_objects: Some(5),
min_unique_classes: Some(2),
required_classes: vec!["person".to_string()],
class_counts: HashMap::new(),
time_range: Some((10.0, 30.0)),
};
let json = serde_json::to_string(&criteria).unwrap();
assert!(json.contains("min_avg_confidence"));
assert!(json.contains("required_classes"));
let deserialized: VisualChunkSearchCriteria = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.min_avg_confidence, Some(0.85));
assert_eq!(deserialized.required_classes.len(), 1);
}
#[test]
fn test_visual_chunk_search_criteria_with_class_counts() {
let mut criteria = VisualChunkSearchCriteria::default();
criteria.class_counts.insert("person".to_string(), (5, 20));
criteria.class_counts.insert("car".to_string(), (1, 10));
assert_eq!(criteria.class_counts.len(), 2);
assert_eq!(criteria.class_counts.get("person"), Some(&(5, 20)));
assert_eq!(criteria.class_counts.get("car"), Some(&(1, 10)));
}
#[test]
fn test_chunk_type_conversion() {
// Test chunk type string to enum conversion logic
let test_cases = vec![
("visual", ChunkType::Visual),
("sentence", ChunkType::Sentence),
("time_based", ChunkType::TimeBased),
("cut", ChunkType::Cut),
("trace", ChunkType::Trace),
("story", ChunkType::Story),
("unknown", ChunkType::TimeBased), // Default fallback
];
for (input, expected) in test_cases {
let chunk_type = match input {
"visual" => ChunkType::Visual,
"sentence" => ChunkType::Sentence,
"time_based" => ChunkType::TimeBased,
"cut" => ChunkType::Cut,
"trace" => ChunkType::Trace,
"story" => ChunkType::Story,
_ => ChunkType::TimeBased,
};
assert_eq!(chunk_type, expected);
}
}
}

217
src/api/visual_search.rs
View File

@@ -1,217 +0,0 @@
use axum::{extract::State, http::StatusCode, response::Json, routing::post, Router};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use super::types::AppState;
use super::visual_chunk_search;
use crate::core::cache::keys;
use crate::core::chunk::types::Chunk;
use crate::core::db::{Database, PostgresDb};
fn generate_visual_search_hash(
uuid: &str,
criteria: &visual_chunk_search::VisualChunkSearchCriteria,
) -> String {
let data = serde_json::json!({
"uuid": uuid,
"criteria": criteria,
});
let mut hasher = Sha256::new();
hasher.update(data.to_string().as_bytes());
format!("{:x}", hasher.finalize())[..16].to_string()
}
#[derive(Debug, Deserialize)]
struct VisualChunkSearchRequest {
file_uuid: String,
criteria: visual_chunk_search::VisualChunkSearchCriteria,
}
#[derive(Debug, Serialize)]
struct VisualChunkSearchResponse {
chunks: Vec<Chunk>,
total: usize,
}
async fn search_visual_chunks(
State(state): State<AppState>,
Json(req): Json<VisualChunkSearchRequest>,
) -> Result<Json<VisualChunkSearchResponse>, StatusCode> {
let criteria_hash = generate_visual_search_hash(&req.file_uuid, &req.criteria);
let cache_key = keys::visual_search(&req.file_uuid, &criteria_hash);
let ttl = state.mongo_cache.ttl_visual_search();
let chunks = state
.mongo_cache
.get_or_fetch(&cache_key, ttl, keys::CATEGORY_VISUAL_SEARCH, || async {
let db = PostgresDb::init()
.await
.map_err(|e| anyhow::anyhow!("PG init failed: {}", e))?;
visual_chunk_search::search_visual_chunks(&db, &req.file_uuid, &req.criteria)
.await
.map_err(|e| anyhow::anyhow!("Visual search failed: {}", e))
})
.await
.map_err(|e| {
tracing::error!("Visual chunk search failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(VisualChunkSearchResponse {
total: chunks.len(),
chunks,
}))
}
#[derive(Debug, Deserialize)]
struct VisualChunkSearchByClassRequest {
uuid: String,
object_class: String,
min_count: Option<u32>,
max_count: Option<u32>,
}
#[derive(Debug, Deserialize)]
struct VisualChunkSearchByDensityRequest {
uuid: String,
min_density: f32,
max_density: Option<f32>,
}
#[derive(Debug, Deserialize)]
struct VisualChunkStatsRequest {
uuid: String,
}
async fn search_visual_chunks_by_class(
State(state): State<AppState>,
Json(req): Json<VisualChunkSearchByClassRequest>,
) -> Result<Json<VisualChunkSearchResponse>, StatusCode> {
let db = PostgresDb::init()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let chunks = visual_chunk_search::search_visual_chunks_by_class(
&db,
&req.uuid,
&req.object_class,
req.min_count,
req.max_count,
)
.await
.map_err(|e| {
tracing::error!("Visual chunk search by class failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(VisualChunkSearchResponse {
total: chunks.len(),
chunks,
}))
}
async fn search_visual_chunks_by_density(
State(state): State<AppState>,
Json(req): Json<VisualChunkSearchByDensityRequest>,
) -> Result<Json<VisualChunkSearchResponse>, StatusCode> {
let db = PostgresDb::init()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let chunks = visual_chunk_search::search_visual_chunks_by_density(
&db,
&req.uuid,
req.min_density,
req.max_density,
)
.await
.map_err(|e| {
tracing::error!("Visual chunk search by density failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(VisualChunkSearchResponse {
total: chunks.len(),
chunks,
}))
}
#[derive(Debug, Serialize)]
struct VisualChunkStatsResponse {
uuid: String,
stats: std::collections::HashMap<String, serde_json::Value>,
}
async fn get_visual_chunk_stats(
State(state): State<AppState>,
Json(req): Json<VisualChunkStatsRequest>,
) -> Result<Json<VisualChunkStatsResponse>, StatusCode> {
let db = PostgresDb::init()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let stats = visual_chunk_search::get_visual_chunk_statistics(&db, &req.uuid)
.await
.map_err(|e| {
tracing::error!("Get visual chunk stats failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(VisualChunkStatsResponse {
uuid: req.uuid,
stats,
}))
}
#[derive(Debug, Deserialize)]
struct VisualChunkSearchByCombinationRequest {
uuid: String,
combination: Vec<(String, u32)>,
}
async fn search_visual_chunks_by_combination(
State(state): State<AppState>,
Json(req): Json<VisualChunkSearchByCombinationRequest>,
) -> Result<Json<VisualChunkSearchResponse>, StatusCode> {
let db = PostgresDb::init()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let combination: Vec<(&str, u32)> = req
.combination
.iter()
.map(|(c, n)| (c.as_str(), *n))
.collect();
let chunks =
visual_chunk_search::search_visual_chunks_by_combination(&db, &req.uuid, &combination)
.await
.map_err(|e| {
tracing::error!("Visual chunk search by combination failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(VisualChunkSearchResponse {
total: chunks.len(),
chunks,
}))
}
pub fn visual_search_routes() -> Router<AppState> {
Router::new()
.route("/api/v1/search/visual", post(search_visual_chunks))
.route(
"/api/v1/search/visual/class",
post(search_visual_chunks_by_class),
)
.route(
"/api/v1/search/visual/density",
post(search_visual_chunks_by_density),
)
.route("/api/v1/search/visual/stats", post(get_visual_chunk_stats))
.route(
"/api/v1/search/visual/combination",
post(search_visual_chunks_by_combination),
)
}