diff --git a/.env.development b/.env.development index fd5b6b2..3251d43 100644 --- a/.env.development +++ b/.env.development @@ -71,4 +71,11 @@ REDIS_CACHE_TTL_VIDEO_META=3600 # TMDb Integration (probe phase - auto-create identities from movie metadata) TMDB_API_KEY=e9cde52197f6f8df4d9db99da93db1fb -MOMENTRY_TMDB_PROBE_ENABLED=true \ No newline at end of file +MOMENTRY_TMDB_PROBE_ENABLED=true +# LLM for 5W1H summary (points to M5 Gemma4) +MOMENTRY_LLM_SUMMARY_URL=http://localhost:8082/v1/chat/completions +MOMENTRY_LLM_SUMMARY_MODEL=google_gemma-4-26B-A4B-it-Q5_K_M.gguf +MOMENTRY_LLM_SUMMARY_ENABLED=true + +# Embedding (ANE CoreML server) +MOMENTRY_EMBED_URL=http://localhost:11436 diff --git a/docs_v1.0/M4_workspace/2026-05-07_M5_release_ready.md b/docs_v1.0/M4_workspace/2026-05-07_M5_release_ready.md new file mode 100644 index 0000000..1c8966b --- /dev/null +++ b/docs_v1.0/M4_workspace/2026-05-07_M5_release_ready.md @@ -0,0 +1,32 @@ +# M5 Release Ready + +## 可 sync + +M5 已完成 v1.0.0 release candidate 的所有程式碼變更，所有服務正常運行。 + +```bash +# 1. DB +scp accusys@192.168.110.201:/tmp/momentry_3abeee81.sql /tmp/ +psql -U accusys -d momentry -c "DROP SCHEMA IF EXISTS dev CASCADE; CREATE SCHEMA dev;" +psql -U accusys -d momentry -f /tmp/momentry_3abeee81.sql + +# 2. Binary（可選，M4 通常不需跑 playground） +scp accusys@192.168.110.201:/Users/accusys/momentry_core_0.1/target/debug/momentry_playground /tmp/ + +# 3. Git +cd /path/to/momentry_docs +git pull +``` + +## 5W1H+ 仍在背景 + +目前完成 8/721 scene，預計還需要一段時間。完成後會自動 vectorize。屆時再做一次增量 sync。 + +## 本次 Release 變更 + +- 5W1H+ Agent（遞迴式摘要、face/speaker 資訊整合） +- Identity Agent 自動觸發 +- EmbeddingGemma 300M（768D，多語） +- Pipeline bug fixes + +詳細變更請見 `git log`。 diff --git a/src/api/five_w1h_agent_api.rs b/src/api/five_w1h_agent_api.rs index 26f201b..2b5e637 100644 --- a/src/api/five_w1h_agent_api.rs +++ b/src/api/five_w1h_agent_api.rs @@ -10,7 +10,10 @@ use serde::{Deserialize, Serialize}; use sqlx::Row; use crate::api::server::AppState; -use crate::core::db::PostgresDb; +use crate::core::db::qdrant_db::QdrantDb; +use crate::core::db::schema; +use crate::core::db::{PostgresDb, VectorPayload}; +use crate::core::embedding::Embedder; pub fn five_w1h_agent_routes() -> Router { Router::new() @@ -19,56 +22,24 @@ pub fn five_w1h_agent_routes() -> Router { .route("/api/v1/agents/5w1h/status", get(get_5w1h_status)) } +// ── Data Structures ── + #[derive(Debug, Deserialize)] pub struct Analyze5W1HRequest { pub file_uuid: String, - pub scene_group_size: Option, - pub model: Option, } #[derive(Debug, Serialize)] pub struct Analyze5W1HResponse { pub success: bool, pub file_uuid: String, - pub summaries: Vec, - pub processing_status: FiveW1HProcessingStatus, -} - -#[derive(Debug, Serialize)] -pub struct SummaryChunk { - pub chunk_id: String, - pub summary: String, - pub analysis_5w1h: FiveW1HAnalysis, - pub start_frame: i64, - pub end_frame: i64, - pub start_time: f64, - pub end_time: f64, - pub fps: f64, - pub scene_count: usize, -} - -#[derive(Debug, Serialize)] -pub struct FiveW1HAnalysis { - pub who: Vec, - pub what: Vec, - pub location: Vec, - pub when: String, - pub why: String, - pub how: String, -} - -#[derive(Debug, Serialize)] -pub struct FiveW1HProcessingStatus { - pub status: String, - pub scenes_processed: i32, - pub scenes_total: i32, - pub progress_pct: f64, + pub scenes_processed: usize, + pub scenes_total: usize, } #[derive(Debug, Deserialize)] pub struct BatchAnalyze5W1HRequest { pub file_uuids: Vec, - pub scene_group_size: Option, } #[derive(Debug, Serialize)] @@ -84,64 +55,410 @@ pub struct BatchJobStatus { pub message: String, } +#[derive(Debug, Clone)] +struct CutScene { + chunk_id: String, + chunk_index: i32, + start_frame: i64, + end_frame: i64, + fps: f64, + start_time: f64, + end_time: f64, + content: serde_json::Value, + metadata: serde_json::Value, +} + +#[derive(Debug, Clone)] +struct SentenceChunk { + chunk_id: String, + text: String, + start_time: f64, + end_time: f64, + start_frame: i64, + end_frame: i64, + content: serde_json::Value, +} + +#[derive(Debug)] +struct ChildSummary { + chunk_id: String, + enhanced: String, + five_w1h: serde_json::Value, +} + +#[derive(Debug)] +struct SceneSummaryResult { + parent_summary: String, + five_w1h: serde_json::Value, + child_summaries: Vec, +} + +// ── LLM Endpoint ── + +fn llm_base_url() -> String { + std::env::var("MOMENTRY_LLM_SUMMARY_URL") + .unwrap_or_else(|_| "http://localhost:8081/v1/chat/completions".to_string()) +} + +fn llm_model() -> String { + std::env::var("MOMENTRY_LLM_SUMMARY_MODEL") + .unwrap_or_else(|_| "gemma-4-31B-it-Q5_K_M.gguf".to_string()) +} + +// ── Data Fetching ── + +async fn fetch_cut_scenes(db: &PostgresDb, file_uuid: &str) -> anyhow::Result> { + let table = schema::table_name("chunks"); + sqlx::query_as::<_, (String, i32, i64, i64, f64, f64, f64, serde_json::Value, serde_json::Value)>(&format!( + r#"SELECT chunk_id, chunk_index, start_frame, end_frame, fps, start_time, end_time, content, metadata + FROM {} WHERE file_uuid = $1 AND chunk_type = 'cut' ORDER BY start_frame"#, table + )) + .bind(file_uuid) + .fetch_all(db.pool()).await? + .into_iter().map(|r| Ok(CutScene { + chunk_id: r.0, chunk_index: r.1, start_frame: r.2, end_frame: r.3, + fps: r.4, start_time: r.5, end_time: r.6, content: r.7, metadata: r.8, + })).collect() +} + +async fn fetch_sentences_in_scene(db: &PostgresDb, file_uuid: &str, cut: &CutScene) -> anyhow::Result> { + let table = schema::table_name("chunks"); + sqlx::query_as::<_, (String, String, f64, f64, i64, i64, serde_json::Value)>(&format!( + r#"SELECT chunk_id, COALESCE(text_content,''), start_time, end_time, start_frame, end_frame, content + FROM {} WHERE file_uuid = $1 AND chunk_type = 'sentence' + AND start_time >= $2 AND end_time <= $3 ORDER BY start_time"#, table + )) + .bind(file_uuid).bind(cut.start_time).bind(cut.end_time) + .fetch_all(db.pool()).await? + .into_iter().map(|r| Ok(SentenceChunk { + chunk_id: r.0, text: r.1, start_time: r.2, end_time: r.3, + start_frame: r.4, end_frame: r.5, content: r.6, + })).collect() +} + +/// Fetch actor names present in this scene from face_detections + identity_bindings + identities +async fn fetch_identity_names_for_scene(db: &PostgresDb, file_uuid: &str, cut: &CutScene) -> anyhow::Result> { + let fd_table = schema::table_name("face_detections"); + let ib_table = schema::table_name("identity_bindings"); + let id_table = schema::table_name("identities"); + let rows = sqlx::query_scalar::<_, String>(&format!( + r#"SELECT DISTINCT i.name + FROM {} fd + JOIN {} ib ON ib.identity_value = fd.trace_id::text AND ib.identity_type = 'trace' + JOIN {} i ON i.id = ib.identity_id + WHERE fd.file_uuid = $1 AND fd.frame_number >= $2 AND fd.frame_number <= $3 + AND fd.trace_id IS NOT NULL + ORDER BY i.name"#, fd_table, ib_table, id_table + )) + .bind(file_uuid).bind(cut.start_frame).bind(cut.end_frame) + .fetch_all(db.pool()).await?; + Ok(rows) +} + +/// Fetch YOLO object labels detected in this scene from pre_chunks +async fn fetch_yolo_objects_for_scene(db: &PostgresDb, file_uuid: &str, cut: &CutScene) -> anyhow::Result> { + let table = schema::table_name("pre_chunks"); + let rows = sqlx::query_scalar::<_, String>(&format!( + r#"SELECT DISTINCT data->>'label' + FROM {} WHERE file_uuid = $1 AND processor_type = 'yolo' + AND frame_number >= $2 AND frame_number <= $3 + AND data->>'label' IS NOT NULL + ORDER BY data->>'label'"#, table + )) + .bind(file_uuid).bind(cut.start_frame).bind(cut.end_frame) + .fetch_all(db.pool()).await?; + Ok(rows) +} + +/// Fetch active speakers + their actor names for a scene's frame range +/// Uses identity_bindings to map SPEAKER_X to actor names +async fn fetch_speakers_for_scene(db: &PostgresDb, file_uuid: &str, cut: &CutScene) -> anyhow::Result> { + let pc_table = schema::table_name("pre_chunks"); + let speakers = sqlx::query_scalar::<_, String>(&format!( + r#"SELECT DISTINCT data->>'speaker_id' + FROM {} WHERE file_uuid = $1 AND processor_type = 'asrx' + AND data->>'speaker_id' IS NOT NULL + AND start_frame <= $3 AND end_frame >= $2 + ORDER BY data->>'speaker_id'"#, pc_table + )) + .bind(file_uuid).bind(cut.start_frame).bind(cut.end_frame) + .fetch_all(db.pool()).await?; + + if speakers.is_empty() { return Ok(vec![]); } + + // Map speaker_ids to actor names via identity_bindings + let ib_table = schema::table_name("identity_bindings"); + let id_table = schema::table_name("identities"); + let mut result = Vec::new(); + for spk in &speakers { + let name: Option = sqlx::query_scalar(&format!( + r#"SELECT i.name FROM {} ib JOIN {} i ON i.id = ib.identity_id + WHERE ib.identity_type = 'speaker' AND ib.identity_value = $1 AND i.name IS NOT NULL + LIMIT 1"#, ib_table, id_table + )) + .bind(spk) + .fetch_optional(db.pool()).await?; + match name { + Some(n) => result.push(format!("{} ({})", spk, n)), + None => result.push(spk.clone()), + } + } + Ok(result) +} + +/// Fetch trace IDs with identity names for a scene's frame range +async fn fetch_trace_info(db: &PostgresDb, file_uuid: &str, cut: &CutScene) -> anyhow::Result> { + let fd_table = schema::table_name("face_detections"); + let ib_table = schema::table_name("identity_bindings"); + let id_table = schema::table_name("identities"); + let rows = sqlx::query_as::<_, (i32, Option)>(&format!( + r#"SELECT DISTINCT fd.trace_id, i.name + FROM {} fd + LEFT JOIN {} ib ON ib.identity_value = fd.trace_id::text AND ib.identity_type = 'trace' + LEFT JOIN {} i ON i.id = ib.identity_id + WHERE fd.file_uuid = $1 AND fd.frame_number >= $2 AND fd.frame_number <= $3 + AND fd.trace_id IS NOT NULL + ORDER BY fd.trace_id"#, fd_table, ib_table, id_table + )) + .bind(file_uuid).bind(cut.start_frame).bind(cut.end_frame) + .fetch_all(db.pool()).await?; + + Ok(rows.iter().map(|(trace, name)| { + if let Some(n) = name { + format!("trace_{} ({})", trace, n) + } else { + format!("trace_{}", trace) + } + }).collect()) +} + +// ── LLM Prompt (Embedding-Optimized) ── + +async fn summarize_one_scene( + db: &PostgresDb, + file_uuid: &str, + cut: &CutScene, + sentences: &[SentenceChunk], + prev_context: &str, +) -> anyhow::Result { + if sentences.is_empty() { + return Ok(SceneSummaryResult { + parent_summary: String::new(), five_w1h: serde_json::Value::Null, child_summaries: vec![], + }); + } + + let faces = fetch_identity_names_for_scene(db, file_uuid, cut).await.unwrap_or_default(); + let objects = fetch_yolo_objects_for_scene(db, file_uuid, cut).await.unwrap_or_default(); + let traces = fetch_trace_info(db, file_uuid, cut).await.unwrap_or_default(); + let speakers = fetch_speakers_for_scene(db, file_uuid, cut).await.unwrap_or_default(); + + let mut dialogue = String::new(); + for (i, s) in sentences.iter().enumerate() { + let t = s.text.trim(); + if !t.is_empty() { dialogue.push_str(&format!("[{}] {}\n", i + 1, t)); } + } + + let story_so_far = if prev_context.is_empty() { + String::new() + } else { + format!("\nStory so far (previous scenes):\n{}\n", prev_context) + }; + + let prompt = format!( + r#"Analyze this movie scene and produce a structured summary. Be specific — quote actual dialogue. Avoid template phrases like "within the established dramatic setting." + +Scene time: {:.0}s–{:.0}s + +Dialogue: +{}Actors: {} +Objects: {} +Face traces: {} +Speakers: {} +{} +Output EXACTLY this JSON format: +{{ + "scene_summary": "5 flowing sentences: who+what+where+when+why+how. Quote actual lines.", + "5w1h": {{ + "who": "1 sentence with actor/character name", + "what": "1 sentence describing the action, quote the line", + "where": "1 sentence about setting", + "when": "1 sentence about timing in story", + "why": "1 sentence explaining why this moment matters", + "how": "1 sentence about delivery, emotion, tone" + }}, + "sentences": [ + {{ + "index": 1, + "who": "1 sentence", + "what": "1 sentence referencing the actual line", + "where": "1 sentence", + "when": "1 sentence", + "why": "1 sentence why this is said", + "how": "1 sentence describing delivery", + "enhanced": "1 sentence with actual dialogue, self-contained for search" + }} + ] +}} + +Rules: +- scene_summary: 5 sentences, natural paragraph. Use quotes. No template phrases. +- Each 5w1h field: exactly 1 sentence. Specific details. Character names. Quotes. +- Each sentence.enhanced: self-contained for search, include actual spoken words. +- Return ONLY valid JSON. No markdown. +- A short scene with 1-2 lines should have a short summary."#, + cut.start_time, cut.end_time, dialogue, faces.join(", "), objects.join(", "), traces.join(", "), speakers.join(", "), story_so_far, + ); + + let body = serde_json::json!({ + "model": llm_model(), + "messages": [ + {"role": "system", "content": "You output JSON only. Be specific. Quote actual dialogue. Avoid template phrases."}, + {"role": "user", "content": prompt} + ], + "temperature": 0.1, + "max_tokens": 2048, + "stream": false + }); + + let client = Client::new(); + let resp = client.post(llm_base_url()).json(&body) + .timeout(std::time::Duration::from_secs(180)) + .send().await? + .json::().await?; + + let content = resp["choices"][0]["message"]["content"].as_str().unwrap_or("{}"); + // Strip markdown code fences if present + let cleaned = content + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim(); + let parsed: serde_json::Value = serde_json::from_str(cleaned).unwrap_or(serde_json::Value::Null); + + let parent_summary = parsed["scene_summary"].as_str().unwrap_or("").to_string(); + let five_w1h = parsed.get("5w1h").cloned().unwrap_or(serde_json::Value::Null); + let mut child_summaries = Vec::new(); + + if let Some(arr) = parsed["sentences"].as_array() { + for entry in arr { + let idx = entry["index"].as_u64().unwrap_or(0).saturating_sub(1) as usize; + if let Some(enhanced) = entry["enhanced"].as_str() { + if idx < sentences.len() { + let child_5w1h = serde_json::json!({ + "who": entry["who"].as_str().unwrap_or(""), + "what": entry["what"].as_str().unwrap_or(""), + "where": entry["where"].as_str().unwrap_or(""), + "when": entry["when"].as_str().unwrap_or(""), + "why": entry["why"].as_str().unwrap_or(""), + "how": entry["how"].as_str().unwrap_or(""), + }); + child_summaries.push(ChildSummary { + chunk_id: sentences[idx].chunk_id.clone(), + enhanced: enhanced.to_string(), + five_w1h: child_5w1h, + }); + } + } + } + } + + // Fallback + if child_summaries.is_empty() && !parent_summary.is_empty() { + for s in sentences { + let text = s.text.trim(); + if !text.is_empty() { + child_summaries.push(ChildSummary { + chunk_id: s.chunk_id.clone(), + enhanced: format!("{} Scene: {}", text, parent_summary), + five_w1h: serde_json::Value::Null, + }); + } + } + } + + Ok(SceneSummaryResult { parent_summary, five_w1h, child_summaries }) +} + +// ── DB Storage ── + +async fn store_parent_summary( + db: &PostgresDb, cut_chunk_id: &str, file_uuid: &str, + summary: &str, five_w1h: &serde_json::Value, sentences: &[SentenceChunk], +) -> anyhow::Result<()> { + let table = schema::table_name("chunks"); + let meta = serde_json::json!({ + "5w1h": five_w1h, + "sentence_ids": sentences.iter().map(|s| s.chunk_id.clone()).collect::>(), + "sentence_count": sentences.len(), + }); + sqlx::query(&format!( + r#"UPDATE {} SET summary_text = $1, metadata = metadata || $2::jsonb + WHERE chunk_id = $3 AND file_uuid = $4"#, table + )) + .bind(summary).bind(&meta).bind(cut_chunk_id).bind(file_uuid) + .execute(db.pool()).await?; + Ok(()) +} + +async fn store_child_summaries( + db: &PostgresDb, file_uuid: &str, children: &[ChildSummary], +) -> anyhow::Result<()> { + let table = schema::table_name("chunks"); + for c in children { + let text = c.enhanced.trim(); + if text.is_empty() || text.len() < 10 { continue; } + // Update text_content (for embedding) + merge 5w1h into content + let merge = serde_json::json!({ "5w1h": c.five_w1h }); + sqlx::query(&format!( + r#"UPDATE {} SET text_content = $1, content = content || $2::jsonb, embedding = NULL + WHERE chunk_id = $3 AND file_uuid = $4"#, table + )) + .bind(text).bind(&merge).bind(&c.chunk_id).bind(file_uuid) + .execute(db.pool()).await?; + } + Ok(()) +} + +// ── API Handlers ── + async fn analyze_5w1h( State(state): State, Json(req): Json, ) -> Result, (StatusCode, String)> { let db = PostgresDb::from_pool(state.db.pool().clone()); - let scene_group_size = req.scene_group_size.unwrap_or(7); - let model = req.model.unwrap_or_else(|| "gemma4:latest".to_string()); - - let rule3_chunks = fetch_rule3_chunks(&db, &req.file_uuid) - .await + let cuts = fetch_cut_scenes(&db, &req.file_uuid).await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - if rule3_chunks.is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "No Rule 3 chunks found for this video".to_string(), - )); + let total = cuts.len(); + let mut processed = 0usize; + let mut prev_context: Vec = Vec::new(); + + for cut in &cuts { + let sentences = fetch_sentences_in_scene(&db, &req.file_uuid, cut).await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + if sentences.is_empty() { continue; } + + let context = prev_context.join("\n"); + let result = summarize_one_scene(&db, &req.file_uuid, cut, &sentences, &context).await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if let Err(e) = store_parent_summary(&db, &cut.chunk_id, &req.file_uuid, &result.parent_summary, &result.five_w1h, &sentences).await { + tracing::error!("[5W1H] parent: {}", e); + } + if let Err(e) = store_child_summaries(&db, &req.file_uuid, &result.child_summaries).await { + tracing::error!("[5W1H] child: {}", e); + } + + if !result.parent_summary.is_empty() { + prev_context.push(format!("Scene {} (t={:.0}s): {}", cut.chunk_index, cut.start_time, result.parent_summary)); + } + processed += 1; } - let scenes_total = rule3_chunks.len() as i32; - - update_agent_status(&db, &req.file_uuid, "running", 0, scenes_total, 0.0) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - let summaries = - process_scene_groups(&db, &req.file_uuid, &rule3_chunks, scene_group_size, &model) - .await - .map_err(|e| { - tracing::error!("Failed to process scene groups: {}", e); - (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) - })?; - - let scenes_processed = rule3_chunks.len() as i32; - let progress_pct = 100.0; - - update_agent_status( - &db, - &req.file_uuid, - "completed", - scenes_processed, - scenes_total, - progress_pct, - ) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - Ok(Json(Analyze5W1HResponse { - success: true, - file_uuid: req.file_uuid, - summaries, - processing_status: FiveW1HProcessingStatus { - status: "completed".to_string(), - scenes_processed, - scenes_total, - progress_pct, - }, + success: true, file_uuid: req.file_uuid, + scenes_processed: processed, scenes_total: total, })) } @@ -149,525 +466,139 @@ async fn batch_analyze_5w1h( State(state): State, Json(req): Json, ) -> Result, (StatusCode, String)> { - let scene_group_size = req.scene_group_size.unwrap_or(7); - let jobs: Vec = req - .file_uuids - .iter() - .map(|uuid| { - let uuid_clone = uuid.clone(); - let db_clone = PostgresDb::from_pool(state.db.pool().clone()); - let group_size = scene_group_size; + let db = PostgresDb::from_pool(state.db.pool().clone()); + let mut jobs = Vec::new(); - tokio::spawn(async move { - let _ = process_single_video_5w1h(&db_clone, &uuid_clone, group_size).await; - }); + for uuid in &req.file_uuids { + let cuts = fetch_cut_scenes(&db, uuid).await.unwrap_or_default(); + let total = cuts.len(); + let mut processed = 0usize; + let mut prev_context: Vec = Vec::new(); - BatchJobStatus { - file_uuid: uuid.clone(), - status: "queued".to_string(), - message: "Job queued for async processing".to_string(), + for cut in &cuts { + let sentences = fetch_sentences_in_scene(&db, uuid, cut).await.unwrap_or_default(); + if sentences.is_empty() { continue; } + let context = prev_context.join("\n"); + if let Ok(result) = summarize_one_scene(&db, uuid, cut, &sentences, &context).await { + let _ = store_parent_summary(&db, &cut.chunk_id, uuid, &result.parent_summary, &result.five_w1h, &sentences).await; + let _ = store_child_summaries(&db, uuid, &result.child_summaries).await; + if !result.parent_summary.is_empty() { + prev_context.push(format!("Scene {} (t={:.0}s): {}", cut.chunk_index, cut.start_time, result.parent_summary)); + } } - }) - .collect(); + processed += 1; + } - Ok(Json(BatchAnalyze5W1HResponse { - success: true, - jobs, - })) + jobs.push(BatchJobStatus { + file_uuid: uuid.clone(), + status: if processed > 0 { "completed".to_string() } else { "no_cut_scenes".to_string() }, + message: format!("{}/{} scenes processed", processed, total), + }); + } + + Ok(Json(BatchAnalyze5W1HResponse { success: true, jobs })) } async fn get_5w1h_status( State(state): State, ) -> Result, (StatusCode, String)> { - let db = PostgresDb::from_pool(state.db.pool().clone()); + let table = schema::table_name("videos"); + let rows = sqlx::query(&format!( + r#"SELECT uuid, processing_status->'agents'->'five_w1h' as s + FROM {} WHERE processing_status->'agents'->'five_w1h' IS NOT NULL + ORDER BY updated_at DESC LIMIT 50"#, table + )) + .fetch_all(state.db.pool()).await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - let videos_with_5w1h = fetch_videos_with_5w1h_status(&db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - Ok(Json(serde_json::json!({ - "success": true, - "videos": videos_with_5w1h - }))) -} - -async fn fetch_rule3_chunks(db: &PostgresDb, file_uuid: &str) -> anyhow::Result> { - let table = crate::core::db::schema::table_name("chunks"); - let query = format!( - r#" - SELECT chunk_id, start_frame, end_frame, fps, content, metadata - FROM {} - WHERE uuid = $1 AND (chunk_type = 'scene' OR chunk_type = 'cut') - ORDER BY start_frame - "#, - table - ); - - let rows = sqlx::query(&query) - .bind(file_uuid) - .fetch_all(db.pool()) - .await?; - - let chunks: Vec = rows - .iter() - .map(|row| { - let chunk_id: String = row.try_get("chunk_id").unwrap_or_default(); - let start_frame: i64 = row.try_get("start_frame").unwrap_or(0); - let end_frame: i64 = row.try_get("end_frame").unwrap_or(0); - let fps: f64 = row.try_get("fps").unwrap_or(30.0); - let content: serde_json::Value = - row.try_get("content").unwrap_or(serde_json::Value::Null); - let metadata: serde_json::Value = - row.try_get("metadata").unwrap_or(serde_json::Value::Null); - - let summary = content - .get("data") - .and_then(|d| d.get("scene_number")) - .and_then(|s| s.as_u64()) - .map(|n| format!("Scene {}", n)) - .unwrap_or_else(|| { - content - .get("data") - .and_then(|d| d.get("summary")) - .and_then(|s| s.as_str()) - .unwrap_or("No summary") - .to_string() - }); - - Rule3Chunk { - chunk_id, - start_frame, - end_frame, - fps, - summary, - metadata, - } + let videos: Vec = rows.iter().map(|r| { + serde_json::json!({ + "uuid": r.try_get::("uuid").unwrap_or_default(), + "five_w1h_status": r.try_get::,_>("s").ok().flatten(), }) - .collect(); + }).collect(); - Ok(chunks) + Ok(Json(serde_json::json!({ "success": true, "videos": videos }))) } -async fn process_scene_groups( - db: &PostgresDb, - file_uuid: &str, - rule3_chunks: &[Rule3Chunk], - group_size: usize, - model: &str, -) -> anyhow::Result> { - let mut summaries = Vec::new(); - let chunks_total = rule3_chunks.len(); +/// Pipeline-triggered entry point: run 5W1H agent for a file. +pub async fn run_5w1h_agent(db: &PostgresDb, file_uuid: &str) -> anyhow::Result<()> { + let cuts = fetch_cut_scenes(db, file_uuid).await?; + let total = cuts.len(); + let mut processed = 0usize; + let mut prev_context: Vec = Vec::new(); - for (group_idx, group) in rule3_chunks.chunks(group_size).enumerate() { - let context_text = group - .iter() - .map(|c| c.summary.clone()) - .collect::>() - .join("\n\n"); + for cut in &cuts { + let sentences = fetch_sentences_in_scene(db, file_uuid, cut).await?; + if sentences.is_empty() { continue; } - let faces = aggregate_faces(group); - let objects = aggregate_objects(group); - - let llm_result = call_llm_for_5w1h(&context_text, &faces, &objects, model).await?; - - let start_frame = group.first().map(|c| c.start_frame).unwrap_or(0); - let end_frame = group.last().map(|c| c.end_frame).unwrap_or(0); - let fps = group.first().map(|c| c.fps).unwrap_or(30.0); - - let start_time = start_frame as f64 / fps; - let end_time = end_frame as f64 / fps; - - let chunk_id = format!("summary_{}_{}", file_uuid, group_idx); - - let summary = SummaryChunk { - chunk_id: chunk_id.clone(), - summary: llm_result.summary, - analysis_5w1h: llm_result.analysis_5w1h, - start_frame, - end_frame, - start_time, - end_time, - fps, - scene_count: group.len(), - }; - - store_summary_chunk(db, file_uuid, &summary, group).await?; - - let scenes_processed = ((group_idx + 1) * group_size).min(chunks_total) as i32; - let progress_pct = (scenes_processed as f64 / chunks_total as f64) * 100.0; - - update_agent_status( - db, - file_uuid, - "running", - scenes_processed, - chunks_total as i32, - progress_pct, - ) - .await?; - - summaries.push(summary); + let context = prev_context.join("\n"); + match summarize_one_scene(db, file_uuid, cut, &sentences, &context).await { + Ok(result) => { + let _ = store_parent_summary(db, &cut.chunk_id, file_uuid, &result.parent_summary, &result.five_w1h, &sentences).await; + let _ = store_child_summaries(db, file_uuid, &result.child_summaries).await; + if !result.parent_summary.is_empty() { + prev_context.push(format!("Scene {} (t={:.0}s): {}", cut.chunk_index, cut.start_time, result.parent_summary)); + } + processed += 1; + } + Err(e) => tracing::error!("[5W1H] Scene {} failed: {}", cut.chunk_id, e), + } } - Ok(summaries) -} + tracing::info!("[5W1H] Done for {}: {}/{} scenes", file_uuid, processed, total); -async fn call_llm_for_5w1h( - context_text: &str, - faces: &[String], - objects: &[String], - model: &str, -) -> anyhow::Result { - let system_prompt = r#"You are a video scene analysis assistant for Momentry Core. + // Auto-vectorize sentences with EmbeddingGemma (768D) + tracing::info!("[5W1H] Starting vectorize for sentence chunks..."); + let embedder = Embedder::new("embeddinggemma-300M-Q8_0.gguf".to_string()); + let qdrant = QdrantDb::new(); + qdrant.init_collection(768).await?; -## Task: -Analyze the provided video scenes and extract structured 5W1H information. + let rows = sqlx::query_as::<_, (String, String, String, f64, f64)>( + r#"SELECT chunk_id, chunk_type, text_content, start_time, end_time + FROM dev.chunks WHERE file_uuid = $1 AND chunk_type = 'sentence' AND embedding IS NULL + AND (text_content IS NOT NULL AND text_content != '') ORDER BY chunk_index"# + ) + .bind(file_uuid) + .fetch_all(db.pool()).await?; -## Output Format (JSON): -{ - "summary": "A brief 2-3 sentence summary of these scenes", - "5w1h": { - "who": ["List of main characters/actors"], - "what": ["List of main events/actions"], - "where": ["List of locations/settings"], - "when": "Time of day or temporal context", - "why": "Motivation or reason for events", - "how": "Method or process used" - } -} - -## Guidelines: -- Keep summaries concise and natural -- Extract key information, not details -- Return ONLY valid JSON, no explanations"#; - - let prompt = format!( - r#"Analyze these video scenes: - -## Scene Summaries: -{} - -## Detected Faces: -{} - -## Detected Objects: -{} - -Return the 5W1H analysis in JSON format."#, - context_text, - faces.join(", "), - objects.join(", ") - ); - - let client = Client::new(); - let ollama_url = "http://localhost:11434/api/generate"; - - let body = serde_json::json!({ - "model": model, - "prompt": prompt, - "system": system_prompt, - "stream": false, - "format": "json" - }); - - let response = client - .post(ollama_url) - .json(&body) - .timeout(std::time::Duration::from_secs(60)) - .send() - .await?; - - let ollama_resp: serde_json::Value = response.json().await?; - - let response_text = ollama_resp - .get("response") - .and_then(|v| v.as_str()) - .unwrap_or("{}"); - - let parsed: serde_json::Value = serde_json::from_str(response_text)?; - - let summary = parsed - .get("summary") - .and_then(|s| s.as_str()) - .unwrap_or("No summary generated") - .to_string(); - - let analysis_5w1h = FiveW1HAnalysis { - who: parsed - .get("5w1h") - .and_then(|w| w.get("who")) - .and_then(|w| w.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect() - }) - .unwrap_or_default(), - what: parsed - .get("5w1h") - .and_then(|w| w.get("what")) - .and_then(|w| w.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect() - }) - .unwrap_or_default(), - location: parsed - .get("5w1h") - .and_then(|w| w.get("where")) - .and_then(|w| w.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect() - }) - .unwrap_or_default(), - when: parsed - .get("5w1h") - .and_then(|w| w.get("when")) - .and_then(|w| w.as_str()) - .unwrap_or("unknown") - .to_string(), - why: parsed - .get("5w1h") - .and_then(|w| w.get("why")) - .and_then(|w| w.as_str()) - .unwrap_or("unknown") - .to_string(), - how: parsed - .get("5w1h") - .and_then(|w| w.get("how")) - .and_then(|w| w.as_str()) - .unwrap_or("unknown") - .to_string(), - }; - - Ok(LLM5W1HResult { - summary, - analysis_5w1h, - }) -} - -async fn store_summary_chunk( - db: &PostgresDb, - file_uuid: &str, - summary: &SummaryChunk, - group: &[Rule3Chunk], -) -> anyhow::Result<()> { - let table = crate::core::db::schema::table_name("chunks"); - - let content = serde_json::json!({ - "rule": "rule4", - "data": { - "summary": summary.summary, - "5w1h": { - "who": summary.analysis_5w1h.who, - "what": summary.analysis_5w1h.what, - "where": summary.analysis_5w1h.location, - "when": summary.analysis_5w1h.when, - "why": summary.analysis_5w1h.why, - "how": summary.analysis_5w1h.how, + let total_vec = rows.len(); + let mut stored = 0usize; + for (chunk_id, _ctype, text, start_time, end_time) in &rows { + let text = text.trim(); + if text.is_empty() || text.len() < 5 { continue; } + match embedder.embed_document(text).await { + Ok(vector) => { + if let Err(e) = sqlx::query( + "UPDATE dev.chunks SET embedding = $1::vector WHERE chunk_id = $2 AND file_uuid = $3" + ) + .bind(&vector as &[f32]) + .bind(chunk_id) + .bind(file_uuid) + .execute(db.pool()).await { + tracing::error!("[Vectorize] PG failed for {}: {}", chunk_id, e); + continue; + } + let payload = VectorPayload { + uuid: file_uuid.to_string(), + chunk_id: chunk_id.clone(), + chunk_type: "sentence".to_string(), + start_time: *start_time, + end_time: *end_time, + text: Some(text.to_string()), + }; + if let Err(e) = qdrant.upsert_vector(chunk_id, &vector, payload).await { + tracing::error!("[Vectorize] Qdrant failed for {}: {}", chunk_id, e); + continue; + } + stored += 1; + if stored % 50 == 0 { + tracing::info!("[Vectorize] {}/{}", stored, total_vec); + } } + Err(e) => tracing::error!("[Vectorize] Embed failed for {}: {}", chunk_id, e), } - }); - - let metadata = serde_json::json!({ - "scene_count": summary.scene_count, - "scene_chunk_ids": group.iter().map(|c| c.chunk_id.clone()).collect::>(), - }); - - let query = format!( - r#" - INSERT INTO {} ( - uuid, chunk_id, chunk_index, chunk_type, - start_frame, end_frame, fps, start_time, end_time, content, metadata - ) - VALUES ($1, $2, $3, 'summary', $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb) - ON CONFLICT (uuid, chunk_id) DO UPDATE SET - content = EXCLUDED.content, - metadata = EXCLUDED.metadata, - updated_at = CURRENT_TIMESTAMP - "#, - table - ); - - let start_time = summary.start_time; - let end_time = summary.end_time; - - sqlx::query(&query) - .bind(file_uuid) - .bind(&summary.chunk_id) - .bind(0) - .bind(summary.start_frame) - .bind(summary.end_frame) - .bind(summary.fps) - .bind(start_time) - .bind(end_time) - .bind(&content) - .bind(&metadata) - .execute(db.pool()) - .await?; - - Ok(()) -} - -async fn update_agent_status( - db: &PostgresDb, - file_uuid: &str, - status: &str, - scenes_processed: i32, - scenes_total: i32, - progress_pct: f64, -) -> anyhow::Result<()> { - let table = crate::core::db::schema::table_name("videos"); - - let agent_status = serde_json::json!({ - "five_w1h": { - "status": status, - "scenes_processed": scenes_processed, - "scenes_total": scenes_total, - "progress_pct": progress_pct - } - }); - - let query = format!( - r#" - UPDATE {} - SET processing_status = jsonb_set( - COALESCE(processing_status, '{{}}'::jsonb), - '{{agents}}', - $1::jsonb - ) - WHERE uuid = $2 - "#, - table - ); - - sqlx::query(&query) - .bind(&agent_status) - .bind(file_uuid) - .execute(db.pool()) - .await?; - - Ok(()) -} - -async fn fetch_videos_with_5w1h_status(db: &PostgresDb) -> anyhow::Result> { - let table = crate::core::db::schema::table_name("videos"); - - let query = format!( - r#" - SELECT uuid, processing_status->'agents'->'five_w1h' as agent_status - FROM {} - WHERE processing_status->'agents'->'five_w1h' IS NOT NULL - ORDER BY updated_at DESC - LIMIT 50 - "#, - table - ); - - let rows = sqlx::query(&query).fetch_all(db.pool()).await?; - - let videos: Vec = rows - .iter() - .map(|row| { - let uuid: String = row.try_get("uuid").unwrap_or_default(); - let status: Option = row.try_get("agent_status").ok(); - - serde_json::json!({ - "uuid": uuid, - "five_w1h_status": status - }) - }) - .collect(); - - Ok(videos) -} - -async fn process_single_video_5w1h( - db: &PostgresDb, - file_uuid: &str, - scene_group_size: usize, -) -> anyhow::Result<()> { - let rule3_chunks = fetch_rule3_chunks(db, file_uuid).await?; - - if rule3_chunks.is_empty() { - return Ok(()); } - - let scenes_total = rule3_chunks.len() as i32; - - update_agent_status(db, file_uuid, "running", 0, scenes_total, 0.0).await?; - - let _ = process_scene_groups( - db, - file_uuid, - &rule3_chunks, - scene_group_size, - "gemma4:latest", - ) - .await?; - - update_agent_status( - db, - file_uuid, - "completed", - scenes_total, - scenes_total, - 100.0, - ) - .await?; - + tracing::info!("[5W1H] Vectorize done: {}/{} stored", stored, total_vec); Ok(()) } - -fn aggregate_faces(group: &[Rule3Chunk]) -> Vec { - group - .iter() - .flat_map(|c| { - c.metadata - .get("faces") - .and_then(|f| f.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect::>() - }) - .unwrap_or_default() - }) - .collect() -} - -fn aggregate_objects(group: &[Rule3Chunk]) -> Vec { - group - .iter() - .flat_map(|c| { - c.metadata - .get("objects") - .and_then(|o| o.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect::>() - }) - .unwrap_or_default() - }) - .collect() -} - -#[derive(Debug, Clone)] -struct Rule3Chunk { - chunk_id: String, - start_frame: i64, - end_frame: i64, - fps: f64, - summary: String, - metadata: serde_json::Value, -} - -#[derive(Debug)] -struct LLM5W1HResult { - summary: String, - analysis_5w1h: FiveW1HAnalysis, -} diff --git a/src/api/identity_agent_api.rs b/src/api/identity_agent_api.rs index ae32dd2..3de87da 100644 --- a/src/api/identity_agent_api.rs +++ b/src/api/identity_agent_api.rs @@ -10,6 +10,7 @@ use sqlx::Row; use std::path::PathBuf; use crate::api::server::AppState; +use crate::core::db::PostgresDb; pub fn identity_agent_routes() -> Router { Router::new() @@ -124,6 +125,13 @@ async fn analyze_identity( let face_clustered_path = video_dir.join(format!("{}.face_clustered.json", req.file_uuid)); let asrx_path = video_dir.join(format!("{}.asrx.json", req.file_uuid)); + // 如果子目錄找不到，試根目錄 + let face_clustered_path = if face_clustered_path.exists() { + face_clustered_path + } else { + PathBuf::from(&output_dir).join(format!("{}.face_clustered.json", req.file_uuid)) + }; + if !face_clustered_path.exists() { return Err(( StatusCode::NOT_FOUND, @@ -132,37 +140,15 @@ async fn analyze_identity( } let face_data: serde_json::Value = std::fs::read_to_string(&face_clustered_path) - .map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read face data: {}", e), - ) - })? + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read face data: {}", e)))? .parse() - .map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to parse face data: {}", e), - ) - })?; + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to parse face data: {}", e)))?; let asrx_data: Option = if asrx_path.exists() { - Some( - std::fs::read_to_string(&asrx_path) - .map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read asrx data: {}", e), - ) - })? - .parse() - .map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to parse asrx data: {}", e), - ) - })?, - ) + Some(std::fs::read_to_string(&asrx_path) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read asrx data: {}", e)))? + .parse() + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to parse asrx data: {}", e)))?) } else { None }; @@ -172,6 +158,37 @@ async fn analyze_identity( let identities = analyze_person_speaker_overlap(&persons, &speakers); + // 將 identity 結果寫入 DB + let pool = state.db.pool(); + for id_result in &identities { + let identity_name = format!("person_{}", id_result.person_ids.first().map(|s| &**s).unwrap_or("unknown")); + let metadata = serde_json::json!({ + "source": "identity_agent", + "trace_ids": id_result.person_ids, + "speaker_ids": id_result.speaker_ids, + "confidence": id_result.confidence, + "evidence": { + "speaker_overlap": id_result.evidence.speaker_overlap, + "frame_ratio": id_result.evidence.frame_ratio, + }, + "reasoning": id_result.reasoning, + }); + + let _ = sqlx::query( + "INSERT INTO dev.identities (name, identity_type, source, metadata, status) VALUES ($1, 'people', 'auto', $2::jsonb, 'pending') ON CONFLICT DO NOTHING" + ) + .bind(&identity_name) + .bind(&metadata) + .execute(pool) + .await; + } + + // 迭代多角度 face embedding 比對（TMDb seed → 傳播） + let _ = match_faces_iterative(pool, &req.file_uuid).await.unwrap_or(0); + + // 將 ASRX speaker 綁定到已匹配 identity 的 trace + let _ = bind_speakers(pool, &req.file_uuid).await.unwrap_or(0); + let processing_status = IdentityProcessingStatus { status: "completed".to_string(), persons_analyzed: persons.len() as i32, @@ -287,33 +304,27 @@ fn extract_persons_from_face_data(face_data: &serde_json::Value) -> Vec) -> Vec { let mut speakers = Vec::new(); - if let Some(data) = asrx_data { if let Some(segments) = data.get("segments").and_then(|s| s.as_array()) { let mut speaker_segments_map: std::collections::HashMap> = std::collections::HashMap::new(); - for segment in segments { - if let Some(speaker_id) = segment.get("speaker").and_then(|s| s.as_str()) { - let start = segment.get("start").and_then(|s| s.as_f64()).unwrap_or(0.0); - let end = segment.get("end").and_then(|e| e.as_f64()).unwrap_or(0.0); - + let speaker_id = segment.get("speaker_id").and_then(|s| s.as_str()) + .or_else(|| segment.get("speaker").and_then(|s| s.as_str())); + if let Some(speaker_id) = speaker_id { + let start = segment.get("start").or_else(|| segment.get("start_time")).and_then(|s| s.as_f64()).unwrap_or(0.0); + let end = segment.get("end").or_else(|| segment.get("end_time")).and_then(|e| e.as_f64()).unwrap_or(0.0); speaker_segments_map .entry(speaker_id.to_string()) .or_insert_with(Vec::new) .push((start, end)); } } - for (speaker_id, segments) in speaker_segments_map { - speakers.push(SpeakerData { - speaker_id, - segments, - }); + speakers.push(SpeakerData { speaker_id, segments }); } } } - speakers } @@ -374,10 +385,8 @@ fn analyze_person_speaker_overlap( fn calculate_overlap(person: &PersonData, speaker: &SpeakerData) -> i32 { let mut overlap_count = 0; - for frame_num in &person.frames { - let frame_time = *frame_num as f64 / 23.976; - + let frame_time = *frame_num as f64 / 25.0; // default fps=25 for (start, end) in &speaker.segments { if frame_time >= *start && frame_time <= *end { overlap_count += 1; @@ -385,7 +394,6 @@ fn calculate_overlap(person: &PersonData, speaker: &SpeakerData) -> i32 { } } } - overlap_count } @@ -416,31 +424,31 @@ async fn suggest_clustering( State(state): State, Json(req): Json, ) -> Result, (StatusCode, String)> { - let min_cluster_size = req.min_cluster_size.unwrap_or(3); - let file_filter = match &req.file_uuid { - Some(uuid) => format!("AND fc.file_uuid = '{}'", uuid), + Some(uuid) => format!("AND fd.file_uuid = '{}'", uuid), None => String::new(), }; let query = format!( r#" - SELECT fc.cluster_id, fc.file_uuid, fc.n_faces, fc.metadata - FROM face_clusters fc - WHERE fc.n_faces >= $1 + SELECT trace_id, file_uuid, COUNT(*) as face_count + FROM dev.face_detections fd + WHERE fd.trace_id IS NOT NULL AND NOT EXISTS ( - SELECT 1 FROM identities i - WHERE i.metadata->>'cluster_id' = fc.cluster_id + SELECT 1 FROM dev.identities i + WHERE i.metadata->>'trace_id' = fd.trace_id::text ) {} - ORDER BY fc.n_faces DESC + GROUP BY trace_id, file_uuid + HAVING COUNT(*) >= $1 + ORDER BY face_count DESC "#, file_filter ); let pool = state.db.pool(); let rows = sqlx::query(&query) - .bind(min_cluster_size as i64) + .bind(req.min_cluster_size.unwrap_or(3) as i64) .fetch_all(pool) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; @@ -448,27 +456,14 @@ async fn suggest_clustering( let suggestions: Vec = rows .into_iter() .map(|row| { - let cluster_id: String = row.get("cluster_id"); - let n_faces: i32 = row.get("n_faces"); - let metadata: serde_json::Value = - row.try_get("metadata").unwrap_or(serde_json::Value::Null); - - let avg_confidence = metadata - .get("avg_confidence") - .and_then(|v| v.as_f64()) - .unwrap_or(0.0); - - let representative_face = metadata - .get("representative_face_id") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - + let trace_id: Option = row.try_get("trace_id").ok(); + let face_count: i64 = row.get("face_count"); ClusteringSuggestion { - cluster_id, - face_count: n_faces as usize, - avg_confidence, + cluster_id: format!("trace_{}", trace_id.unwrap_or(0)), + face_count: face_count as usize, + avg_confidence: 0.0, suggested_name: None, - representative_face, + representative_face: None, } }) .collect(); @@ -601,3 +596,325 @@ struct SpeakerData { speaker_id: String, segments: Vec<(f64, f64)>, } + +fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { + if a.len() != b.len() || a.is_empty() { return 0.0; } + let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum(); + let na: f32 = a.iter().map(|x| x * x).sum::().sqrt(); + let nb: f32 = b.iter().map(|x| x * x).sum::().sqrt(); + if na == 0.0 || nb == 0.0 { 0.0 } else { dot / (na * nb) } +} + +/// 迭代多角度 face embedding 比對 + 傳播 +/// Round 1: 用 TMDb seed face_embedding 比對 face_detections (threshold 0.50) +/// Round 2+: 用已匹配 trace 的所有 face 作為 seed，傳播到未匹配 trace +async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Result { + // Step 1: 載入 TMDb identities (source='tmdb' 且有 face_embedding) + let tmdb_rows = sqlx::query_as::<_, (i32, String, Vec)>( + "SELECT id, name, face_embedding::real[] FROM dev.identities WHERE source='tmdb' AND face_embedding IS NOT NULL" + ) + .fetch_all(pool).await?; + + if tmdb_rows.is_empty() { + tracing::warn!("[FaceMatch] No TMDb identities with face embeddings"); + return Ok(0); + } + tracing::info!("[FaceMatch] Loaded {} TMDb seed identities", tmdb_rows.len()); + + // Step 2: 載入所有 face_detections，按 trace_id 分組 + let fd_rows = sqlx::query_as::<_, (i32, Vec)>( + "SELECT trace_id, embedding FROM dev.face_detections \ + WHERE file_uuid=$1 AND trace_id IS NOT NULL AND embedding IS NOT NULL \ + ORDER BY trace_id" + ) + .bind(file_uuid) + .fetch_all(pool).await?; + + if fd_rows.is_empty() { + tracing::warn!("[FaceMatch] No face detections with embeddings"); + return Ok(0); + } + + // 分組：trace_id → Vec + use std::collections::HashMap; + let mut trace_faces: HashMap>> = HashMap::new(); + for (tid, emb) in &fd_rows { + trace_faces.entry(*tid).or_insert_with(Vec::new).push(emb.clone()); + } + + // 去重：同一個 trace 內，embedding 太接近的只留一個 + for faces in trace_faces.values_mut() { + faces.sort_by(|a, b| b[0].partial_cmp(&a[0]).unwrap_or(std::cmp::Ordering::Equal)); + faces.dedup_by(|a, b| cosine_similarity(a, b) > 0.99); + } + + let total_traces = trace_faces.len(); + tracing::info!("[FaceMatch] Loaded {} traces with {} faces", total_traces, fd_rows.len()); + + // Step 3: 建立 TMDb 查找表 + let tmdb_seeds: Vec<(i32, String, Vec)> = tmdb_rows; + + // Step 4: 迭代匹配 + const TH: f32 = 0.50; + let mut matched: HashMap = HashMap::new(); // trace_id → identity_name + + // Round 1: 直接比對 TMDb + for (&tid, faces) in &trace_faces { + let mut best_name = String::new(); + let mut best_sim = 0.0f32; + for (_, ref name, ref tmdb_emb) in &tmdb_seeds { + for face_emb in faces { + let s = cosine_similarity(face_emb, tmdb_emb); + if s > best_sim { best_sim = s; best_name = name.clone(); } + } + } + if best_sim >= TH { + matched.insert(tid, best_name); + } + } + tracing::info!("[FaceMatch] Round 1: {} matched ({}%)", matched.len(), matched.len() * 100 / total_traces); + + // Round 2+: 用已匹配的 face 作為 seed 傳播 + for round_n in 2..=10 { + let prev = matched.len(); + // 建立 seed pool: name → Vec + let mut seed_pool: HashMap>> = HashMap::new(); + for (&tid, name) in &matched { + if let Some(faces) = trace_faces.get(&tid) { + seed_pool.entry(name.clone()).or_default().extend(faces.iter()); + } + } + + let mut new_matches: Vec<(i32, String)> = Vec::new(); + for (&tid, faces) in &trace_faces { + if matched.contains_key(&tid) { continue; } + let mut best_name = String::new(); + let mut best_sim = 0.0f32; + if faces.is_empty() { continue; } + let ref_face = &faces[0]; + for (name, seed_faces) in &seed_pool { + for seed in seed_faces { + let s = cosine_similarity(ref_face, seed); + if s > best_sim { best_sim = s; best_name = name.clone(); } + } + } + if best_sim >= TH { + new_matches.push((tid, best_name)); + } + } + for (tid, name) in new_matches { + matched.insert(tid, name); + } + let new = matched.len() - prev; + tracing::info!("[FaceMatch] Round {}: +{} matched (total {}, {}%)", round_n, new, matched.len(), matched.len() * 100 / total_traces); + if new < 5 { break; } + } + + // Step 5: 寫入 DB + let mut updated = 0usize; + for (tid, name) in &matched { + let id_opt = sqlx::query_scalar::<_, Option>( + "SELECT id FROM dev.identities WHERE name=$1 AND source='tmdb'" + ) + .bind(name) + .fetch_optional(pool).await?; + if let Some(identity_id) = id_opt { + let _ = sqlx::query( + "UPDATE dev.face_detections SET identity_id=$1 WHERE file_uuid=$2 AND trace_id=$3" + ) + .bind(identity_id) + .bind(file_uuid) + .bind(tid) + .execute(pool).await; + updated += 1; + } + } + + tracing::info!("[FaceMatch] Done: {}/{} traces matched ({}%)", matched.len(), total_traces, matched.len() * 100 / total_traces); + Ok(updated) +} + +/// Bind ASRX speakers to face traces based on temporal overlap. +/// Reads face_detections (trace_id, identity_id, frame_number) and ASRX +/// segments (speaker_id, start_time, end_time), computes overlap, +/// and stores bindings in identity_bindings table. +pub async fn bind_speakers(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Result { + // Load face traces with identity_id and frame numbers + let traces = sqlx::query_as::<_, (i32, Vec)>( + "SELECT trace_id, array_agg(frame_number ORDER BY frame_number) \ + FROM dev.face_detections WHERE file_uuid=$1 AND trace_id IS NOT NULL AND identity_id IS NOT NULL \ + GROUP BY trace_id" + ) + .bind(file_uuid) + .fetch_all(pool).await?; + + if traces.is_empty() { + tracing::info!("[SpeakerBind] No face traces with identities"); + return Ok(0); + } + + // Load ASRX speakers from the output JSON + let output_dir = std::env::var("MOMENTRY_OUTPUT_DIR") + .unwrap_or_else(|_| "/Users/accusys/momentry/output".to_string()); + let asrx_path = std::path::Path::new(&output_dir).join(format!("{}.asrx.json", file_uuid)); + + let asrx_data: serde_json::Value = match std::fs::read_to_string(&asrx_path) { + Ok(s) => serde_json::from_str(&s).unwrap_or_default(), + Err(_) => { + tracing::info!("[SpeakerBind] No ASRX file found"); + return Ok(0); + } + }; + + // Extract speaker segments: speaker_id → [(start_time, end_time)] + use std::collections::HashMap; + let mut speakers: HashMap> = HashMap::new(); + if let Some(segments) = asrx_data.get("segments").and_then(|s| s.as_array()) { + for seg in segments { + let sid = seg.get("speaker_id").and_then(|s| s.as_str()) + .or_else(|| seg.get("speaker").and_then(|s| s.as_str())); + if let Some(sid) = sid { + let start = seg.get("start_time").or_else(|| seg.get("start")).and_then(|v| v.as_f64()).unwrap_or(0.0); + let end = seg.get("end_time").or_else(|| seg.get("end")).and_then(|v| v.as_f64()).unwrap_or(0.0); + speakers.entry(sid.to_string()).or_default().push((start, end)); + } + } + } + + if speakers.is_empty() { + tracing::info!("[SpeakerBind] No speakers found in ASRX data"); + return Ok(0); + } + + // Get fps for frame-to-time conversion + let fps: f64 = 25.0; // default, could also read from DB + + // For each trace, compute overlap with each speaker + let mut bindings = 0usize; + for (trace_id, frames) in &traces { + if frames.is_empty() { continue; } + + // Get identity_id for this trace + let identity_id: Option = sqlx::query_scalar( + "SELECT identity_id FROM dev.face_detections WHERE file_uuid=$1 AND trace_id=$2 AND identity_id IS NOT NULL LIMIT 1" + ) + .bind(file_uuid).bind(trace_id) + .fetch_optional(pool).await?.flatten(); + + if identity_id.is_none() { continue; } + let identity_id = identity_id.unwrap(); + + // Compute overlap with each speaker + let mut best_speaker = String::new(); + let mut best_overlap = 0usize; + + for (speaker_id, segments) in &speakers { + let mut overlap = 0usize; + for &fn_num in frames { + let frame_time = fn_num as f64 / fps; + for (start, end) in segments { + if frame_time >= *start && frame_time <= *end { + overlap += 1; + break; + } + } + } + if overlap > best_overlap { + best_overlap = overlap; + best_speaker = speaker_id.clone(); + } + } + + // Only bind if meaningful overlap + let overlap_ratio = best_overlap as f64 / frames.len() as f64; + if overlap_ratio > 0.3 && !best_speaker.is_empty() { + let metadata = serde_json::json!({ + "trace_id": trace_id, + "overlap_frames": best_overlap, + "total_frames": frames.len(), + "overlap_ratio": overlap_ratio, + }); + + let _ = sqlx::query( + "INSERT INTO dev.identity_bindings (identity_id, identity_type, identity_value, confidence, metadata) \ + VALUES ($1, 'speaker', $2, $3, $4::jsonb) \ + ON CONFLICT (identity_id, identity_type, identity_value) DO UPDATE SET confidence = EXCLUDED.confidence, metadata = EXCLUDED.metadata" + ) + .bind(identity_id) + .bind(&best_speaker) + .bind(overlap_ratio) + .bind(&metadata) + .execute(pool).await; + + bindings += 1; + } + } + + tracing::info!("[SpeakerBind] Created {}/{} speaker bindings", bindings, traces.len()); + Ok(bindings) +} + +/// Pipeline-triggered entry point: runs the full identity agent for a file. +/// Reads face_clustered.json + asrx.json, extracts persons/speakers, creates identities, +/// runs iterative face matching, and binds speakers. +pub async fn run_identity_agent(db: &PostgresDb, file_uuid: &str) -> anyhow::Result<()> { + let output_dir = std::env::var("MOMENTRY_OUTPUT_DIR") + .unwrap_or_else(|_| "/Users/accusys/momentry/output".to_string()); + + let video_dir = PathBuf::from(&output_dir).join(file_uuid); + let face_clustered_path = video_dir.join(format!("{}.face_clustered.json", file_uuid)); + let face_clustered_path = if face_clustered_path.exists() { + face_clustered_path + } else { + PathBuf::from(&output_dir).join(format!("{}.face_clustered.json", file_uuid)) + }; + + if !face_clustered_path.exists() { + tracing::warn!("[IdentityAgent] face_clustered.json not found for {}", file_uuid); + return Ok(()); + } + + let face_data: serde_json::Value = std::fs::read_to_string(&face_clustered_path)?.parse()?; + let asrx_path = video_dir.join(format!("{}.asrx.json", file_uuid)); + let asrx_data: Option = if asrx_path.exists() { + Some(std::fs::read_to_string(&asrx_path)?.parse()?) + } else { + None + }; + + let persons = extract_persons_from_face_data(&face_data); + let speakers = extract_speakers_from_asrx_data(&asrx_data); + let identities = analyze_person_speaker_overlap(&persons, &speakers); + + let pool = db.pool(); + for id_result in &identities { + let identity_name = format!("person_{}", id_result.person_ids.first().map(|s| &**s).unwrap_or("unknown")); + let metadata = serde_json::json!({ + "source": "identity_agent", + "trace_ids": id_result.person_ids, + "speaker_ids": id_result.speaker_ids, + "confidence": id_result.confidence, + "evidence": { + "speaker_overlap": id_result.evidence.speaker_overlap, + "frame_ratio": id_result.evidence.frame_ratio, + }, + "reasoning": id_result.reasoning, + }); + let _ = sqlx::query( + "INSERT INTO dev.identities (name, identity_type, source, metadata, status) VALUES ($1, 'people', 'auto', $2::jsonb, 'pending') ON CONFLICT DO NOTHING" + ) + .bind(&identity_name) + .bind(&metadata) + .execute(pool) + .await; + } + + let matched = match_faces_iterative(pool, file_uuid).await.unwrap_or(0); + let bound = bind_speakers(pool, file_uuid).await.unwrap_or(0); + + tracing::info!( + "[IdentityAgent] Done for {}: {} identities, {} face matches, {} speaker bindings", + file_uuid, identities.len(), matched, bound + ); + Ok(()) +} diff --git a/src/api/media_api.rs b/src/api/media_api.rs index 2ffb48c..62e471e 100644 --- a/src/api/media_api.rs +++ b/src/api/media_api.rs @@ -282,16 +282,44 @@ async fn trace_video( let duration = (last_frame - first_frame) as f64 / fps + padding * 2.0; let seek = (start_sec - padding).max(0.0); - // Build filters: per-frame bbox + text + // Build filters: bbox+text holding at last detection until next one let mut parts: Vec = Vec::new(); - for (frame, x, y, w, h) in &rows { - let offset = frame - first_frame + (padding * fps) as i32; + for (i, (frame, x, y, w, h)) in rows.iter().enumerate() { + // Hold this detection until the next one (or end) + let next_frame = if i + 1 < rows.len() { + rows[i + 1].0 + } else { + // For last detection, extend to duration end + last_frame + (padding * fps) as i32 + }; + let start_offset = frame - first_frame + (padding * fps) as i32; + let end_offset = next_frame - first_frame + (padding * fps) as i32; + + // Bbox: visible from this frame until next detection parts.push(format!( - "drawbox=x={}:y={}:w={}:h={}:color=red@0.8:thickness=8:enable='eq(n,{})'", - x, y, w, h, offset + "drawbox=x={}:y={}:w={}:h={}:color=red@0.8:thickness=8:enable='between(n,{},{})'", + x, y, w, h, start_offset, end_offset - 1 )); + // Text: same hold behavior let label = format!("t{}", trace_id); - render_text(&mut parts, &label, *x + 6, *y + 6, Some(offset)); + let mut tx = *x + 6; + let mut ty = *y + 6; + for ch in label.chars() { + let bm = bitmap_char(ch); + for (row, bits) in bm.iter().enumerate() { + for col in 0..5 { + if bits & (1 << (4 - col)) != 0 { + let dx = tx + col as i32 * 3; + let dy = ty + row as i32 * 3; + parts.push(format!( + "drawbox=x={}:y={}:w=3:h=3:color=white@1.0:t=fill:enable='between(n,{},{})'", + dx, dy, start_offset, end_offset - 1 + )); + } + } + } + tx += CHAR_ADVANCE; + } } let vf = if parts.is_empty() { diff --git a/src/api/search.rs b/src/api/search.rs index 929beb5..262c143 100644 --- a/src/api/search.rs +++ b/src/api/search.rs @@ -162,7 +162,7 @@ async fn get_ollama_embedding( ) -> Result, Box> { let client = reqwest::Client::new(); let payload = serde_json::json!({ - "model": "nomic-embed-text", + "model": "mxbai-embed-large", "prompt": text }); diff --git a/src/api/server.rs b/src/api/server.rs index a5ff50e..968ae6f 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -26,6 +26,7 @@ use super::identity_api; use super::identity_binding; use super::middleware::api_key_validation; use super::search::search_routes; +use super::trace_agent_api; use super::universal_search::universal_search_routes; use super::visual_chunk_search; use crate::core::chunk::types::Chunk; @@ -794,8 +795,6 @@ async fn register_single_file( .arg(&cut_script) .arg(&canonical_path) .arg(&cut_path) - .arg("--threshold") - .arg("27") .output(); if let Ok(output) = cut_output { if output.status.success() { @@ -2246,42 +2245,58 @@ async fn list_jobs(Query(params): Query) -> Result = status_filter + .split(',') + .map(|s| format!("'{}'", s.trim())) + .collect(); + let status_clause = statuses.join(","); + + let query = format!( + "SELECT id, uuid, video_path, status, current_processor, progress_total, progress_current, + error_count, last_error, started_at::TEXT, updated_at::TEXT, created_at::TEXT, + processors, completed_processors, failed_processors, video_id + FROM {} + WHERE status IN ({}) + ORDER BY created_at DESC + LIMIT {} OFFSET {}", + table, status_clause, page_size, offset + ); + + let count_query = format!( + "SELECT COUNT(*) FROM {} WHERE status IN ({})", + table, status_clause + ); + + let total_count: i64 = sqlx::query_scalar(&count_query) + .fetch_one(pg.pool()) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - // 過濾狀態 - let filtered_jobs: Vec<_> = jobs - .into_iter() - .filter(|j| { - let job_status = j.status.as_str(); - status_filter.split(',').any(|s| s.trim() == job_status) - }) - .collect(); + use crate::core::db::MonitorJobStatus; - let total_count = filtered_jobs.len() as i64; + let rows = sqlx::query(&query) + .fetch_all(pg.pool()) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - // 手動分頁 - let paginated_jobs: Vec<_> = filtered_jobs + let job_infos: Vec = rows .into_iter() - .skip(offset as usize) - .take(page_size) - .collect(); - - let job_infos: Vec = paginated_jobs - .into_iter() - .map(|j| JobInfoResponse { - id: j.id, - uuid: j.uuid, - status: j.status.as_str().to_string(), - current_processor: j.current_processor, - progress_current: j.progress_current, - progress_total: j.progress_total, - created_at: j.created_at.to_string(), - started_at: j.started_at.map(|t| t.to_string()), + .map(|r| { + let status_str: String = r.try_get("status").unwrap_or_default(); + let status = MonitorJobStatus::from_db_str(&status_str).unwrap_or(MonitorJobStatus::Pending); + JobInfoResponse { + id: r.try_get("id").unwrap_or(0), + uuid: r.try_get("uuid").unwrap_or_default(), + status: status.as_str().to_string(), + current_processor: r.try_get("current_processor").ok(), + progress_current: r.try_get("progress_current").unwrap_or(0), + progress_total: r.try_get("progress_total").unwrap_or(0), + created_at: r.try_get::("created_at").unwrap_or_default(), + started_at: r.try_get::("started_at").ok(), + } }) .collect(); @@ -2537,6 +2552,7 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> { .merge(super::identity_agent_api::identity_agent_routes()) // Phase 5 Routes .merge(five_w1h_agent_api::five_w1h_agent_routes()) // Phase 3 Routes (5W1H Agent) .merge(super::media_api::bbox_routes()) // Media: video/bbox/thumbnail + .merge(super::trace_agent_api::trace_agent_routes()) // Trace listing .merge(search_routes()) // Smart search drill-down .merge(universal_search_routes()) // Universal / frames / persons search .merge(protected_routes) @@ -3242,7 +3258,7 @@ async fn list_pre_chunks( let data_query = format!( "SELECT id, processor_type, coordinate_type, coordinate_index, start_frame, end_frame, start_time, end_time, fps, - data, identity_id, confidence, created_at + data, created_at FROM {} WHERE file_uuid = $1 {} ORDER BY coordinate_index ASC @@ -3261,8 +3277,6 @@ async fn list_pre_chunks( Option, Option, serde_json::Value, - Option, - Option, chrono::DateTime, )> = sqlx::query_as(&data_query) .bind(&uuid) @@ -3283,9 +3297,9 @@ async fn list_pre_chunks( end_time: row.7, fps: row.8, data: row.9.clone(), - identity_id: row.10.map(|id| id.to_string()), - confidence: row.11, - created_at: row.12.to_rfc3339(), + identity_id: None, + confidence: None, + created_at: row.10.to_rfc3339(), }) .collect(); diff --git a/src/api/trace_agent_api.rs b/src/api/trace_agent_api.rs index f380c63..e142c61 100644 --- a/src/api/trace_agent_api.rs +++ b/src/api/trace_agent_api.rs @@ -1,8 +1,8 @@ use axum::{ - extract::{Path, State}, + extract::{Path, Query, State}, http::StatusCode, response::Json, - routing::post, + routing::{get, post}, Router, }; use serde::{Deserialize, Serialize}; @@ -10,10 +10,15 @@ use serde::{Deserialize, Serialize}; use crate::core::db::PostgresDb; pub fn trace_agent_routes() -> Router { - Router::new().route( - "/api/v1/file/:file_uuid/face_trace/sortby", - post(list_traces_sorted), - ) + Router::new() + .route( + "/api/v1/file/:file_uuid/face_trace/sortby", + post(list_traces_sorted), + ) + .route( + "/api/v1/file/:file_uuid/trace/:trace_id/faces", + get(list_trace_faces), + ) } #[derive(Debug, Deserialize)] @@ -21,6 +26,8 @@ struct TracesRequest { min_faces: Option, sort_by: Option, limit: Option, + min_confidence: Option, + max_confidence: Option, } #[derive(Debug, Serialize)] @@ -53,14 +60,15 @@ async fn list_traces_sorted( let min_faces = req.min_faces.unwrap_or(1); let sort = req.sort_by.as_deref().unwrap_or("first_appearance"); let limit = req.limit.unwrap_or(500); + let min_confidence = req.min_confidence.unwrap_or(0.0); + let max_confidence = req.max_confidence.unwrap_or(1.0); let order_clause = match sort { "face_count" => "face_count DESC", - "duration" => "duration_sec DESC", + "duration" => "(MAX(frame_number) - MIN(frame_number)) DESC", _ => "first_frame ASC", }; - // Get actual video FPS let fps: f64 = sqlx::query_scalar("SELECT COALESCE(fps, 24.0) FROM dev.videos WHERE file_uuid = $1") .bind(&file_uuid) @@ -84,6 +92,7 @@ async fn list_traces_sorted( AVG(confidence) AS avg_confidence FROM dev.face_detections WHERE file_uuid = $1 AND trace_id IS NOT NULL + AND confidence >= $4 AND confidence <= $5 GROUP BY trace_id HAVING COUNT(*) >= $2 ORDER BY {} @@ -103,6 +112,8 @@ async fn list_traces_sorted( .bind(&file_uuid) .bind(min_faces) .bind(limit) + .bind(min_confidence) + .bind(max_confidence) .fetch_all(state.db.pool()) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; @@ -138,3 +149,146 @@ async fn list_traces_sorted( traces, })) } + +// ── Individual face detections for a trace ── + +#[derive(Debug, Deserialize)] +struct TraceFacesQuery { + limit: Option, + offset: Option, + interpolate: Option, +} + +#[derive(Debug, Serialize)] +struct TraceFaceItem { + id: i32, + start_frame: i32, + start_time: f64, + x: Option, + y: Option, + width: Option, + height: Option, + confidence: f64, + interpolated: bool, +} + +#[derive(Debug, Serialize)] +struct TraceFacesResponse { + success: bool, + file_uuid: String, + trace_id: i32, + total: i64, + faces: Vec, +} + +fn lerp_i32(a: Option, b: Option, t: f64) -> Option { + match (a, b) { + (Some(av), Some(bv)) => Some((av as f64 + (bv - av) as f64 * t).round() as i32), + _ => None, + } +} + +async fn list_trace_faces( + State(state): State, + Path((file_uuid, trace_id)): Path<(String, i32)>, + Query(q): Query, +) -> Result, (StatusCode, String)> { + let limit = q.limit.unwrap_or(200).min(1000); + let offset = q.offset.unwrap_or(0); + let interpolate = q.interpolate.unwrap_or(false); + + let fps: f64 = + sqlx::query_scalar("SELECT COALESCE(fps, 24.0) FROM dev.videos WHERE file_uuid = $1") + .bind(&file_uuid) + .fetch_optional(state.db.pool()) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .unwrap_or(24.0); + + let total_detected: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid = $1 AND trace_id = $2" + ) + .bind(&file_uuid) + .bind(trace_id) + .fetch_one(state.db.pool()) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let rows: Vec<(i32, i32, Option, Option, Option, Option, f32)> = + sqlx::query_as( + "SELECT id, frame_number, x, y, width, height, confidence + FROM dev.face_detections + WHERE file_uuid = $1 AND trace_id = $2 + ORDER BY frame_number ASC + LIMIT $3 OFFSET $4" + ) + .bind(&file_uuid) + .bind(trace_id) + .bind(limit) + .bind(offset) + .fetch_all(state.db.pool()) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let mut faces: Vec = Vec::new(); + + for (i, (id, frame, x, y, w, h, conf)) in rows.iter().enumerate() { + let cur = (x, y, w, h); + + // Add interpolated frames between previous and current detection + if interpolate && i > 0 { + let prev = &rows[i - 1]; + let prev_frame = prev.1; + let gap = frame - prev_frame; + if gap > 1 { + for mid in 1..gap { + let t = mid as f64 / gap as f64; + let mid_x = lerp_i32(prev.2, *x, t); + let mid_y = lerp_i32(prev.3, *y, t); + let mid_w = lerp_i32(prev.4, *w, t); + let mid_h = lerp_i32(prev.5, *h, t); + let mid_frame = prev_frame + mid; + faces.push(TraceFaceItem { + id: 0, + start_frame: mid_frame, + start_time: (mid_frame as f64 / fps * 10.0).round() / 10.0, + x: mid_x, + y: mid_y, + width: mid_w, + height: mid_h, + confidence: 0.0, + interpolated: true, + }); + } + } + } + + // Add the real detection + let frame_val = *frame; + faces.push(TraceFaceItem { + id: *id, + start_frame: frame_val, + start_time: (frame_val as f64 / fps * 10.0).round() / 10.0, + x: *x, + y: *y, + width: *w, + height: *h, + confidence: *conf as f64, + interpolated: false, + }); + } + + let total = if interpolate && faces.len() as i64 > total_detected { + faces.len() as i64 + } else { + total_detected + }; + + Ok(Json(TraceFacesResponse { + success: true, + file_uuid, + trace_id, + total, + faces, + })) +} diff --git a/src/core/chunk/rule3_ingest.rs b/src/core/chunk/rule3_ingest.rs index 82462aa..d31ddec 100644 --- a/src/core/chunk/rule3_ingest.rs +++ b/src/core/chunk/rule3_ingest.rs @@ -74,7 +74,7 @@ pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result { let rule1_rows: Vec<(String,)> = sqlx::query_as( r#" SELECT chunk_id FROM chunks - WHERE uuid = $1 AND chunk_type = 'sentence' AND rule = 'rule_1' + WHERE file_uuid = $1 AND chunk_type = 'sentence' AND start_frame >= $2 AND end_frame <= $3 "#, @@ -99,7 +99,7 @@ pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result { let texts: Vec = sqlx::query_scalar( r#" SELECT text_content FROM chunks - WHERE uuid = $1 AND chunk_type = 'sentence' AND rule = 'rule_1' + WHERE file_uuid = $1 AND chunk_type = 'sentence' AND start_frame >= $2 AND end_frame <= $3 ORDER BY start_frame ASC @@ -135,7 +135,7 @@ pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result { ); // 4. Insert into dev.chunks - let fps_query: Option = sqlx::query_scalar("SELECT fps FROM videos WHERE uuid = $1") + let fps_query: Option = sqlx::query_scalar("SELECT fps FROM videos WHERE file_uuid = $1") .bind(file_uuid) .fetch_optional(&mut *tx) .await?; @@ -150,11 +150,11 @@ pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result { sqlx::query( r#" INSERT INTO chunks ( - uuid, chunk_id, chunk_index, chunk_type, + file_uuid, chunk_id, old_chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, content, text_content, summary_text, metadata, child_chunk_ids - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) - ON CONFLICT (uuid, chunk_id) DO NOTHING + ) VALUES ($1, $2, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + ON CONFLICT (file_uuid, old_chunk_id) DO NOTHING "#, ) .bind(file_uuid) diff --git a/src/core/db/postgres_db.rs b/src/core/db/postgres_db.rs index fafc2ee..b9a4dc3 100644 --- a/src/core/db/postgres_db.rs +++ b/src/core/db/postgres_db.rs @@ -1241,7 +1241,7 @@ impl PostgresDb { .execute(&self.pool) .await?; - sqlx::query(&format!("DELETE FROM {} WHERE uuid = $1", chunks)) + sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", chunks)) .bind(uuid) .execute(&self.pool) .await?; @@ -1279,7 +1279,7 @@ impl PostgresDb { pub async fn get_chunk_count(&self, uuid: &str) -> Result<(i64, i64)> { let chunks = schema::table_name("chunks"); let sentence_count: i64 = sqlx::query_scalar(&format!( - "SELECT COUNT(*) FROM {} WHERE uuid = $1 AND chunk_type = 'sentence'", + "SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND chunk_type = 'sentence'", chunks )) .bind(uuid) @@ -1287,7 +1287,7 @@ impl PostgresDb { .await?; let time_count: i64 = sqlx::query_scalar(&format!( - "SELECT COUNT(*) FROM {} WHERE uuid = $1 AND chunk_type = 'time_based'", + "SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND chunk_type = 'time_based'", chunks )) .bind(uuid) @@ -2567,9 +2567,9 @@ impl PostgresDb { sqlx::query(&format!( r#" - INSERT INTO {} (file_id, file_uuid, chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15, $16, $17, $18) - ON CONFLICT (file_uuid, chunk_id) DO UPDATE SET + INSERT INTO {} (file_id, file_uuid, chunk_id, old_chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids) + VALUES ($1, $2, $3, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15, $16, $17, $18) + ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE SET start_time = EXCLUDED.start_time, end_time = EXCLUDED.end_time, fps = EXCLUDED.fps, @@ -2642,9 +2642,9 @@ impl PostgresDb { sqlx::query(&format!( r#" - INSERT INTO {} (file_id, file_uuid, chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15, $16, $17, $18) - ON CONFLICT (file_uuid, chunk_id) DO UPDATE SET + INSERT INTO {} (file_id, file_uuid, chunk_id, old_chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids) + VALUES ($1, $2, $3, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15, $16, $17, $18) + ON CONFLICT (file_uuid, old_chunk_id) DO UPDATE SET start_time = EXCLUDED.start_time, end_time = EXCLUDED.end_time, fps = EXCLUDED.fps, @@ -4453,7 +4453,7 @@ impl PostgresDb { COUNT(*) as chunks_count, COALESCE(SUM(end_frame - start_frame), 0) as chunks_frames FROM {} - WHERE uuid = $1 + WHERE file_uuid = $1 "#, chunks_table )) @@ -4720,7 +4720,7 @@ impl PostgresDb { 1 - (embedding <=> $1::vector) as similarity, bbox FROM {} - WHERE file_uuid = $2 + WHERE uuid = $2 AND embedding IS NOT NULL AND 1 - (embedding <=> $1::vector) >= $3 ORDER BY embedding <=> $1::vector diff --git a/src/core/db/qdrant_db.rs b/src/core/db/qdrant_db.rs index 236c525..3674745 100644 --- a/src/core/db/qdrant_db.rs +++ b/src/core/db/qdrant_db.rs @@ -88,6 +88,44 @@ impl QdrantDb { Ok(()) } + /// 確保指定 collection 存在，不存在則自動建立 + pub async fn ensure_collection(&self, collection: &str, vector_dim: usize) -> Result<()> { + let url = format!("{}/collections/{}", self.base_url, collection); + + let exists = self + .client + .get(&url) + .header("api-key", &self.api_key) + .send() + .await + .map(|r| r.status().is_success()) + .unwrap_or(false); + + if exists { + return Ok(()); + } + + let create_url = format!("{}/collections", self.base_url); + let body = serde_json::json!({ + "vectors": { + "size": vector_dim, + "distance": "Cosine" + } + }); + + self.client + .post(&create_url) + .header("api-key", &self.api_key) + .header("Content-Type", "application/json") + .json(&body) + .send() + .await + .context(format!("Failed to create Qdrant collection: {}", collection))?; + + tracing::info!("Created Qdrant collection: {} (dim={})", collection, vector_dim); + Ok(()) + } + /// 將向量寫入指定 collection（支援多 collection） pub async fn upsert_vector_to_collection( &self, @@ -687,14 +725,13 @@ pub async fn sync_face_embeddings(file_uuid: &str) -> Result<()> { use sqlx::Row; let pool = sqlx::PgPool::connect(&DATABASE_URL).await?; - let schema = crate::core::config::DATABASE_SCHEMA.as_str(); let table = crate::core::db::schema::table_name("face_detections"); let qdrant: QdrantDb = QdrantDb::new(); let query = format!( - "SELECT id, trace_id, frame_number, embedding FROM {}.{} WHERE file_uuid = $1 AND embedding IS NOT NULL", - schema, table + "SELECT id, trace_id, frame_number, embedding FROM {} WHERE file_uuid = $1 AND embedding IS NOT NULL", + table ); let rows = sqlx::query(&query).bind(file_uuid).fetch_all(&pool).await?; diff --git a/src/core/embedding/comic_embed.rs b/src/core/embedding/comic_embed.rs index 2dbfe96..1c35ee8 100644 --- a/src/core/embedding/comic_embed.rs +++ b/src/core/embedding/comic_embed.rs @@ -19,15 +19,34 @@ struct EmbedResponse { embedding: Vec, } +#[derive(Deserialize, Debug)] +struct OpenAIEmbedResponse { + data: Vec, +} + +#[derive(Deserialize, Debug)] +struct OpenAIEmbedData { + embedding: Vec, +} + impl Embedder { pub fn new(model: String) -> Self { + Self::with_url(model, Self::default_url()) + } + + pub fn with_url(model: String, base_url: String) -> Self { Self { model, client: Client::new(), - base_url: "http://localhost:11434".to_string(), + base_url, } } + fn default_url() -> String { + std::env::var("MOMENTRY_EMBED_URL") + .unwrap_or_else(|_| "http://localhost:11434".to_string()) + } + pub async fn embed_text(&self, text: &str) -> Result> { self.embed_with_prefix(text, "").await } @@ -41,32 +60,64 @@ impl Embedder { } async fn embed_with_prefix(&self, text: &str, prefix: &str) -> Result> { - let url = format!("{}/api/embeddings", self.base_url); let prompt = format!("{}{}", prefix, text); - let response = self - .client - .post(&url) - .json(&EmbedRequest { - model: self.model.clone(), - prompt, - }) - .send() - .await - .context("Failed to send embedding request to Ollama")?; + // Ollama API: POST {base_url}/api/embeddings with {model, prompt} + // OpenAI-compatible: POST {base_url}/v1/embeddings with {input, model} + let is_openai = self.base_url.contains(":1143"); // llama.cpp ports: 11436, 11437 - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("Ollama API error ({}): {}", status, body); + if is_openai { + let url = format!("{}/v1/embeddings", self.base_url); + let body = serde_json::json!({ + "input": prompt, + "model": self.model, + }); + let response = self + .client + .post(&url) + .json(&body) + .send() + .await + .context("Failed to send embedding request")?; + + if !response.status().is_success() { + let status = response.status(); + let body_text = response.text().await.unwrap_or_default(); + anyhow::bail!("Embedding API error ({}): {}", status, body_text); + } + + let result: OpenAIEmbedResponse = response + .json() + .await + .context("Failed to parse embedding response")?; + + Ok(result.data.into_iter().next().map(|d| d.embedding).unwrap_or_default()) + } else { + let url = format!("{}/api/embeddings", self.base_url); + let response = self + .client + .post(&url) + .json(&EmbedRequest { + model: self.model.clone(), + prompt, + }) + .send() + .await + .context("Failed to send embedding request to Ollama")?; + + if !response.status().is_success() { + let status = response.status(); + let body_text = response.text().await.unwrap_or_default(); + anyhow::bail!("Ollama API error ({}): {}", status, body_text); + } + + let result: EmbedResponse = response + .json() + .await + .context("Failed to parse Ollama response")?; + + Ok(result.embedding) } - - let result: EmbedResponse = response - .json() - .await - .context("Failed to parse Ollama response")?; - - Ok(result.embedding) } pub async fn embed_chunk_content(&self, chunk: &crate::core::chunk::Chunk) -> Result> { diff --git a/src/core/processor/executor.rs b/src/core/processor/executor.rs index d756956..494ee2b 100644 --- a/src/core/processor/executor.rs +++ b/src/core/processor/executor.rs @@ -233,14 +233,24 @@ impl PythonExecutor { Ok(()) }; - // 錯誤時 rename .json.tmp → .json.err + // 錯誤時 rename .json.tmp → .json.err（若 .tmp 非有效 JSON） + // 若 .tmp 是有效 JSON，保留為 .json（保留部分結果） let mark_failed = || { if let Some(tmp) = &tmp_path { if tmp.exists() { if let Some(out) = &output_path { - let mut err_path = out.to_path_buf(); - err_path.set_extension("json.err"); - let _ = std::fs::rename(tmp, &err_path); + let is_valid = std::fs::read_to_string(tmp) + .ok() + .and_then(|c| serde_json::from_str::(&c).ok()) + .is_some(); + if is_valid { + let _ = std::fs::rename(tmp, out); + tracing::warn!("[Executor] Partial output preserved: {:?}", out); + } else { + let mut err_path = out.to_path_buf(); + err_path.set_extension("json.err"); + let _ = std::fs::rename(tmp, &err_path); + } } } } diff --git a/src/core/processor/scene_classification.rs b/src/core/processor/scene_classification.rs index 2c98c17..1f71f09 100644 --- a/src/core/processor/scene_classification.rs +++ b/src/core/processor/scene_classification.rs @@ -65,10 +65,17 @@ pub async fn process_scene_classification( }); } + let coreml_path = "/Users/accusys/models/resnet18_places365.mlpackage"; + let mut args = vec![video_path, output_path]; + if std::path::Path::new(coreml_path).exists() { + args.push("--model"); + args.push(coreml_path); + } + executor .run( "scene_classifier.py", - &[video_path, output_path], + &args, uuid, "SCENE", Some(SCENE_TIMEOUT), diff --git a/src/core/tmdb/face_agent.rs b/src/core/tmdb/face_agent.rs index c914511..3b77be7 100644 --- a/src/core/tmdb/face_agent.rs +++ b/src/core/tmdb/face_agent.rs @@ -1,146 +1,235 @@ use anyhow::{Context, Result}; use serde::Deserialize; -use tracing::{error, info}; +use std::collections::HashMap; +use tracing::{error, info, warn}; use crate::core::db::PostgresDb; -#[derive(Debug, Deserialize)] -struct FaceDetection { - face_id: String, - embedding: Vec, -} - #[derive(Debug, Deserialize)] struct TmdbIdentity { - id: i64, + id: i32, name: String, face_embedding: Vec, } -const MATCH_THRESHOLD: f32 = 0.55; - fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { - if a.len() != b.len() || a.is_empty() { - return 0.0; - } - let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum(); - let norm_a: f32 = a.iter().map(|x| x * x).sum::().sqrt(); - let norm_b: f32 = b.iter().map(|x| x * x).sum::().sqrt(); - if norm_a == 0.0 || norm_b == 0.0 { - return 0.0; - } - dot / (norm_a * norm_b) + if a.len() != b.len() || a.is_empty() { return 0.0; } + let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum(); + let na: f32 = a.iter().map(|x| x * x).sum::().sqrt(); + let nb: f32 = b.iter().map(|x| x * x).sum::().sqrt(); + if na == 0.0 || nb == 0.0 { 0.0 } else { dot / (na * nb) } } -/// Match unassigned face detections against TMDb-sourced identities. -/// For each face detection with identity_id IS NULL, compute cosine similarity -/// against all TMDb identities that have face_embedding set. -/// If similarity > MATCH_THRESHOLD, bind the face to the identity. +/// Match face detections against TMDb identities using iterative multi-angle propagation. +/// Round 1: seed match against TMDb face_embeddings (threshold 0.50) +/// Round 2+: propagate to remaining traces using matched faces as reference pub async fn match_faces_against_tmdb(db: &PostgresDb, file_uuid: &str) -> Result { - // Step 1: Fetch unassigned face detections for this file - let detections: Vec = sqlx::query_as::<_, (String, Vec)>( - "SELECT face_id, embedding FROM dev.face_detections \ - WHERE file_uuid = $1 AND identity_id IS NULL AND embedding IS NOT NULL", + let pool = db.pool(); + + // Step 1: Load TMDb identities with face embeddings + let tmdb_rows = sqlx::query_as::<_, (i32, String, Vec)>( + "SELECT id, name, face_embedding::real[] FROM dev.identities WHERE source='tmdb' AND face_embedding IS NOT NULL" + ) + .fetch_all(pool).await?; + + if tmdb_rows.is_empty() { + info!("[TKG-MATCH] No TMDb identities with face embeddings"); + return Ok(0); + } + info!("[TKG-MATCH] {} TMDb seeds loaded", tmdb_rows.len()); + + // Step 2: Load face_detections grouped by trace_id + let fd_rows = sqlx::query_as::<_, (i32, Vec)>( + "SELECT trace_id, embedding FROM dev.face_detections \ + WHERE file_uuid=$1 AND trace_id IS NOT NULL AND embedding IS NOT NULL \ + ORDER BY trace_id" ) .bind(file_uuid) - .fetch_all(db.pool()) - .await - .context("Failed to fetch unassigned face detections")? - .into_iter() - .map(|(face_id, embedding)| FaceDetection { face_id, embedding }) - .collect(); + .fetch_all(pool).await?; - if detections.is_empty() { - info!( - "[TMDB-FACE] No unassigned face detections for {}", - file_uuid - ); + if fd_rows.is_empty() { + info!("[TKG-MATCH] No face detections for {}", file_uuid); return Ok(0); } - // Step 2: Fetch TMDb identities with face embeddings - let identities: Vec = sqlx::query_as::<_, (i64, String, Vec)>( - "SELECT id, name, face_embedding::real[] FROM dev.identities \ - WHERE source = 'tmdb' AND face_embedding IS NOT NULL", - ) - .fetch_all(db.pool()) - .await - .context("Failed to fetch TMDb identities")? - .into_iter() - .map(|(id, name, emb)| TmdbIdentity { - id, - name, - face_embedding: emb, - }) - .collect(); - - if identities.is_empty() { - info!("[TMDB-FACE] No TMDb identities with face embeddings for matching"); - return Ok(0); + let mut trace_faces: HashMap>> = HashMap::new(); + for (tid, emb) in &fd_rows { + trace_faces.entry(*tid).or_default().push(emb.clone()); + } + // Dedup near-identical embeddings within trace + for faces in trace_faces.values_mut() { + faces.sort_by(|a, b| a[0].partial_cmp(&b[0]).unwrap_or(std::cmp::Ordering::Equal)); + faces.dedup_by(|a, b| cosine_similarity(a, b) > 0.99); } - info!( - "[TMDB-FACE] Matching {} face detections against {} TMDb identities", - detections.len(), - identities.len() - ); + let total = trace_faces.len(); + info!("[TKG-MATCH] {} traces with {} faces", total, fd_rows.len()); - // Step 3: For each face detection, find best matching identity - let mut bindings_created = 0usize; + // Step 3: Iterative matching + const TH: f32 = 0.50; + let mut matched: HashMap = HashMap::new(); // trace_id → (identity_id, name) - for det in &detections { - let mut best_match: Option<(i64, f32)> = None; + // Round 1: against TMDb seeds + for (&tid, faces) in &trace_faces { + let mut best_id = 0i32; + let mut best_name = String::new(); + let mut best_sim = 0.0f32; + for (id, name, tmdb_emb) in &tmdb_rows { + for face in faces { + let s = cosine_similarity(face, tmdb_emb); + if s > best_sim { best_sim = s; best_id = *id; best_name = name.clone(); } + } + } + if best_sim >= TH { + matched.insert(tid, (best_id, best_name)); + } + } + info!("[TKG-MATCH] Round 1: {} ({}/{})", matched.len(), matched.len() * 100 / total, total); - for identity in &identities { - let sim = cosine_similarity(&det.embedding, &identity.face_embedding); - if sim > MATCH_THRESHOLD { - match best_match { - Some((_, best_sim)) if sim > best_sim => { - best_match = Some((identity.id, sim)); - } - None => { - best_match = Some((identity.id, sim)); - } - _ => {} - } + // Round 2+: propagate + for round_n in 2..=10 { + let prev = matched.len(); + let mut seed_pool: HashMap>> = HashMap::new(); + for (&tid, (id, _)) in &matched { + if let Some(faces) = trace_faces.get(&tid) { + seed_pool.entry(*id).or_default().extend(faces.iter()); } } - if let Some((identity_id, similarity)) = best_match { - // Update face_detection with identity_id - let _ = sqlx::query( - "UPDATE dev.face_detections SET identity_id = $1, identity_confidence = $2 \ - WHERE file_uuid = $3 AND face_id = $4", - ) - .bind(identity_id) - .bind(similarity as f64) - .bind(file_uuid) - .bind(&det.face_id) - .execute(db.pool()) - .await - .ok(); - - // Also create identity_binding - let _ = sqlx::query( - "INSERT INTO dev.identity_bindings (identity_id, identity_type, identity_value, source, confidence) \ - VALUES ($1, 'face', $2, 'tmdb_agent', $3) \ - ON CONFLICT (identity_id, identity_type, identity_value) DO UPDATE SET confidence = EXCLUDED.confidence" - ) - .bind(identity_id) - .bind(&det.face_id) - .bind(similarity as f64) - .execute(db.pool()) - .await - .ok(); - - bindings_created += 1; + let mut new_matches: Vec<(i32, i32, String)> = Vec::new(); + for (&tid, faces) in &trace_faces { + if matched.contains_key(&tid) || faces.is_empty() { continue; } + let ref_face = &faces[0]; + let mut best_id = 0i32; + let mut best_name = String::new(); + let mut best_sim = 0.0f32; + for (&id, seed_faces) in &seed_pool { + for seed in seed_faces { + let s = cosine_similarity(ref_face, seed); + if s > best_sim { best_sim = s; best_id = id; } + } + } + if best_sim >= TH { + // Look up name for this id + for (id, name, _) in &tmdb_rows { + if *id == best_id { best_name = name.clone(); break; } + } + new_matches.push((tid, best_id, best_name)); + } } + for (tid, id, name) in new_matches { + matched.insert(tid, (id, name)); + } + let new = matched.len() - prev; + if new < 5 { break; } } - info!( - "[TMDB-FACE] Created {} face-to-TMDb bindings for {}", - bindings_created, file_uuid - ); + // Step 4: Quality control + // 4a: Remove low-confidence traces (fewer than 4 face detections) + let mut after_qc = HashMap::new(); + for (&tid, &(id, ref name)) in &matched { + let cnt: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid=$1 AND trace_id=$2" + ) + .bind(file_uuid).bind(tid) + .fetch_one(pool).await.unwrap_or(0); + if cnt >= 4 { + after_qc.insert(tid, (id, name.clone())); + } else { + info!("[TKG-QC] trace {} removed: only {} face(s), need >= 4", tid, cnt); + } + } + let matched = after_qc; + let removed_low = total - matched.len(); + if removed_low > 0 { + info!("[TKG-QC] Removed {} low-confidence traces (< 4 faces)", removed_low); + } - Ok(bindings_created) + // 4b: Temporal collision check + let removed_collisions = quality_check_temporal_collisions(pool, file_uuid).await?; + if removed_collisions > 0 { + info!("[TKG-QC] Resolved {} temporal collisions", removed_collisions); + } + + // Step 5: Update DB + let mut updated = 0usize; + for (&tid, &(id, _)) in &matched { + let r = sqlx::query( + "UPDATE dev.face_detections SET identity_id=$1 WHERE file_uuid=$2 AND trace_id=$3" + ) + .bind(id).bind(file_uuid).bind(tid) + .execute(pool).await?; + if r.rows_affected() > 0 { updated += 1; } + } + + info!("[TKG-MATCH] Done: {}/{} traces matched ({}%)", + matched.len(), total, matched.len() * 100 / total); + Ok(updated) +} + +/// Quality check: detect temporal collisions where two different traces of the same +/// identity appear in the same frame (impossible for one person). +/// Unbind the lower-confidence trace from the conflicting pair. +/// RCA reference: docs_v1.0/API_V1.0.0/INTERNAL/RCA_TRACE39_TRACE45_COLLISION_V1.0.0.md +async fn quality_check_temporal_collisions(pool: &sqlx::PgPool, file_uuid: &str) -> Result { + // Find all collision pairs: same identity, same frame, different trace + let collisions = sqlx::query_as::<_, (i32, i32, i32, i32)>( + r#" + SELECT a.identity_id, a.trace_id, b.trace_id, a.frame_number + FROM dev.face_detections a + JOIN dev.face_detections b + ON a.file_uuid = b.file_uuid + AND a.frame_number = b.frame_number + AND a.trace_id < b.trace_id + WHERE a.file_uuid = $1 + AND a.identity_id IS NOT NULL + AND a.identity_id = b.identity_id + ORDER BY a.identity_id, a.frame_number + "# + ) + .bind(file_uuid) + .fetch_all(pool).await?; + + if collisions.is_empty() { + return Ok(0); + } + + // Group collisions by (identity_id, trace_a, trace_b) and count frames + use std::collections::HashMap; + let mut collision_groups: HashMap<(i32, i32, i32), usize> = HashMap::new(); + for (id, ta, tb, _) in &collisions { + *collision_groups.entry((*id, *ta, *tb)).or_default() += 1; + } + + let mut unbound = 0usize; + for ((id, ta, tb), overlap_frames) in &collision_groups { + // Get face detection count for each trace + let cnt_a: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid=$1 AND trace_id=$2 AND identity_id=$3" + ) + .bind(file_uuid).bind(ta).bind(id) + .fetch_one(pool).await.unwrap_or(0); + + let cnt_b: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid=$1 AND trace_id=$2 AND identity_id=$3" + ) + .bind(file_uuid).bind(tb).bind(id) + .fetch_one(pool).await.unwrap_or(0); + + // Unbind the trace with fewer detections (likely the false positive) + let victim = if cnt_a <= cnt_b { *ta } else { *tb }; + let victim_cnt = if cnt_a <= cnt_b { cnt_a } else { cnt_b }; + + sqlx::query( + "UPDATE dev.face_detections SET identity_id=NULL WHERE file_uuid=$1 AND trace_id=$2" + ) + .bind(file_uuid).bind(victim) + .execute(pool).await?; + + unbound += 1; + warn!("[TKG-QC] Collision identity={}: trace {} vs trace {} ({} overlap frames). Unbound trace {} ({} detections)", + id, ta, tb, overlap_frames, victim, victim_cnt); + } + + Ok(unbound) } diff --git a/src/main.rs.backup b/src/main.rs.backup new file mode 100644 index 0000000..b606a16 --- /dev/null +++ b/src/main.rs.backup @@ -0,0 +1,3322 @@ +//! Momentry Core - Digital asset management system with video analysis and RAG +//! +//! This is the main entry point for the CLI application. + +use anyhow::{Context, Result}; +use clap::Parser; +use futures_util::StreamExt; +use std::io::Write; +use std::path::Path; +use std::str; +use std::sync::{Arc, Mutex}; + +// Local modules +mod cli; +mod processing; + +use cli::*; +use processing::*; +use processing::modules::*; + +// Core dependencies +use momentry_core::core::api_key::{ApiKeyService, ApiKeyType}; +use momentry_core::core::chunk::types::{Chunk, ChunkRule, ChunkType}; +use momentry_core::core::db::Database; +use momentry_core::core::time::FrameTime; +use momentry_core::ui::progress::{ProcessorType, ProgressState, ProgressUi}; +use momentry_core::{ + Embedder, OutputDir, PostgresDb, QdrantDb, RedisClient, VectorPayload, VideoRecord, VideoStatus, +}; + +#[derive(Debug, Clone)] +pub struct SystemResources { + pub cpu_idle_percent: f64, + pub memory_available_mb: u64, + pub memory_total_mb: u64, + pub memory_used_percent: f64, + pub gpu_available: bool, + pub gpu_type: GpuType, + pub gpu_utilization: Option, +} + +#[derive(Debug, Clone, Copy)] +pub enum GpuType { + Nvidia, + AppleMps, +} + +impl SystemResources { + pub fn check() -> Self { + let cpu_idle = Self::get_cpu_idle(); + let (mem_available, mem_total) = Self::get_memory_info(); + let mem_used_pct = if mem_total > 0 && mem_available <= mem_total { + ((mem_total - mem_available) as f64 / mem_total as f64) * 100.0 + } else if mem_total > 0 { + 100.0 + } else { + 0.0 + }; + let (gpu_available, gpu_type, gpu_util) = Self::get_gpu_info(); + + Self { + cpu_idle_percent: cpu_idle, + memory_available_mb: mem_available, + memory_total_mb: mem_total, + memory_used_percent: mem_used_pct, + gpu_available, + gpu_type, + gpu_utilization: gpu_util, + } + } + + pub fn can_parallel(&self, required_memory_mb: u64) -> bool { + const MIN_CPU_IDLE: f64 = 30.0; + const MIN_MEMORY_MB: u64 = 4096; + + self.cpu_idle_percent >= MIN_CPU_IDLE + && self.memory_available_mb >= required_memory_mb + && self.memory_available_mb >= MIN_MEMORY_MB + } + + pub fn recommend_parallel_modules(&self) -> Vec<&'static str> { + let mut recommended = Vec::new(); + + if self.gpu_available { + recommended.push("yolo"); + } + + if self.memory_available_mb >= 8192 { + recommended.push("ocr"); + recommended.push("face"); + recommended.push("pose"); + } + + recommended + } + + fn get_cpu_idle() -> f64 { + use std::process::Command; + let output = Command::new("top").args(["-l", "1", "-n", "1"]).output(); + match output { + Ok(o) => { + let s = String::from_utf8_lossy(&o.stdout); + if let Some(line) = s.lines().find(|l| l.contains("idle")) { + if let Some(pct) = line + .split_whitespace() + .find_map(|s| s.strip_suffix("%idle")) + { + pct.trim().parse().ok().unwrap_or(50.0) + } else { + 50.0 + } + } else { + 50.0 + } + } + Err(_) => 50.0, + } + } + + fn get_memory_info() -> (u64, u64) { + use std::process::Command; + let output = Command::new("sysctl").args(["hw.memsize"]).output(); + match output { + Ok(o) => { + let s = String::from_utf8_lossy(&o.stdout); + let total = s + .split_whitespace() + .nth(1) + .and_then(|v| v.parse::().ok()) + .unwrap_or(0) + / 1024 + / 1024; + + let vm_stat = Command::new("vm_stat").output(); + let available = match vm_stat { + Ok(v) => { + let vs = String::from_utf8_lossy(&v.stdout); + let mut free_pages: u64 = 0; + let mut inactive_pages: u64 = 0; + + for line in vs.lines() { + if line.contains("Pages free:") { + free_pages = line + .split_whitespace() + .last() + .and_then(|v| v.trim_end_matches('.').parse().ok()) + .unwrap_or(0); + } else if line.contains("Pages inactive:") { + inactive_pages = line + .split_whitespace() + .last() + .and_then(|v| v.trim_end_matches('.').parse().ok()) + .unwrap_or(0); + } + } + + // Pages * 4096 bytes / 1024 / 1024 = MB + (free_pages + inactive_pages) * 4096 / 1024 / 1024 + } + Err(_) => total / 4, + }; + + (available, total) + } + Err(_) => (0, 0), + } + } + + fn get_gpu_info() -> (bool, GpuType, Option) { + use std::process::Command; + + // Check NVIDIA GPU + let nvidia_output = Command::new("nvidia-smi") + .args([ + "--query-gpu=utilization.gpu", + "--format=csv,noheader,nounits", + ]) + .output(); + + if let Ok(o) = nvidia_output { + if o.status.success() { + let s = String::from_utf8_lossy(&o.stdout); + let util = s.trim().parse::().ok(); + return (true, GpuType::Nvidia, util); + } + } + + // Check Apple MPS (Metal Performance Shaders) + let mps_output = Command::new("system_profiler") + .args(["SPDisplaysDataType", "-detailLevel", "mini"]) + .output(); + + if let Ok(o) = mps_output { + let s = String::from_utf8_lossy(&o.stdout); + if s.contains("Metal") || s.contains("Apple") { + return (true, GpuType::AppleMps, Some(0.0)); + } + } + + (false, GpuType::Nvidia, None) + } +} + +impl std::fmt::Display for SystemResources { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "CPU: {:.1}% idle, Memory: {:.1}GB/{:.1}GB ({:.0}% used), GPU: {}", + self.cpu_idle_percent, + self.memory_available_mb as f64 / 1024.0, + self.memory_total_mb as f64 / 1024.0, + self.memory_used_percent, + if self.gpu_available { + format!("{:.0}% utilized", self.gpu_utilization.unwrap_or(0.0)) + } else { + "N/A".to_string() + } + ) + } +} + +fn decide_processing(json_path: &Path, force: bool, resume: bool) -> ProcessingDecision { + if !json_path.exists() { + return ProcessingDecision::Process; + } + + if force { + return ProcessingDecision::ForceReprocess; + } + + if resume { + return ProcessingDecision::ResumePartial; + } + + match check_json_completeness(json_path) { + JsonCompleteness::Complete => ProcessingDecision::SkipComplete, + JsonCompleteness::Partial { current, total } => { + eprintln!("\n⚠️ Found incomplete JSON file: {}", json_path.display()); + eprintln!( + " Progress: {}/{} ({:.1}%)", + current, + total, + (current as f64 / total as f64) * 100.0 + ); + eprintln!(" Use --resume to continue from checkpoint"); + eprintln!(" Use --force to reprocess from scratch"); + ProcessingDecision::SkipComplete + } + JsonCompleteness::Empty => ProcessingDecision::Process, + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum JsonCompleteness { + Complete, + Partial { current: u32, total: u32 }, + Empty, +} + +fn check_json_completeness(json_path: &Path) -> JsonCompleteness { + let content = match std::fs::read_to_string(json_path) { + Ok(c) => c, + Err(_) => return JsonCompleteness::Empty, + }; + + if content.trim().is_empty() { + return JsonCompleteness::Empty; + } + + let json: serde_json::Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(_) => return JsonCompleteness::Empty, + }; + + match json.get("segments") { + Some(serde_json::Value::Array(arr)) if !arr.is_empty() => JsonCompleteness::Complete, + Some(serde_json::Value::Object(obj)) => { + let current = obj.get("current").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + let total = obj.get("total").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + if total > 0 && current < total { + JsonCompleteness::Partial { current, total } + } else { + JsonCompleteness::Complete + } + } + _ => JsonCompleteness::Complete, + } +} + +async fn process_asr_module( + asr_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Asr).start(1); + } + let asr_result = momentry_core::core::processor::process_asr( + video_path, + asr_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let asr_json = serde_json::to_string_pretty(&asr_result)?; + std::fs::write(asr_path, &asr_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "asr.json"); + println!(" ✓ ASR saved: {} segments", asr_result.segments.len()); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Asr) + .complete(&format!("{} segments", asr_result.segments.len())); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_cut_module( + cut_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Cut).start(1); + } + let cut_result = momentry_core::core::processor::process_cut( + video_path, + cut_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let cut_json = serde_json::to_string_pretty(&cut_result)?; + std::fs::write(cut_path, &cut_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "cut.json"); + println!(" ✓ CUT saved: {} scenes", cut_result.scenes.len()); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Cut) + .complete(&format!("{} scenes", cut_result.scenes.len())); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_asrx_module( + asrx_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Asrx).start(1); + } + let asrx_result = momentry_core::core::processor::process_asrx( + video_path, + asrx_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let asrx_json = serde_json::to_string_pretty(&asrx_result)?; + std::fs::write(asrx_path, &asrx_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "asrx.json"); + println!(" ✓ ASRX saved: {} segments", asrx_result.segments.len()); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Asrx) + .complete(&format!("{} segments", asrx_result.segments.len())); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_yolo_module( + yolo_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Yolo).start(1); + } + let yolo_result = momentry_core::core::processor::process_yolo( + video_path, + yolo_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let yolo_json = serde_json::to_string_pretty(&yolo_result)?; + std::fs::write(yolo_path, &yolo_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "yolo.json"); + println!(" ✓ YOLO saved: {} frames", yolo_result.frame_count); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Yolo) + .complete(&format!("{} frames", yolo_result.frame_count)); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_ocr_module( + ocr_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Ocr).start(1); + } + let ocr_result = momentry_core::core::processor::process_ocr( + video_path, + ocr_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let ocr_json = serde_json::to_string_pretty(&ocr_result)?; + std::fs::write(ocr_path, &ocr_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "ocr.json"); + println!( + " ✓ OCR saved: {} frames with text", + ocr_result.frames.len() + ); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Ocr) + .complete(&format!("{} frames", ocr_result.frames.len())); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_face_module( + face_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Face).start(1); + } + let face_result = momentry_core::core::processor::process_face( + video_path, + face_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let face_json = serde_json::to_string_pretty(&face_result)?; + std::fs::write(face_path, &face_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "face.json"); + println!(" ✓ Face saved: {} frames", face_result.frames.len()); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Face) + .complete(&format!("{} frames", face_result.frames.len())); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_pose_module( + pose_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Pose).start(1); + } + let pose_result = momentry_core::core::processor::process_pose( + video_path, + pose_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let pose_json = serde_json::to_string_pretty(&pose_result)?; + std::fs::write(pose_path, &pose_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "pose.json"); + println!(" ✓ Pose saved: {} frames", pose_result.frames.len()); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Pose) + .complete(&format!("{} frames", pose_result.frames.len())); + state.stop(); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_story_module( + story_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Story).start(1); + } + let story_result = momentry_core::core::processor::process_story( + video_path, + story_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let story_json = serde_json::to_string_pretty(&story_result)?; + std::fs::write(story_path, &story_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "story.json"); + println!( + " ✓ Story saved: {} parent chunks, {} child chunks", + story_result.stats.total_parent_chunks, story_result.stats.total_child_chunks + ); + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Story).complete(&format!( + "{} parents, {} children", + story_result.stats.total_parent_chunks, story_result.stats.total_child_chunks + )); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +async fn process_caption_module( + caption_path: &Path, + video_path: &str, + uuid: &str, + progress_state: &Arc>, + ui: &Arc>>, +) -> anyhow::Result<()> { + { + let mut state = progress_state.lock().unwrap(); + state.get_processor(ProcessorType::Caption).start(1); + } + let caption_result = momentry_core::core::processor::process_caption( + video_path, + caption_path.to_str().unwrap(), + Some(uuid), + ) + .await?; + let caption_json = serde_json::to_string_pretty(&caption_result)?; + std::fs::write(caption_path, &caption_json)?; + let output_dir = OutputDir::new(); + let _ = output_dir.backup_file(uuid, "caption.json"); + println!(" ✓ Caption saved: {} frames", caption_result.total_frames); + { + let mut state = progress_state.lock().unwrap(); + state + .get_processor(ProcessorType::Caption) + .complete(&format!("{} frames", caption_result.total_frames)); + } + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + Ok(()) +} + +#[derive(Parser)] +#[command(name = "momentry")] +#[command(about = "Digital asset management system with video analysis and RAG")] +#[command(version = env!("BUILD_VERSION"))] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Register a video file + Register { + /// Video file path or URL + path: String, + }, + /// Process video (generate all JSON files) + Process { + /// UUID or path + target: String, + /// Modules to process (comma separated: asr,cut,asrx,yolo,ocr,face,pose,story,caption) + /// If not specified, processes all modules + #[arg(short, long, value_delimiter = ',')] + modules: Option>, + /// Modules to process via cloud (comma separated) + /// Example: --cloud asr,yolo + #[arg(long, value_delimiter = ',')] + cloud: Option>, + /// Force reprocess even if JSON exists (skip completeness check) + #[arg(long, default_value = "false")] + force: bool, + /// Resume from last checkpoint if processing was interrupted + #[arg(long, default_value = "false")] + resume: bool, + }, + /// Generate chunks and store in database + Chunk { + /// UUID + uuid: String, + }, + /// Generate story for cut scenes + Story { + /// UUID + uuid: String, + }, + /// Vectorize chunks + Vectorize { + /// UUID (or 'all' for all) + uuid: String, + }, + /// Play video with overlays + Play { + /// Video path or UUID + target: String, + }, + /// Start watching directories + Watch { + /// Directories to watch (comma separated) + directories: Option, + }, + /// Check system resources and recommend processing strategy + System { + /// Show detailed GPU info (NVIDIA/MPS) + #[arg(long)] + gpu: bool, + }, + /// Start API server + Server { + /// Host + #[arg(long, default_value = "127.0.0.1")] + host: String, + /// Port (defaults to MOMENTRY_SERVER_PORT env var, or3002 for production) + #[arg(long)] + port: Option, + }, + /// Start job worker + Worker { + /// Max concurrent processors + #[arg(long)] + max_concurrent: Option, + /// Poll interval in seconds + #[arg(long)] + poll_interval: Option, + /// Batch size + #[arg(long)] + batch_size: Option, + }, + /// Query using RAG + Query { + /// Query text + query: String, + }, + /// Lookup UUID from path + Lookup { + /// File path + path: String, + }, + /// Resolve path from UUID + Resolve { + /// UUID + uuid: String, + }, + /// Generate thumbnails for videos + Thumbnails { + /// UUID (optional, generates for all if not specified) + uuid: Option, + /// Number of thumbnails per video + #[arg(short, long, default_value = "6")] + count: u32, + }, + /// Show storage status report + Status { + /// UUID (optional, shows all if not specified) + uuid: Option, + }, + /// Manage output backups + Backup { + /// Action: list, cleanup + action: String, + /// Days to keep (for cleanup) + days: Option, + }, + /// Manage API keys + ApiKey { + /// Action: create, list, validate, revoke, rotate, stats + #[arg(value_enum)] + action: ApiKeyAction, + /// Key name (for create) + name: Option, + /// Key type (system, user, service, integration, emergency) + #[arg(long)] + key_type: Option, + /// TTL in days (for create) + #[arg(long)] + ttl: Option, + /// API key to validate/revoke + #[arg(long)] + key: Option, + }, + /// Manage Gitea API tokens + Gitea { + /// Action: create, list, delete, verify + #[arg(value_enum)] + action: GiteaAction, + /// Gitea username + #[arg(long)] + username: Option, + /// Gitea password (for create/list/delete) + #[arg(long)] + password: Option, + /// Token name (for create/delete) + #[arg(long)] + token_name: Option, + /// Token scopes (comma separated: read:repository,write:issue) + #[arg(long)] + scopes: Option, + }, + /// Manage n8n API keys + N8n { + /// Action: create, list, delete, verify + #[arg(value_enum)] + action: N8nAction, + /// n8n API key (for create/list/delete) + #[arg(long)] + api_key: Option, + /// API key label (for create/delete) + #[arg(long)] + label: Option, + /// Expiration days (for create) + #[arg(long)] + expires_in_days: Option, + }, +} + +#[derive(clap::ValueEnum, Clone)] +enum ApiKeyAction { + Create, + List, + Validate, + Revoke, + Rotate, + Stats, +} + +#[derive(clap::ValueEnum, Clone)] +enum GiteaAction { + Create, + List, + Delete, + Verify, +} + +#[derive(clap::ValueEnum, Clone)] +enum N8nAction { + Create, + List, + Delete, + Verify, +} + +#[tokio::main] +async fn main() -> Result<()> { + dotenv::dotenv().ok(); + tracing_subscriber::fmt::init(); + + let cli = Cli::parse(); + + match cli.command { + Commands::Register { path } => { + println!("Registering: {}", path); + + // Compute UUID + let uuid = momentry_core::uuid::compute_uuid_from_path(&path); + println!("UUID: {}", uuid); + + // Run ffprobe + let probe_result = momentry_core::core::probe::probe_video(&path)?; + + println!("\nVideo probe results:"); + let duration = probe_result + .format + .duration + .as_ref() + .and_then(|s| s.parse::().ok()) + .unwrap_or(0.0); + println!(" Duration: {}s", duration); + if let Some(size) = &probe_result.format.size { + println!(" Size: {}", size); + } + + let mut width = 0u32; + let mut height = 0u32; + let mut fps = 0.0; + + for stream in &probe_result.streams { + if stream.codec_type.as_deref() == Some("video") { + width = stream.width.unwrap_or(0); + height = stream.height.unwrap_or(0); + if let Some(fps_str) = &stream.r_frame_rate { + if let Some((num, den)) = fps_str.split_once('/') { + if let (Ok(n), Ok(d)) = (num.parse::(), den.parse::()) { + if d > 0.0 { + fps = n / d; + } + } + } + } + println!(" Video: {}x{}", width, height); + if let Some(fps) = &stream.r_frame_rate { + println!(" FPS: {}", fps); + } + } + if stream.codec_type.as_deref() == Some("audio") { + println!(" Audio: {} channels", stream.channels.unwrap_or(0)); + if let Some(sr) = &stream.sample_rate { + println!(" Sample Rate: {}", sr); + } + } + } + + // Save probe JSON to file + let file_manager = momentry_core::FileManager::new(std::path::PathBuf::from(".")); + let json_str = serde_json::to_string_pretty(&probe_result)?; + let json_path = file_manager.save_json(&uuid, "probe", &json_str)?; + println!("\nProbe JSON saved to: {:?}", json_path); + + // Store in PostgreSQL + println!("\nStoring in database..."); + let db = PostgresDb::init().await?; + let file_path = Path::new(&path) + .canonicalize() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|_| path.clone()); + let file_name = Path::new(&path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + + let record = VideoRecord { + id: 0, + uuid: uuid.clone(), + file_path, + file_name, + duration, + width, + height, + fps, + probe_json: Some(json_str), + storage: Default::default(), + status: VideoStatus::Pending, + user_id: None, + job_id: None, + created_at: String::new(), + registration_time: None, + }; + + let video_id = db.register_video(&record).await?; + println!("Video registered with ID: {}", video_id); + + Ok(()) + } + Commands::Process { + target, + modules, + cloud, + force, + resume, + } => { + println!("Processing: {}", target); + println!(" force: {}, resume: {}", force, resume); + + // Parse selected modules + let selected_modules: Option> = modules.as_ref().map(|m| { + m.iter() + .filter_map(|name| { + let name_lower = name.to_lowercase(); + match name_lower.as_str() { + "asr" => Some(ProcessorType::Asr), + "cut" => Some(ProcessorType::Cut), + "asrx" => Some(ProcessorType::Asrx), + "yolo" => Some(ProcessorType::Yolo), + "ocr" => Some(ProcessorType::Ocr), + "face" => Some(ProcessorType::Face), + "pose" => Some(ProcessorType::Pose), + "story" => Some(ProcessorType::Story), + "caption" => Some(ProcessorType::Caption), + _ => { + eprintln!("Unknown module: {}", name); + None + } + } + }) + .collect() + }); + + // Parse cloud modules + let cloud_modules: Vec = cloud + .as_ref() + .map(|c| { + c.iter() + .filter_map(|name| { + let name_lower = name.to_lowercase(); + match name_lower.as_str() { + "asr" => Some(ProcessorType::Asr), + "cut" => Some(ProcessorType::Cut), + "asrx" => Some(ProcessorType::Asrx), + "yolo" => Some(ProcessorType::Yolo), + "ocr" => Some(ProcessorType::Ocr), + "face" => Some(ProcessorType::Face), + "pose" => Some(ProcessorType::Pose), + "story" => Some(ProcessorType::Story), + "caption" => Some(ProcessorType::Caption), + _ => { + eprintln!("Unknown cloud module: {}", name); + None + } + } + }) + .collect() + }) + .unwrap_or_default(); + + if let Some(ref mods) = selected_modules { + println!( + " Modules: {}", + mods.iter() + .map(|m| m.to_string()) + .collect::>() + .join(", ") + ); + } else { + println!(" Modules: ALL"); + } + + if !cloud_modules.is_empty() { + println!( + " Cloud: {}", + cloud_modules + .iter() + .map(|m| m.to_string()) + .collect::>() + .join(", ") + ); + } + + let processing_mode = if force { + "FORCE (reprocess all)" + } else if resume { + "RESUME (continue from checkpoint)" + } else { + "SMART (skip complete, resume partial)" + }; + println!(" Mode: {}", processing_mode); + + // Compute UUID if path is given + let uuid = if target.len() == 16 && !target.contains('/') { + target.clone() + } else { + momentry_core::uuid::compute_uuid_from_path(&target) + }; + + // Get video from database + let db = PostgresDb::init().await?; + let video = db + .get_video_by_uuid(&uuid) + .await? + .ok_or_else(|| anyhow::anyhow!("Video not found: {}", uuid))?; + + let video_path = &video.file_path; + let video_name = video.file_name.clone(); + let _file_manager = momentry_core::FileManager::new(std::path::PathBuf::from(".")); + + // Initialize output directory + let output_dir = OutputDir::new(); + output_dir.ensure_dir()?; + println!("Output directory: {:?}", output_dir.get_base_path()); + + // Initialize progress UI + let progress_state = Arc::new(Mutex::new(ProgressState::new(&video_name))); + progress_state.lock().unwrap().start(); + + // Helper closure to check if a module should be processed + let should_process = |module: ProcessorType| -> bool { + selected_modules + .as_ref() + .map(|mods| mods.contains(&module)) + .unwrap_or(true) + }; + + // Helper closure to check if a module should run in the cloud + let is_cloud = |module: ProcessorType| -> bool { cloud_modules.contains(&module) }; + + // Create UI and wrap in Arc for sharing with Redis subscriber + let ui = Arc::new(Mutex::new(ProgressUi::new(&video_name).ok())); + if let Some(ref mut ui) = *ui.lock().unwrap() { + let _ = ui.render(); + } + + // Spawn Redis subscriber for real-time progress updates + let redis_progress_state = progress_state.clone(); + let redis_ui = ui.clone(); + let redis_uuid = uuid.clone(); + let redis_handle = tokio::spawn(async move { + if let Ok(redis_client) = momentry_core::core::db::RedisClient::new() { + loop { + if let Ok(mut pubsub) = redis_client.subscribe_progress(&redis_uuid).await { + let mut stream = pubsub.on_message(); + while let Some(msg) = stream.next().await { + if let Ok(payload) = msg.get_payload::() { + if let Ok(progress_msg) = + serde_json::from_str::< + momentry_core::core::db::ProgressMessage, + >(&payload) + { + let mut state = redis_progress_state.lock().unwrap(); + state.update_from_redis( + &progress_msg.msg_type, + &progress_msg.processor, + progress_msg.data.current, + progress_msg.data.total, + progress_msg.data.message.as_deref(), + ); + + // Store progress in Redis Hash for HTTP API + let uuid = progress_msg.uuid.clone(); + let processor = progress_msg.processor.clone(); + let msg_type = progress_msg.msg_type.clone(); + let current = progress_msg.data.current; + let total = progress_msg.data.total; + let message = progress_msg.data.message.clone(); + + tokio::spawn(async move { + if let Ok(redis_client) = + momentry_core::core::db::RedisClient::new() + { + if let Ok(mut conn) = redis_client.get_conn().await + { + let prefix = momentry_core::core::config::REDIS_KEY_PREFIX.as_str(); + let key = format!( + "{}job:{}:processor:{}", + prefix, uuid, processor + ); + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("status") + .arg(&msg_type) + .query_async(&mut conn) + .await + .unwrap_or(()); + if let Some(c) = current { + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("current") + .arg(c) + .query_async(&mut conn) + .await + .unwrap_or(()); + } + if let Some(t) = total { + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("total") + .arg(t) + .query_async(&mut conn) + .await + .unwrap_or(()); + } + if let Some(ref m) = message { + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("message") + .arg(m) + .query_async(&mut conn) + .await + .unwrap_or(()); + } + let _: () = redis::cmd("EXPIRE") + .arg(&key) + .arg(86400i64) + .query_async(&mut conn) + .await + .unwrap_or(()); + } + } + }); + + // Trigger UI render on progress update + if let Some(ref mut ui) = *redis_ui.lock().unwrap() { + let _ = ui.render(); + } + } + } + } + } + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } + } + }); + + // Process ASR (Automatic Speech Recognition) + if should_process(ProcessorType::Asr) { + let asr_path = output_dir.get_output_path(&uuid, "asr.json"); + let decision = decide_processing(&asr_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nASR: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nASR: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&asr_path).ok(); + if is_cloud(ProcessorType::Asr) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_asr_module(&asr_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nASR: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Asr) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_asr_module(&asr_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Asr) { + println!("\nASR: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nASR: ⚙️ Processing..."); + process_asr_module(&asr_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + } + } + + // Update storage status + db.update_storage_status(&uuid, "fs_json", true).await?; + + // Process CUT (scene detection) + if should_process(ProcessorType::Cut) { + let cut_path = output_dir.get_output_path(&uuid, "cut.json"); + let decision = decide_processing(&cut_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nCUT: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nCUT: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&cut_path).ok(); + if is_cloud(ProcessorType::Cut) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_cut_module(&cut_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nCUT: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Cut) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_cut_module(&cut_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Cut) { + println!("\nCUT: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nCUT: ⚙️ Processing..."); + process_cut_module(&cut_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + } + } + + // Process ASRX (speaker diarization) + if should_process(ProcessorType::Asrx) { + let asrx_path = output_dir.get_output_path(&uuid, "asrx.json"); + let decision = decide_processing(&asrx_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nASRX: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nASRX: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&asrx_path).ok(); + if is_cloud(ProcessorType::Asrx) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_asrx_module( + &asrx_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nASRX: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Asrx) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_asrx_module( + &asrx_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Asrx) { + println!("\nASRX: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nASRX: ⚙️ Processing..."); + process_asrx_module( + &asrx_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + } + } + + // Process YOLO (object detection) + if should_process(ProcessorType::Yolo) { + let yolo_path = output_dir.get_output_path(&uuid, "yolo.json"); + let decision = decide_processing(&yolo_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nYOLO: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nYOLO: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&yolo_path).ok(); + if is_cloud(ProcessorType::Yolo) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_yolo_module( + &yolo_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nYOLO: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Yolo) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_yolo_module( + &yolo_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Yolo) { + println!("\nYOLO: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nYOLO: ⚙️ Processing..."); + process_yolo_module( + &yolo_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + } + } + + // Process OCR (text recognition) + if should_process(ProcessorType::Ocr) { + let ocr_path = output_dir.get_output_path(&uuid, "ocr.json"); + let decision = decide_processing(&ocr_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nOCR: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nOCR: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&ocr_path).ok(); + if is_cloud(ProcessorType::Ocr) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_ocr_module(&ocr_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nOCR: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Ocr) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_ocr_module(&ocr_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Ocr) { + println!("\nOCR: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nOCR: ⚙️ Processing..."); + process_ocr_module(&ocr_path, video_path, &uuid, &progress_state, &ui) + .await?; + } + } + } + } + + // Process Face (face detection) + if should_process(ProcessorType::Face) { + let face_path = output_dir.get_output_path(&uuid, "face.json"); + let decision = decide_processing(&face_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nFace: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nFace: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&face_path).ok(); + if is_cloud(ProcessorType::Face) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_face_module( + &face_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nFace: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Face) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_face_module( + &face_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Face) { + println!("\nFace: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nFace: ⚙️ Processing..."); + process_face_module( + &face_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + } + } + + // Process Pose (pose estimation) + if should_process(ProcessorType::Pose) { + let pose_path = output_dir.get_output_path(&uuid, "pose.json"); + let decision = decide_processing(&pose_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nPose: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nPose: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&pose_path).ok(); + if is_cloud(ProcessorType::Pose) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_pose_module( + &pose_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nPose: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Pose) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_pose_module( + &pose_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Pose) { + println!("\nPose: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nPose: ⚙️ Processing..."); + process_pose_module( + &pose_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + } + } + + // Process Story (video narrative) + if should_process(ProcessorType::Story) { + let story_path = output_dir.get_output_path(&uuid, "story.json"); + let decision = decide_processing(&story_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nStory: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nStory: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&story_path).ok(); + if is_cloud(ProcessorType::Story) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_story_module( + &story_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nStory: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Story) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_story_module( + &story_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Story) { + println!("\nStory: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nStory: ⚙️ Processing..."); + process_story_module( + &story_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + } + } + + // Process Caption (image captions) + if should_process(ProcessorType::Caption) { + let caption_path = output_dir.get_output_path(&uuid, "caption.json"); + let decision = decide_processing(&caption_path, force, resume); + + match decision { + ProcessingDecision::SkipComplete => { + println!("\nCaption: ✓ Already complete, skipping"); + } + ProcessingDecision::ForceReprocess => { + println!("\nCaption: ⟳ Force reprocessing from scratch..."); + std::fs::remove_file(&caption_path).ok(); + if is_cloud(ProcessorType::Caption) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_caption_module( + &caption_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::ResumePartial => { + println!("\nCaption: ↻ Resuming from checkpoint..."); + if is_cloud(ProcessorType::Caption) { + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + process_caption_module( + &caption_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + ProcessingDecision::Process => { + if is_cloud(ProcessorType::Caption) { + println!("\nCaption: ☁️ Running via cloud..."); + println!(" [Cloud processing not implemented yet - run locally]"); + } else { + println!("\nCaption: ⚙️ Processing..."); + process_caption_module( + &caption_path, + video_path, + &uuid, + &progress_state, + &ui, + ) + .await?; + } + } + } + } + + // TODO: Store pre_chunks and frames to database + + // Stop Redis subscriber + redis_handle.abort(); + + println!("\n✓ Process stage completed!"); + if should_process(ProcessorType::Asr) { + let path = output_dir.get_output_path(&uuid, "asr.json"); + println!(" - ASR JSON: {}", path.display()); + } + if should_process(ProcessorType::Cut) { + let path = output_dir.get_output_path(&uuid, "cut.json"); + println!(" - CUT JSON: {}", path.display()); + } + if should_process(ProcessorType::Asrx) { + let path = output_dir.get_output_path(&uuid, "asrx.json"); + println!(" - ASRX JSON: {}", path.display()); + } + if should_process(ProcessorType::Yolo) { + let path = output_dir.get_output_path(&uuid, "yolo.json"); + println!(" - YOLO JSON: {}", path.display()); + } + if should_process(ProcessorType::Ocr) { + let path = output_dir.get_output_path(&uuid, "ocr.json"); + println!(" - OCR JSON: {}", path.display()); + } + if should_process(ProcessorType::Face) { + let path = output_dir.get_output_path(&uuid, "face.json"); + println!(" - Face JSON: {}", path.display()); + } + if should_process(ProcessorType::Pose) { + let path = output_dir.get_output_path(&uuid, "pose.json"); + println!(" - Pose JSON: {}", path.display()); + } + if should_process(ProcessorType::Story) { + let path = output_dir.get_output_path(&uuid, "story.json"); + println!(" - Story JSON: {}", path.display()); + } + if should_process(ProcessorType::Caption) { + let path = output_dir.get_output_path(&uuid, "caption.json"); + println!(" - Caption JSON: {}", path.display()); + } + + Ok(()) + } + Commands::Chunk { uuid } => { + println!("Chunking: {}", uuid); + + let db = PostgresDb::init().await?; + let video = db + .get_video_by_uuid(&uuid) + .await? + .ok_or_else(|| anyhow::anyhow!("Video not found: {}", uuid))?; + + let file_id = video.id; + let fps = video.fps; + + // ========== Read all JSON files ========== + + // Read ASR JSON + let asr_path = format!("{}.asr.json", uuid); + let asr_json = std::fs::read_to_string(&asr_path) + .context("ASR file not found. Run 'process' first.")?; + let asr_result: momentry_core::core::processor::asr::AsrResult = + serde_json::from_str(&asr_json)?; + println!("Loaded ASR: {} segments", asr_result.segments.len()); + + // Read CUT JSON + let cut_path = format!("{}.cut.json", uuid); + let cut_json = std::fs::read_to_string(&cut_path) + .context("CUT file not found. Run 'process' first.")?; + let cut_result: momentry_core::core::processor::cut::CutResult = + serde_json::from_str(&cut_json)?; + println!("Loaded CUT: {} scenes", cut_result.scenes.len()); + + // Read YOLO JSON (optional) + let yolo_path = format!("{}.yolo.json", uuid); + let yolo_result = match std::fs::read_to_string(&yolo_path) { + Ok(yolo_json) => match serde_json::from_str::< + momentry_core::core::processor::yolo::YoloResult, + >(&yolo_json) + { + Ok(result) => { + println!("Loaded YOLO: {} frames", result.frames.len()); + result + } + Err(e) => { + println!("Warning: Failed to parse YOLO JSON: {}. Skipping YOLO.", e); + momentry_core::core::processor::yolo::YoloResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }, + Err(_) => { + println!("Warning: YOLO file not found. Skipping YOLO."); + momentry_core::core::processor::yolo::YoloResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }; + + // Read OCR JSON (optional) + let ocr_path = format!("{}.ocr.json", uuid); + let ocr_result = match std::fs::read_to_string(&ocr_path) { + Ok(ocr_json) => match serde_json::from_str::< + momentry_core::core::processor::ocr::OcrResult, + >(&ocr_json) + { + Ok(result) => { + println!("Loaded OCR: {} frames", result.frames.len()); + result + } + Err(e) => { + println!("Warning: Failed to parse OCR JSON: {}. Skipping OCR.", e); + momentry_core::core::processor::ocr::OcrResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }, + Err(_) => { + println!("Warning: OCR file not found. Skipping OCR."); + momentry_core::core::processor::ocr::OcrResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }; + + // Read Face JSON (optional) + let face_path = format!("{}.face.json", uuid); + let face_result = match std::fs::read_to_string(&face_path) { + Ok(face_json) => match serde_json::from_str::< + momentry_core::core::processor::face::FaceResult, + >(&face_json) + { + Ok(result) => { + println!("Loaded Face: {} frames", result.frames.len()); + result + } + Err(e) => { + println!("Warning: Failed to parse Face JSON: {}. Skipping Face.", e); + momentry_core::core::processor::face::FaceResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }, + Err(_) => { + println!("Warning: Face file not found. Skipping Face."); + momentry_core::core::processor::face::FaceResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }; + + // Read Pose JSON (optional) + let pose_path = format!("{}.pose.json", uuid); + let pose_result = match std::fs::read_to_string(&pose_path) { + Ok(pose_json) => match serde_json::from_str::< + momentry_core::core::processor::pose::PoseResult, + >(&pose_json) + { + Ok(result) => { + println!("Loaded Pose: {} frames", result.frames.len()); + result + } + Err(e) => { + println!("Warning: Failed to parse Pose JSON: {}. Skipping Pose.", e); + momentry_core::core::processor::pose::PoseResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }, + Err(_) => { + println!("Warning: Pose file not found. Skipping Pose."); + momentry_core::core::processor::pose::PoseResult { + frame_count: 0, + fps: 0.0, + frames: vec![], + } + } + }; + + // Read ASRX JSON (optional) + let asrx_path = format!("{}.asrx.json", uuid); + let asrx_result = match std::fs::read_to_string(&asrx_path) { + Ok(asrx_json) => match serde_json::from_str::< + momentry_core::core::processor::asrx::AsrxResult, + >(&asrx_json) + { + Ok(result) => { + println!("Loaded ASRX: {} segments", result.segments.len()); + result + } + Err(e) => { + println!("Warning: Failed to parse ASRX JSON: {}. Skipping ASRX.", e); + momentry_core::core::processor::asrx::AsrxResult { + language: None, + segments: vec![], + } + } + }, + Err(_) => { + println!("Warning: ASRX file not found. Skipping ASRX."); + momentry_core::core::processor::asrx::AsrxResult { + language: None, + segments: vec![], + } + } + }; + + // ========== Store pre_chunks (from ASR, CUT) ========== + + println!("\nStoring pre_chunks..."); + + // Store ASR sentence pre_chunks + let mut asr_pre_chunk_ids = Vec::new(); + for seg in asr_result.segments.iter() { + let start_frame = FrameTime::from_seconds(seg.start, fps).frames(); + let end_frame = FrameTime::from_seconds(seg.end, fps).frames(); + let pre_chunk = momentry_core::core::db::postgres_db::PreChunk { + id: 0, + file_id, + source_type: "asr".to_string(), + source_file: Some(asr_path.clone()), + chunk_type: "sentence".to_string(), + start_frame, + end_frame, + fps, + raw_json: serde_json::json!({"text": seg.text}), + text_content: Some(seg.text.clone()), + processed: false, + chunk_id: None, + created_at: String::new(), + }; + let pre_chunk_id = db.store_pre_chunk(&pre_chunk).await?; + asr_pre_chunk_ids.push(pre_chunk_id); + } + + // Store CUT scene pre_chunks + let mut cut_pre_chunk_ids = Vec::new(); + for scene in &cut_result.scenes { + let pre_chunk = momentry_core::core::db::postgres_db::PreChunk { + id: 0, + file_id, + source_type: "cut".to_string(), + source_file: Some(cut_path.clone()), + chunk_type: "cut".to_string(), + start_frame: scene.start_frame as i64, + end_frame: scene.end_frame as i64, + fps, + raw_json: serde_json::json!({ + "scene_number": scene.scene_number, + }), + text_content: None, + processed: false, + chunk_id: None, + created_at: String::new(), + }; + let pre_chunk_id = db.store_pre_chunk(&pre_chunk).await?; + cut_pre_chunk_ids.push(pre_chunk_id); + } + + // Store time-based pre_chunks (every 10 seconds) + let duration = video.duration; + let mut time_pre_chunk_ids = Vec::new(); + let mut time_start = 0.0; + while time_start < duration { + let time_end = (time_start + 10.0).min(duration); + let start_frame = FrameTime::from_seconds(time_start, fps).frames(); + let end_frame = FrameTime::from_seconds(time_end, fps).frames(); + + let pre_chunk = momentry_core::core::db::postgres_db::PreChunk { + id: 0, + file_id, + source_type: "time".to_string(), + source_file: None, + chunk_type: "time".to_string(), + start_frame, + end_frame, + fps, + raw_json: serde_json::json!({"interval": 10.0}), + text_content: None, + processed: false, + chunk_id: None, + created_at: String::new(), + }; + let pre_chunk_id = db.store_pre_chunk(&pre_chunk).await?; + time_pre_chunk_ids.push(pre_chunk_id); + time_start = time_end; + } + + println!( + "Stored pre_chunks: {} asr + {} cut + {} time", + asr_result.segments.len(), + cut_result.scenes.len(), + time_pre_chunk_ids.len() + ); + + // ========== Store frames (from YOLO, OCR, Face) ========== + + println!("\nStoring frames..."); + + // Group YOLO, OCR, Face results by frame_number + let mut frame_data: std::collections::HashMap< + u64, + momentry_core::core::processor::yolo::YoloFrame, + > = std::collections::HashMap::new(); + for frame in &yolo_result.frames { + frame_data.insert(frame.frame, frame.clone()); + } + + let mut ocr_by_frame: std::collections::HashMap< + u64, + momentry_core::core::processor::ocr::OcrFrame, + > = std::collections::HashMap::new(); + for frame in &ocr_result.frames { + ocr_by_frame.insert(frame.frame, frame.clone()); + } + + let mut face_by_frame: std::collections::HashMap< + u64, + momentry_core::core::processor::face::FaceFrame, + > = std::collections::HashMap::new(); + for frame in &face_result.frames { + face_by_frame.insert(frame.frame, frame.clone()); + } + + let mut pose_by_frame: std::collections::HashMap< + u64, + momentry_core::core::processor::pose::PoseFrame, + > = std::collections::HashMap::new(); + for frame in &pose_result.frames { + pose_by_frame.insert(frame.frame, frame.clone()); + } + + // Store frames (merge data from YOLO, OCR, Face, Pose) + let mut all_frames: Vec = frame_data + .keys() + .cloned() + .chain(ocr_by_frame.keys().cloned()) + .chain(face_by_frame.keys().cloned()) + .chain(pose_by_frame.keys().cloned()) + .collect(); + all_frames.sort(); + all_frames.dedup(); + + for frame_num in &all_frames { + let timestamp = (*frame_num as f64) / fps; + let yolo_frame = frame_data.get(frame_num); + let ocr_frame = ocr_by_frame.get(frame_num); + let face_frame = face_by_frame.get(frame_num); + let pose_frame = pose_by_frame.get(frame_num); + + let frame = momentry_core::core::db::postgres_db::Frame { + id: 0, + file_id, + frame_number: *frame_num as i64, + timestamp, + fps, + yolo_objects: yolo_frame.map(|f| serde_json::json!(&f.objects)), + ocr_results: ocr_frame.map(|f| serde_json::json!(&f.texts)), + face_results: face_frame.map(|f| serde_json::json!(&f.faces)), + pose_results: pose_frame.map(|f| serde_json::json!(&f.persons)), + frame_path: None, + created_at: String::new(), + }; + db.store_frame(&frame).await?; + } + + println!("Stored {} frames", all_frames.len()); + + // ========== Create chunks ========== + + println!("\nCreating chunks..."); + + // Rule 1: Direct conversion (sentence pre_chunk -> sentence chunk) + // Merge ASRX speaker_id by time overlap + let mut sentence_chunks = Vec::new(); + for (i, seg) in asr_result.segments.iter().enumerate() { + let pre_chunk_id = asr_pre_chunk_ids.get(i).copied().unwrap_or(0); + + // Find matching ASRX segment by time overlap + let speaker_id = asrx_result + .segments + .iter() + .find(|ax| { + // Overlap: ASRX segment overlaps with ASR segment + ax.start <= seg.end && ax.end >= seg.start + }) + .and_then(|ax| ax.speaker_id.clone()); + + let content = if let Some(ref sid) = speaker_id { + serde_json::json!({ + "text": seg.text, + "speaker_id": sid, + }) + } else { + serde_json::json!({ + "text": seg.text, + }) + }; + + let mut chunk = Chunk::from_seconds( + file_id as i32, + uuid.clone(), + i as u32, + ChunkType::Sentence, + ChunkRule::Rule1, + seg.start, + seg.end, + fps, + content, + ) + .with_text_content(seg.text.clone()) + .with_pre_chunk_ids(vec![pre_chunk_id as i32]); + + // Add ASRX metadata if available + if speaker_id.is_some() { + chunk = chunk.with_metadata(serde_json::json!({ + "language": asr_result.language, + "language_probability": asr_result.language_probability, + "speaker_matched": true, + })); + } + + sentence_chunks.push(chunk); + } + + if !asrx_result.segments.is_empty() { + let matched = sentence_chunks + .iter() + .filter(|c| { + c.content + .get("speaker_id") + .and_then(|v| v.as_str()) + .is_some() + }) + .count(); + println!( + " ASRX merge: {}/{} sentence chunks matched to speakers", + matched, + sentence_chunks.len() + ); + } + + // Rule 1: CUT chunks + let mut cut_chunks = Vec::new(); + for (i, scene) in cut_result.scenes.iter().enumerate() { + let pre_chunk_id = cut_pre_chunk_ids.get(i).copied().unwrap_or(0); + let chunk = Chunk::from_seconds( + file_id as i32, + uuid.clone(), + i as u32, + ChunkType::Cut, + ChunkRule::Rule1, + scene.start_time, + scene.end_time, + fps, + serde_json::json!({ + "scene_number": scene.scene_number, + }), + ) + .with_pre_chunk_ids(vec![pre_chunk_id as i32]); + cut_chunks.push(chunk); + } + + // Rule 1: Time-based chunks + let splitter = momentry_core::core::chunk::ChunkSplitter::new(10.0); + let mut time_chunks = Vec::new(); + let time_chunk_list = splitter.split_time_based(&uuid, video.duration); + for (i, tc) in time_chunk_list.iter().enumerate() { + let pre_chunk_id = time_pre_chunk_ids.get(i).copied().unwrap_or(0); + let chunk = Chunk::new( + file_id as i32, + uuid.clone(), + i as u32, + ChunkType::TimeBased, + ChunkRule::Rule1, + tc.start_frame, + tc.end_frame, + fps, + serde_json::json!({"interval": 10.0}), + ) + .with_pre_chunk_ids(vec![pre_chunk_id as i32]); + time_chunks.push(chunk); + } + + // Store chunks + println!( + "Storing {} sentence chunks (rule_1)...", + sentence_chunks.len() + ); + for chunk in &sentence_chunks { + db.store_chunk(chunk).await?; + } + + println!("Storing {} cut chunks (rule_1)...", cut_chunks.len()); + for chunk in &cut_chunks { + db.store_chunk(chunk).await?; + } + + println!( + "Storing {} time-based chunks (rule_1)...", + time_chunks.len() + ); + for chunk in &time_chunks { + db.store_chunk(chunk).await?; + } + + let total_chunks = sentence_chunks.len() + cut_chunks.len() + time_chunks.len(); + + // Update storage status + db.update_storage_status(&uuid, "psql_chunk", true).await?; + + println!("\n✓ Chunk stage completed!"); + println!( + " - pre_chunks: {} (asr + cut + time)", + asr_result.segments.len() + cut_result.scenes.len() + time_pre_chunk_ids.len() + ); + println!(" - frames: {}", all_frames.len()); + println!(" - chunks: {} (sentence + cut + time_based)", total_chunks); + + Ok(()) + } + Commands::Story { uuid } => { + println!("Generating story for: {}", uuid); + + let db = PostgresDb::init().await?; + let video = db + .get_video_by_uuid(&uuid) + .await? + .ok_or_else(|| anyhow::anyhow!("Video not found: {}", uuid))?; + + let file_id = video.id; + let _fps = video.fps; + let duration = video.duration; + + // Get all chunks + let all_chunks = db.get_chunks_by_uuid(&uuid).await?; + + // Try cut chunks first, fall back to sentence chunks + let mut story_chunks: Vec<&Chunk> = all_chunks + .iter() + .filter(|c| c.chunk_type == ChunkType::Cut) + .collect(); + + let story_type = if story_chunks.is_empty() { + // Fall back to sentence chunks + story_chunks = all_chunks + .iter() + .filter(|c| c.chunk_type == ChunkType::Sentence && c.text_content.is_some()) + .collect(); + "sentence" + } else { + "cut" + }; + + if story_chunks.is_empty() { + println!("No story chunks found. Run 'chunk' command first."); + return Ok(()); + } + + println!("Found {} {} scenes", story_chunks.len(), story_type); + + // Generate story for each scene + for (i, story_chunk) in story_chunks.iter().enumerate() { + println!("\n=== Scene {} ===", i + 1); + println!( + "Time: {:.2}s - {:.2}s", + story_chunk.start_time().seconds(), + story_chunk.end_time().seconds() + ); + + // Get context: expand time range by 5 seconds before and after + let context_start = (story_chunk.start_time().seconds() - 5.0).max(0.0); + let context_end = (story_chunk.end_time().seconds() + 5.0).min(duration); + + // Get chunks in context range (sentence chunks with ASR text) + let context_chunks = db + .get_chunks_by_time_range(file_id, context_start, context_end) + .await?; + + // Get frames in context range + let context_frames = db + .get_frames_by_time_range(file_id, context_start, context_end) + .await?; + + // Build story + let mut story = String::new(); + story.push_str(&format!( + "Scene {} ({:.1}s - {:.1}s)\n\n", + i + 1, + story_chunk.start_time().seconds(), + story_chunk.end_time().seconds() + )); + + // Add audio/text content + let sentence_chunks: Vec<&Chunk> = context_chunks + .iter() + .filter(|c| c.chunk_type == ChunkType::Sentence) + .collect(); + + if !sentence_chunks.is_empty() { + story.push_str("【Speech】\n"); + for sc in &sentence_chunks { + if let Some(text) = &sc.text_content { + story.push_str(&format!(" - {}\n", text)); + } + } + story.push('\n'); + } + + // Aggregate YOLO objects + let mut all_objects: std::collections::HashMap = + std::collections::HashMap::new(); + for frame in &context_frames { + if let Some(objects) = &frame.yolo_objects { + if let Some(arr) = objects.as_array() { + for obj in arr { + if let Some(class_name) = + obj.get("class_name").and_then(|v| v.as_str()) + { + *all_objects.entry(class_name.to_string()).or_insert(0) += 1; + } + } + } + } + } + + if !all_objects.is_empty() { + story.push_str("【Objects】\n"); + let mut sorted_objects: Vec<_> = all_objects.iter().collect(); + sorted_objects.sort_by(|a, b| b.1.cmp(a.1)); + for (obj, count) in sorted_objects.iter().take(10) { + story.push_str(&format!(" - {} ({} frames)\n", obj, count)); + } + story.push('\n'); + } + + // Aggregate OCR text + let mut all_texts: Vec = Vec::new(); + for frame in &context_frames { + if let Some(texts) = &frame.ocr_results { + if let Some(arr) = texts.as_array() { + for txt in arr { + if let Some(text) = txt.get("text").and_then(|v| v.as_str()) { + if !text.is_empty() && text.len() > 2 { + all_texts.push(text.to_string()); + } + } + } + } + } + } + + if !all_texts.is_empty() { + story.push_str("【Text in video】\n"); + for txt in all_texts.iter().take(10) { + story.push_str(&format!(" - {}\n", txt)); + } + story.push('\n'); + } + + // Aggregate faces + let mut face_count = 0; + for frame in &context_frames { + if let Some(faces) = &frame.face_results { + if let Some(arr) = faces.as_array() { + face_count += arr.len(); + } + } + } + + if face_count > 0 { + story.push_str(&format!( + "【Faces】\n - {} face(s) detected\n\n", + face_count + )); + } + + println!("{}", story); + } + + Ok(()) + } + Commands::Vectorize { uuid } => { + println!("Vectorizing: {}", uuid); + + let pg = PostgresDb::init() + .await + .context("Failed to init PostgreSQL")?; + let qdrant = QdrantDb::init().await.context("Failed to init Qdrant")?; + let embedder = Embedder::new("nomic-embed-text-v2-moe:latest".to_string()); + + let mut stored_count = 0usize; + + // Get list of videos to process + let videos_to_process = if uuid == "all" { + // Get all videos + let videos = pg.list_videos(10000, 0).await?.0; + videos.into_iter().map(|v| v.uuid).collect::>() + } else { + // Process single video + vec![uuid.clone()] + }; + + for target in &videos_to_process { + println!("\n=== Processing video: {} ===", target); + + let chunks = pg.get_chunks_by_uuid(target.as_str()).await?; + let sentence_chunks: Vec<_> = chunks + .into_iter() + .filter(|c| c.chunk_type == ChunkType::Sentence) + .collect(); + + println!( + "Found {} sentence chunks for {}", + sentence_chunks.len(), + target + ); + + let mut video_stored_count = 0usize; + + for chunk in sentence_chunks { + // Try to extract text from different possible locations + let text = chunk + .content + .get("data") // Try data->text structure first + .and_then(|data| data.get("text")) + .and_then(|v| v.as_str()) + .or_else(|| chunk.content.get("text").and_then(|v| v.as_str())) // Try root text structure + .unwrap_or(""); + + if text.is_empty() { + eprintln!( + "Empty text for chunk {}, content: {:?}", + chunk.chunk_id, chunk.content + ); + continue; + } + + print!("Embedding chunk {}... ", chunk.chunk_id); + std::io::stdout().flush().unwrap(); + + match embedder.embed_document(text).await { + Ok(vector) => { + println!("embedding success ({} dims)", vector.len()); + let vector_id = format!("{}_{}", chunk.uuid, chunk.chunk_id); + + if let Err(e) = + pg.store_vector(&chunk.chunk_id, &vector, &chunk.uuid).await + { + eprintln!("store_vector error for {}: {}", chunk.chunk_id, e); + continue; + } + + let qdrant_payload = VectorPayload { + uuid: chunk.uuid.clone(), + chunk_id: chunk.chunk_id.clone(), + chunk_type: "sentence".to_string(), + start_time: chunk.start_time().seconds(), + end_time: chunk.end_time().seconds(), + text: Some(text.to_string()), + }; + if let Err(e) = qdrant + .upsert_vector(&chunk.chunk_id, &vector, qdrant_payload) + .await + { + eprintln!("upsert_vector error for {}: {}", chunk.chunk_id, e); + continue; + } + + if let Err(e) = pg.update_vector_id(&chunk.chunk_id, &vector_id).await { + eprintln!("update_vector_id error for {}: {}", chunk.chunk_id, e); + continue; + } + + stored_count += 1; + video_stored_count += 1; + println!( + "stored (video: {}, total: {})", + video_stored_count, stored_count + ); + } + Err(e) => { + println!("embedding failed: {}", e); + } + } + } + + // Only update storage status if vectors were actually stored for this video + if video_stored_count > 0 { + pg.update_storage_status(target.as_str(), "pvector_chunk", true) + .await?; + pg.update_storage_status(target.as_str(), "qvector_chunk", true) + .await?; + println!( + "✓ Vectorize stage completed for {}! ({} vectors stored)", + target, video_stored_count + ); + } else { + println!( + "✗ Vectorize stage failed for {}! (0 vectors stored)", + target + ); + } + } + + println!("\n=== Vectorization Summary ==="); + println!("Total vectors stored: {}", stored_count); + if uuid == "all" { + println!("✓ Vectorize stage completed for all videos!"); + } + Ok(()) + } + Commands::Play { target } => { + println!("Playing: {}", target); + // TODO: Implement play + Ok(()) + } + Commands::Watch { directories } => { + println!("Starting watcher: {:?}", directories); + // TODO: Implement watch + Ok(()) + } + Commands::System { gpu } => { + let resources = SystemResources::check(); + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ System Resources Report ║"); + println!("╠══════════════════════════════════════════════════════════════╣"); + println!( + "║ CPU: {:.1}% idle ║", + resources.cpu_idle_percent + ); + println!( + "║ Memory: {:.1}GB / {:.1}GB available ({:.0}% used) ║", + resources.memory_available_mb as f64 / 1024.0, + resources.memory_total_mb as f64 / 1024.0, + resources.memory_used_percent + ); + + if resources.gpu_available { + match resources.gpu_type { + GpuType::Nvidia => { + let util = resources.gpu_utilization.unwrap_or(0.0); + println!( + "║ GPU: NVIDIA - {:.0}% utilized ║", + util + ); + } + GpuType::AppleMps => { + println!( + "║ GPU: Apple MPS (Metal) - available ║" + ); + } + } + } else { + println!("║ GPU: None detected ║"); + } + println!("╠══════════════════════════════════════════════════════════════╣"); + + if resources.can_parallel(4096) { + println!("║ Mode: PARALLEL - Can run multiple modules together ║"); + println!( + "║ Recommended modules: {} ║", + resources.recommend_parallel_modules().join(", ") + ); + } else { + println!("║ Mode: SEQUENTIAL - Low resources, run one at a time ║"); + } + println!("╚══════════════════════════════════════════════════════════════╝"); + + if gpu { + println!("\n=== GPU Details ==="); + let output = std::process::Command::new("system_profiler") + .args(["SPDisplaysDataType", "-detailLevel", "mini"]) + .output(); + if let Ok(o) = output { + println!("{}", String::from_utf8_lossy(&o.stdout)); + } + } + + Ok(()) + } + Commands::Server { host, port } => { + let port = port.unwrap_or_else(|| *momentry_core::core::config::SERVER_PORT); + momentry_core::api::start_server(&host, port).await?; + Ok(()) + } + Commands::Worker { + max_concurrent, + poll_interval, + batch_size, + } => { + use momentry_core::worker::{JobWorker, WorkerConfig}; + + let mut config = WorkerConfig::default(); + if let Some(max) = max_concurrent { + config.max_concurrent = max; + } + if let Some(interval) = poll_interval { + config.poll_interval_secs = interval; + } + if let Some(batch) = batch_size { + config.batch_size = batch; + } + + let db = PostgresDb::init().await?; + let redis = RedisClient::new()?; + + let worker = JobWorker::new( + std::sync::Arc::new(db), + std::sync::Arc::new(redis), + config.clone(), + ); + + println!( + "Starting worker with max_concurrent={}, poll_interval={}s", + config.max_concurrent, config.poll_interval_secs + ); + + worker.run().await?; + Ok(()) + } + Commands::Query { query } => { + println!("Query: {}", query); + // TODO: Implement query + Ok(()) + } + Commands::Lookup { path } => { + let uuid = momentry_core::uuid::compute_uuid_from_path(&path); + println!("Path: {}", path); + println!("UUID: {}", uuid); + Ok(()) + } + Commands::Resolve { uuid } => { + println!("Resolving UUID: {}", uuid); + // TODO: Look up path from UUID in database + println!("(Database lookup not implemented yet)"); + Ok(()) + } + Commands::Thumbnails { uuid, count } => { + let db = PostgresDb::init().await?; + + let videos = if let Some(ref uuid) = uuid { + vec![db + .get_video_by_uuid(uuid) + .await? + .ok_or_else(|| anyhow::anyhow!("Video not found: {}", uuid))?] + } else { + db.list_videos(10000, 0).await?.0 + }; + + let output_dir = std::path::PathBuf::from("thumbnails"); + let extractor = momentry_core::ThumbnailExtractor::new(output_dir, count); + + for video in videos { + println!( + "\nGenerating thumbnails for: {} ({})", + video.file_name, video.uuid + ); + + match extractor.get_or_create(&video.file_path, &video.uuid) { + Ok(result) => { + println!(" Generated {} thumbnails", result.count); + } + Err(e) => { + println!(" Error: {}", e); + } + } + } + + println!("\nThumbnails generated successfully!"); + Ok(()) + } + Commands::Status { uuid } => { + let db = PostgresDb::init().await?; + + let videos = if let Some(ref u) = uuid { + vec![db + .get_video_by_uuid(u) + .await? + .ok_or_else(|| anyhow::anyhow!("Video not found: {}", u))?] + } else { + db.list_videos(10000, 0).await?.0 + }; + + println!("\n╔══════════════════════════════════════════════════════════════════════════════════╗"); + println!( + "║ 📊 Storage Status Report ║" + ); + println!("╠══════════════════════════════════════════════════════════════════════════════════╣"); + println!( + "║ {:32} │ {:8} │ {:8} │ {:8} │ {:8} │ {:8} │ {:8} │ {:8} ║", + "Video", "FS", "FS", "PSQL", "PObj", "MObj", "PVec", "QVec" + ); + println!( + "║ {:32} │ {:8} │ {:8} │ {:8} │ {:8} │ {:8} │ {:8} │ {:8} ║", + "", "Video", "JSON", "Chunk", "Chunk", "Chunk", "Chunk", "Chunk" + ); + println!( + "╠{:33}╪{:9}╪{:9}╪{:9}╪{:9}╪{:9}╪{:9}╪{:9}╣", + str::repeat("─", 32), + str::repeat("─", 8), + str::repeat("─", 8), + str::repeat("─", 8), + str::repeat("─", 8), + str::repeat("─", 8), + str::repeat("─", 8), + str::repeat("─", 8) + ); + + for video in videos { + let (sentence_count, time_count) = + db.get_chunk_count(&video.uuid).await.unwrap_or((0, 0)); + let vector_count = db.get_vector_count(&video.uuid).await.unwrap_or(0); + let total_chunks = sentence_count + time_count; + + let psql_status = if total_chunks > 0 { "✓" } else { "-" }; + let pvec_status = if vector_count > 0 && total_chunks > 0 { + if vector_count >= total_chunks { + "✓" + } else { + "◐" + } + } else { + "-" + }; + let qvec_status = if video.storage.qvector_chunk { + "✓" + } else { + "-" + }; + + let file_name = if video.file_name.len() > 30 { + format!("...{}", &video.file_name[video.file_name.len() - 27..]) + } else { + video.file_name + }; + + println!( + "║ {:32} │ {} │ {} │ {} │ - │ - │ {} │ {} ║", + file_name, + if video.storage.fs_video { "✓" } else { "✗" }, + if video.storage.fs_json { "✓" } else { "-" }, + psql_status, + pvec_status, + qvec_status + ); + } + + println!("╠══════════════════════════════════════════════════════════════════════════════════╣"); + println!( + "║ Storage Types: ║" + ); + println!( + "║ FS_Video - Video file on filesystem ║" + ); + println!( + "║ FS_JSON - JSON files (probe, ASR, YOLO, etc.) ║" + ); + println!( + "║ PSQL_Chunk - Chunks stored in PostgreSQL ║" + ); + println!( + "║ PObject - Chunks as JSON objects in PostgreSQL (future) ║" + ); + println!( + "║ MObject - Chunks as JSON objects in MongoDB (future) ║" + ); + println!( + "║ PVector - Vectors in PostgreSQL ║" + ); + println!( + "║ QVector - Vectors in Qdrant ║" + ); + println!("╚══════════════════════════════════════════════════════════════════════════════════╝"); + Ok(()) + } + Commands::Backup { action, days } => { + let output_dir = OutputDir::new(); + output_dir.ensure_dir()?; + + println!("\n📁 Backup directory: {:?}", output_dir.get_backup_dir()); + + match action.as_str() { + "list" => { + let backups = output_dir.list_backups()?; + println!("\n📦 Available backups:"); + if backups.is_empty() { + println!(" (no backups found)"); + } else { + for backup in &backups { + println!(" - {}", backup.filename); + } + } + println!("\nTotal: {} backup(s)", backups.len()); + } + "cleanup" => { + let days = days.unwrap_or(30); + let deleted = output_dir.cleanup_old_backups(days)?; + println!( + "\n🗑️ Cleaned up {} old backup(s) (older than {} days)", + deleted, days + ); + } + "verify" => { + println!("\n🔍 Verifying backups..."); + let backups = output_dir.list_backups()?; + let mut verified = 0; + let mut failed = 0; + for backup in &backups { + match output_dir.verify_backup(&backup.path) { + Ok(true) => { + println!(" ✓ {}", backup.filename); + verified += 1; + } + Ok(false) => { + println!(" ✗ {} (missing checksum)", backup.filename); + failed += 1; + } + Err(e) => { + println!(" ✗ {} ({})", backup.filename, e); + failed += 1; + } + } + } + println!("\nVerified: {} OK, {} failed", verified, failed); + } + _ => { + println!("\n⚠️ Unknown action: {}", action); + println!("Available actions: list, cleanup, verify"); + } + } + Ok(()) + } + Commands::ApiKey { + action, + name, + key_type, + ttl, + key, + } => { + let db = PostgresDb::init().await?; + let db_url = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://accusys@localhost:5432/momentry".to_string()); + + let service = ApiKeyService::new(db_url); + + match action { + ApiKeyAction::Create => { + let name = name.unwrap_or_else(|| "unnamed-key".to_string()); + let kt = parse_key_type(key_type.as_deref()); + let request = momentry_core::core::api_key::CreateApiKeyRequest { + name: name.clone(), + key_type: kt, + user_id: None, + service_name: None, + permissions: vec!["read".to_string(), "write".to_string()], + ttl_days: ttl, + }; + + match service.create_key(request) { + Ok(response) => { + let key_hash = service.hash_key(&response.key); + let key_type_str = + serde_json::to_string(&kt).unwrap_or_else(|_| "user".to_string()); + let permissions = serde_json::json!(["read", "write"]); + + let config = momentry_core::core::db::CreateApiKeyConfig::new( + &response.key_id, + &key_hash, + kt.prefix(), + &name, + &key_type_str, + ) + .with_permissions(&permissions) + .with_expires_at(response.expires_at); + + if let Err(e) = db.create_api_key(config).await { + eprintln!( + "\n⚠️ Key generated but failed to store in database: {}", + e + ); + } + + println!("\n✅ API Key created successfully!"); + println!("\n┌─────────────────────────────────────────────────────────────────────────────┐"); + println!("│ ⚠️ IMPORTANT: Save this key now - it will not be shown again! │"); + println!("└─────────────────────────────────────────────────────────────────────────────┘"); + println!("\nKey ID: {}", response.key_id); + println!("API Key: {}", response.key); + println!("Expires: {}", response.expires_at); + if !response.warning.is_empty() { + println!("\n⚠️ {}", response.warning); + } + } + Err(e) => { + eprintln!("\n❌ Failed to create API key: {}", e); + } + } + } + ApiKeyAction::List => match db.list_api_keys().await { + Ok(keys) => { + println!("\n📋 API Key List"); + if keys.is_empty() { + println!(" (no API keys found)"); + } else { + println!("\n┌────────────────────────────────────────────────────────────────────────────┐"); + println!( + "│ {:8} │ {:20} │ {:12} │ {:8} │ {:15} │", + "Status", "Name", "Type", "Usage", "Last Used" + ); + println!("├────────────────────────────────────────────────────────────────────────────┤"); + for k in &keys { + let status = if k.status == "active" { + "✓ active" + } else { + &k.status + }; + let last_used = k + .last_used_at + .map(|dt| dt.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_else(|| "never".to_string()); + println!( + "│ {:8} │ {:20} │ {:12} │ {:8} │ {:15} │", + status, + if k.name.len() > 20 { + &k.name[..17] + } else { + &k.name + }, + k.key_type, + k.usage_count, + last_used + ); + } + println!("└────────────────────────────────────────────────────────────────────────────┘"); + println!("\nTotal: {} key(s)", keys.len()); + } + } + Err(e) => { + eprintln!("\n❌ Failed to list API keys: {}", e); + } + }, + ApiKeyAction::Validate => { + let api_key = + key.ok_or_else(|| anyhow::anyhow!("--key required for validate"))?; + let key_hash = service.hash_key(&api_key); + + match db.get_api_key_by_hash(&key_hash).await { + Ok(Some(record)) => { + if record.status == "active" { + db.update_api_key_usage(&record.key_id, None).await.ok(); + println!("\n✅ API Key is valid"); + println!("Key ID: {}", record.key_id); + println!("Name: {}", record.name); + println!("Type: {}", record.key_type); + println!("Usage: {} times", record.usage_count + 1); + if record.rotation_required { + println!( + "⚠️ Rotation required: {}", + record.rotation_reason.as_deref().unwrap_or("unknown") + ); + } + } else { + println!("\n❌ API Key is {}", record.status); + } + } + Ok(None) => { + println!("\n❌ API Key is invalid or not found"); + } + Err(e) => { + eprintln!("\n❌ Validation error: {}", e); + } + } + } + ApiKeyAction::Revoke => { + let key = key.ok_or_else(|| anyhow::anyhow!("--key required for revoke"))?; + let key_id = service.extract_key_id(&key); + match db.revoke_api_key(&key_id).await { + Ok(_) => { + println!("\n🔴 API Key {} revoked successfully", key_id); + } + Err(e) => { + eprintln!("\n❌ Failed to revoke API key: {}", e); + } + } + } + ApiKeyAction::Rotate => { + let key = key.ok_or_else(|| anyhow::anyhow!("--key required for rotate"))?; + let key_id = service.extract_key_id(&key); + let grace_period_end = + service.calculate_grace_period_end(parse_key_type(key_type.as_deref())); + match db + .require_api_key_rotation( + &key_id, + "manual rotation requested", + grace_period_end, + ) + .await + { + Ok(_) => { + println!("\n🔄 Rotation requested for key: {}", key_id); + println!("Grace period ends: {}", grace_period_end); + } + Err(e) => { + eprintln!("\n❌ Rotation request failed: {}", e); + } + } + } + ApiKeyAction::Stats => { + match db.get_api_key_stats().await { + Ok(stats) => { + println!("\n📊 API Key Statistics"); + println!("\n┌─────────────────────────────────────────┐"); + println!("│ Total Keys: {:5} │", stats.total_keys); + println!( + "│ Active Keys: {:5} │", + stats.active_keys + ); + println!( + "│ Expired Keys: {:5} │", + stats.expired_keys + ); + println!( + "│ Rotation Required: {:4} │", + stats.rotation_required + ); + println!( + "│ Anomalies (24h): {:5} │", + stats.anomalies_last_24h + ); + println!("└─────────────────────────────────────────┘"); + } + Err(e) => { + eprintln!("\n⚠️ Failed to get stats: {}", e); + } + } + + let config = service.get_config(); + println!("\n┌─────────────────────────────────────────┐"); + println!("│ Anomaly Detection Thresholds │"); + println!("├─────────────────────────────────────────┤"); + println!( + "│ Requests/minute: {:5} │", + config.requests_per_minute_threshold + ); + println!( + "│ Requests/hour: {:5} │", + config.requests_per_hour_threshold + ); + println!( + "│ Error rate: {:5.1}% │", + config.error_rate_threshold * 100.0 + ); + println!( + "│ Unique IPs/hour: {:5} │", + config.unique_ips_per_hour_threshold + ); + println!( + "│ Lockout threshold: {:5} │", + config.lockout_threshold + ); + println!("└─────────────────────────────────────────┘"); + } + } + Ok(()) + } + Commands::Gitea { + action, + username, + password, + token_name, + scopes, + } => { + use momentry_core::core::api_key::gitea::{ + CreateGiteaTokenRequest, GiteaClient, GiteaScope, + }; + + let db = PostgresDb::init().await?; + let gitea = GiteaClient::new()?; + + match action { + GiteaAction::Create => { + let username = username + .ok_or_else(|| anyhow::anyhow!("--username required for create"))?; + let password = password + .ok_or_else(|| anyhow::anyhow!("--password required for create"))?; + let token_name = token_name + .ok_or_else(|| anyhow::anyhow!("--token-name required for create"))?; + + let scopes_vec: Vec = scopes + .map(|s| { + s.split(',') + .filter_map(|scope| scope.trim().parse::().ok()) + .collect() + }) + .unwrap_or_else(|| { + vec![GiteaScope::ReadRepository, GiteaScope::WriteRepository] + }); + + let request = CreateGiteaTokenRequest { + username: username.clone(), + password, + token_name: token_name.clone(), + scopes: scopes_vec.clone(), + }; + + match gitea.create_token(&request).await { + Ok(response) => { + if let Err(e) = db + .create_gitea_token( + response.id, + &username, + &token_name, + &response.token_last_eight, + &serde_json::json!(scopes_vec + .iter() + .map(|s| s.as_str()) + .collect::>()), + None, + ) + .await + { + eprintln!("\n⚠️ Token created but failed to store: {}", e); + } + + println!("\n✅ Gitea Token created successfully!"); + println!("\n┌─────────────────────────────────────────────────────────────────────────────┐"); + println!("│ ⚠️ IMPORTANT: Save this token now - it will not be shown again! │"); + println!("└─────────────────────────────────────────────────────────────────────────────┘"); + println!("\nToken ID: {}", response.id); + println!("Token Name: {}", response.name); + println!("SHA1: {}", response.sha1); + println!("Last 8: {}", response.token_last_eight); + println!("\nAuthorization Header:"); + println!(" Authorization: token {}", response.sha1); + } + Err(e) => { + eprintln!("\n❌ Failed to create Gitea token: {}", e); + } + } + } + GiteaAction::List => { + let username = + username.ok_or_else(|| anyhow::anyhow!("--username required for list"))?; + let password = + password.ok_or_else(|| anyhow::anyhow!("--password required for list"))?; + + match gitea.list_tokens(&username, &password).await { + Ok(tokens) => { + println!("\n📋 Gitea Tokens for user: {}", username); + if tokens.is_empty() { + println!(" (no tokens found)"); + } else { + println!("\n┌────────────────────────────────────────────────────────────────────────────┐"); + println!("│ ID │ Name │ Last 8 │ Registered │"); + println!("├────────────────────────────────────────────────────────────────────────────┤"); + for token in &tokens { + let registered = db + .get_gitea_token_by_name(&username, &token.name) + .await + .ok() + .flatten() + .map(|_| "✓") + .unwrap_or("-"); + println!( + "│ {:8} │ {:20} │ {:9} │ {:27} │", + token.id, + if token.name.len() > 20 { + &token.name[..17] + } else { + &token.name + }, + token.token_last_eight, + registered + ); + } + println!("└────────────────────────────────────────────────────────────────────────────┘"); + println!("\nTotal: {} token(s)", tokens.len()); + } + } + Err(e) => { + eprintln!("\n❌ Failed to list Gitea tokens: {}", e); + } + } + } + GiteaAction::Delete => { + let username = username + .ok_or_else(|| anyhow::anyhow!("--username required for delete"))?; + let password = password + .ok_or_else(|| anyhow::anyhow!("--password required for delete"))?; + let token_name = token_name + .ok_or_else(|| anyhow::anyhow!("--token-name required for delete"))?; + + match gitea.delete_token(&username, &password, &token_name).await { + Ok(_) => { + let _ = db.delete_gitea_token(&username, &token_name).await; + println!("\n🗑️ Token '{}' deleted successfully", token_name); + } + Err(e) => { + eprintln!("\n❌ Failed to delete Gitea token: {}", e); + } + } + } + GiteaAction::Verify => { + let token_name = token_name + .ok_or_else(|| anyhow::anyhow!("--token-name required for verify"))?; + + let record = db + .get_gitea_token_by_name( + &username.unwrap_or_else(|| "unknown".to_string()), + &token_name, + ) + .await?; + + match record { + Some(r) => { + println!("\n📋 Gitea Token: {}", r.token_name); + println!(" User: {}", r.gitea_user); + println!(" Token ID: {}", r.gitea_token_id); + println!(" Last 8: {}", r.token_last_eight); + println!(" Scopes: {}", r.scopes); + println!(" Created: {}", r.created_at); + if let Some(verified) = r.last_verified { + println!(" Last Verified: {}", verified); + } else { + println!(" Last Verified: never"); + } + } + None => { + println!("\n❌ Token not found in local database"); + } + } + } + } + Ok(()) + } + Commands::N8n { + action, + api_key, + label, + expires_in_days, + } => { + use momentry_core::core::api_key::n8n::{ + extract_last_eight, CreateN8nApiKeyRequest, N8nClient, + }; + + let db = PostgresDb::init().await?; + + match action { + N8nAction::Create => { + let api_key_value = api_key.ok_or_else(|| { + anyhow::anyhow!("--api-key required for create (existing n8n API key)") + })?; + let label = + label.ok_or_else(|| anyhow::anyhow!("--label required for create"))?; + + let n8n = N8nClient::new(api_key_value)?; + + let expires_at = expires_in_days + .map(|days| chrono::Utc::now() + chrono::Duration::days(days)); + + let request = CreateN8nApiKeyRequest { + label: label.clone(), + expires_at, + }; + + match n8n.create_api_key(&request).await { + Ok(response) => { + if let Err(e) = db + .create_n8n_api_key( + &response.id, + &label, + &extract_last_eight(&response.api_key), + None, + response.expires_at, + ) + .await + { + eprintln!("\n⚠️ API key created but failed to store: {}", e); + } + + println!("\n✅ n8n API Key created successfully!"); + println!("\n┌─────────────────────────────────────────────────────────────────────────────┐"); + println!("│ ⚠️ IMPORTANT: Save this API key now - it will not be shown again! │"); + println!("└─────────────────────────────────────────────────────────────────────────────┘"); + println!("\nKey ID: {}", response.id); + println!("Label: {}", response.label); + println!("API Key: {}", response.api_key); + println!("\nUsage:"); + println!(" curl -H 'X-N8N-API-KEY: {}' https://n8n.momentry.ddns.net/api/v1/workflows", response.api_key); + } + Err(e) => { + eprintln!("\n❌ Failed to create n8n API key: {}", e); + } + } + } + N8nAction::List => { + let api_key_value = + api_key.ok_or_else(|| anyhow::anyhow!("--api-key required for list"))?; + + let n8n = N8nClient::new(api_key_value)?; + + match n8n.list_api_keys().await { + Ok(keys) => { + println!("\n📋 n8n API Keys"); + if keys.is_empty() { + println!(" (no API keys found)"); + } else { + println!("\n┌────────────────────────────────────────────────────────────────────────────┐"); + println!("│ Label │ ID │"); + println!("├────────────────────────────────────────────────────────────────────────────┤"); + for key in &keys { + println!( + "│ {:27} │ {:39} │", + if key.label.len() > 27 { + &key.label[..24] + } else { + &key.label + }, + key.id + ); + } + println!("└────────────────────────────────────────────────────────────────────────────┘"); + println!("\nTotal: {} key(s)", keys.len()); + } + } + Err(e) => { + eprintln!("\n❌ Failed to list n8n API keys: {}", e); + } + } + } + N8nAction::Delete => { + let api_key_value = + api_key.ok_or_else(|| anyhow::anyhow!("--api-key required for delete"))?; + let label = + label.ok_or_else(|| anyhow::anyhow!("--label required for delete"))?; + + let record = db.get_n8n_api_key_by_label(&label).await?; + if let Some(r) = record { + let n8n = N8nClient::new(api_key_value)?; + match n8n.delete_api_key(&r.n8n_key_id).await { + Ok(_) => { + let _ = db.delete_n8n_api_key(&label).await; + println!("\n🗑️ API key '{}' deleted successfully", label); + } + Err(e) => { + eprintln!("\n❌ Failed to delete n8n API key: {}", e); + } + } + } else { + println!("\n❌ API key '{}' not found in local database", label); + } + } + N8nAction::Verify => { + let label = + label.ok_or_else(|| anyhow::anyhow!("--label required for verify"))?; + + let record = db.get_n8n_api_key_by_label(&label).await?; + + match record { + Some(r) => { + println!("\n📋 n8n API Key: {}", r.label); + println!(" Key ID: {}", r.n8n_key_id); + println!(" Last 8: {}", r.api_key_last_eight); + println!(" Created: {}", r.created_at); + if let Some(expires) = r.expires_at { + println!(" Expires: {}", expires); + } + if let Some(verified) = r.last_verified { + println!(" Last Verified: {}", verified); + } else { + println!(" Last Verified: never"); + } + } + None => { + println!("\n❌ API key not found in local database"); + } + } + } + } + Ok(()) + } + } +} diff --git a/src/worker/job_worker.rs b/src/worker/job_worker.rs index 8f4f29a..998502e 100644 --- a/src/worker/job_worker.rs +++ b/src/worker/job_worker.rs @@ -6,7 +6,13 @@ use tokio::time::sleep; use tracing::{error, info, warn}; use crate::core::chunk::{rule1_ingest, rule3_ingest}; -use crate::core::db::{MonitorJobStatus, PostgresDb, ProcessorJobStatus, RedisClient, VideoStatus}; +use crate::core::db::qdrant_db::QdrantDb; +use crate::api::five_w1h_agent_api::run_5w1h_agent; +use crate::api::identity_agent_api::run_identity_agent; +use crate::core::db::{ + MonitorJobStatus, PostgresDb, ProcessorJobStatus, RedisClient, VectorPayload, VideoStatus, +}; +use crate::core::embedding::Embedder; use crate::worker::config::WorkerConfig; use crate::worker::processor::{ProcessorPool, ProcessorTask}; use crate::worker::resources::SystemResources; @@ -140,7 +146,12 @@ impl JobWorker { .get_running_jobs_with_all_processors_done(self.config.batch_size) .await?; for job in running_jobs_done { - let should_retry = self.check_and_complete_job(job.id, &job.uuid).await.is_ok(); + let expected_count = if job.processors.is_empty() { + crate::core::db::ProcessorType::all().len() + } else { + job.processors.len() + }; + let should_retry = self.check_and_complete_job(job.id, &job.uuid, expected_count).await.is_ok(); if should_retry && self.processor_pool.can_start().await { if let Err(e) = self.process_job(job.clone()).await { error!("Failed to reprocess job {}: {}", job.uuid, e); @@ -453,22 +464,33 @@ impl JobWorker { // 總是檢查是否可以完成 job（check_and_complete_job 內部會判斷） // processor_results 不足時它會自動略過 - self.check_and_complete_job(job.id, &job.uuid).await?; + let expected_count = if job.processors.is_empty() { + crate::core::db::ProcessorType::all().len() + } else { + job.processors.len() + }; + self.check_and_complete_job(job.id, &job.uuid, expected_count).await?; Ok(()) } - async fn check_and_complete_job(&self, job_id: i32, uuid: &str) -> Result<()> { + async fn check_and_complete_job(&self, job_id: i32, uuid: &str, expected_count: usize) -> Result<()> { let results = self.db.get_processor_results_by_job(job_id).await?; - // 如果 processor_results 筆數少於總 processor 數，代表有 processor 尚未啟動（如依賴未滿足） - let all_processor_types = crate::core::db::ProcessorType::all().len(); - if results.len() < all_processor_types { + info!( + "check_and_complete_job: {} results={}, expected={}", + uuid, + results.len(), + expected_count + ); + + // 如果 processor_results 筆數少於期望的 processor 數，代表有 processor 尚未啟動（如依賴未滿足） + if results.len() < expected_count { info!( "Job {} has {}/{} processor results, not all processors created yet. Skipping completion check.", uuid, results.len(), - all_processor_types + expected_count ); return Ok(()); } @@ -491,6 +513,14 @@ impl JobWorker { .iter() .any(|r| matches!(r.status, crate::core::db::ProcessorJobStatus::Failed)); + let any_pending = results + .iter() + .any(|r| matches!(r.status, crate::core::db::ProcessorJobStatus::Pending)); + + let any_skipped = results + .iter() + .any(|r| matches!(r.status, crate::core::db::ProcessorJobStatus::Skipped)); + let completed_count = results .iter() .filter(|r| { @@ -544,7 +574,14 @@ impl JobWorker { let fps = video.fps; match rule1_ingest::execute_rule1(&db_clone, &uuid_clone, fps).await { Ok(count) => { - info!("✅ Rule 1 Ingestion completed: {} chunks inserted.", count) + info!("✅ Rule 1 Ingestion completed: {} chunks inserted.", count); + // Automatically vectorize new sentence chunks + if count > 0 { + info!("📝 Starting automatic vectorize for {} chunks...", count); + if let Err(e) = Self::vectorize_chunks(&db_clone, &uuid_clone).await { + error!("❌ Auto-vectorize failed for {}: {}", uuid_clone, e); + } + } } Err(e) => error!("❌ Rule 1 Ingestion failed: {}", e), } @@ -640,19 +677,27 @@ impl JobWorker { // 🚀 P3 Trigger: Identity Agent (Face + ASRX) if has_face && has_asrx { info!("📝 Prerequisites met for Identity Agent. Starting analysis..."); + let db_clone = self.db.clone(); let uuid_clone = uuid.to_string(); tokio::spawn(async move { - tracing::info!("Identity Agent started for video {}", uuid_clone); + match run_identity_agent(&db_clone, &uuid_clone).await { + Ok(()) => info!("✅ Identity Agent completed for {}", uuid_clone), + Err(e) => error!("❌ Identity Agent failed for {}: {}", uuid_clone, e), + } }); } // 🚀 P4 Trigger: 5W1H Agent (after Rule 3 completion) if has_cut && has_asr { - info!("📝 Prerequisites met for 5W1H Agent. Will trigger after Rule 3..."); + info!("📝 Prerequisites met for 5W1H Agent. Starting..."); + let db_clone = self.db.clone(); let uuid_clone = uuid.to_string(); tokio::spawn(async move { tokio::time::sleep(tokio::time::Duration::from_secs(30)).await; - tracing::info!("5W1H Agent started for video {}", uuid_clone); + match run_5w1h_agent(&db_clone, &uuid_clone).await { + Ok(()) => info!("✅ 5W1H Agent completed for {}", uuid_clone), + Err(e) => error!("❌ 5W1H Agent failed for {}: {}", uuid_clone, e), + } }); } @@ -679,8 +724,8 @@ impl JobWorker { self.redis.delete_worker_job(uuid).await?; info!("Job {} completed successfully", job_id); - } else if essential_completed && !all_completed { - // 必要 processor 完成但部分非必要失敗 → 仍算完成 + } else if essential_completed && !all_completed && !any_pending && !any_skipped { + // 必要 processor 完成但部分非必要失敗 → 仍算完成（但無 pending 者才觸發） info!( "Job {} completed with non-essential failures. Essential: {:?}", job_id, essential_processors @@ -738,6 +783,66 @@ impl JobWorker { info!("Shutting down worker..."); self.processor_pool.cancel_all().await; } + + /// 自動對 sentence chunks 產生 vector embedding 並存入 PG + Qdrant + async fn vectorize_chunks(db: &PostgresDb, uuid: &str) -> anyhow::Result<()> { + let embedder = Embedder::new("mxbai-embed-large:latest".to_string()); + let qdrant = QdrantDb::new(); + let pool = db.pool(); + + let rows = sqlx::query_as::<_, (String, String, String, f64, f64, String)>( + "SELECT chunk_id, chunk_type, text_content, start_time, end_time, content::text FROM dev.chunks WHERE file_uuid = $1 AND chunk_type = 'sentence' AND embedding IS NULL AND (text_content IS NOT NULL AND text_content != '') ORDER BY chunk_index", + ) + .bind(uuid) + .fetch_all(pool) + .await?; + + if rows.is_empty() { + info!("[Vectorize] No sentence chunks to vectorize for {}", uuid); + return Ok(()); + } + + let total = rows.len(); + info!("[Vectorize] Starting vectorize of {} chunks for {}", total, uuid); + + let mut stored = 0usize; + for (chunk_id, _chunk_type, text, start_time, end_time, _content_str) in &rows { + if text.is_empty() { + continue; + } + + match embedder.embed_document(text).await { + Ok(vector) => { + if let Err(e) = db.store_vector(chunk_id, &vector, uuid).await { + error!("[Vectorize] PG store failed for {}: {}", chunk_id, e); + continue; + } + let payload = VectorPayload { + uuid: uuid.to_string(), + chunk_id: chunk_id.clone(), + chunk_type: "sentence".to_string(), + start_time: *start_time, + end_time: *end_time, + text: Some(text.clone()), + }; + if let Err(e) = qdrant.upsert_vector(chunk_id, &vector, payload).await { + error!("[Vectorize] Qdrant upsert failed for {}: {}", chunk_id, e); + continue; + } + stored += 1; + if stored % 50 == 0 { + info!("[Vectorize] {}/{} vectors stored for {}", stored, total, uuid); + } + } + Err(e) => { + error!("[Vectorize] Embedding failed for {}: {}", chunk_id, e); + } + } + } + + info!("[Vectorize] Completed: {}/{} vectors stored for {}", stored, total, uuid); + Ok(()) + } } #[cfg(test)] diff --git a/src/worker/processor.rs b/src/worker/processor.rs index e51b324..584f0a4 100644 --- a/src/worker/processor.rs +++ b/src/worker/processor.rs @@ -1,10 +1,37 @@ use anyhow::{Context, Result}; +use libc; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; use tokio::sync::{mpsc, RwLock}; use tracing::{error, info, warn}; +/// Guard that ensures processor pool cleanup runs even if the task panics. +struct ProcessorCleanupGuard { + job_id: i32, + running: Arc>>, + running_count: Arc>, +} + +impl Drop for ProcessorCleanupGuard { + fn drop(&mut self) { + use tokio::sync::TryLockError; + // 嘗試同步清理；若 lock 被佔用則跳過（避免 deadlock） + if let Ok(mut guard) = self.running.try_write() { + guard.remove(&self.job_id); + } else { + warn!("[ProcessorCleanupGuard] running lock contended, skipping cleanup"); + } + if let Ok(mut guard) = self.running_count.try_write() { + if *guard > 0 { + *guard -= 1; + } + } else { + warn!("[ProcessorCleanupGuard] running_count lock contended, skipping cleanup"); + } + } +} + use crate::core::config::{OUTPUT_DIR, PYTHON_PATH, SCRIPTS_DIR}; use crate::core::db::{ MonitorJob, PostgresDb, ProcessorJobStatus, ProcessorType, QdrantDb, RedisClient, @@ -93,7 +120,7 @@ impl ProcessorPool { if handle_count == 0 && count == 0 { if let Err(e) = self .db - .reset_stale_processor_results(ProcessorJobStatus::Failed, "Worker restarted") + .reset_stale_processor_results(ProcessorJobStatus::Pending, "Worker restarted") .await { error!("Failed to reset stale processor results: {}", e); @@ -101,7 +128,29 @@ impl ProcessorPool { } } + async fn kill_existing_processor(redis: &RedisClient, uuid: &str, processor: &str) { + let prefix = crate::core::config::REDIS_KEY_PREFIX.as_str(); + let key = format!("{}worker:job:{}:processor:{}", prefix, uuid, processor); + if let Ok(mut conn) = redis.get_conn().await { + let old_pid: Option = redis::cmd("HGET") + .arg(&key) + .arg("pid") + .query_async(&mut conn) + .await + .ok() + .flatten(); + if let Some(pid) = old_pid { + if pid > 0 { + warn!("[PID] Killing existing process {} for {}/{}", pid, uuid, processor); + unsafe { libc::kill(pid, libc::SIGKILL); } + } + } + } + } + pub async fn start_processor(&self, task: ProcessorTask) -> Result<()> { + Self::kill_existing_processor(&*self.redis, &task.job.uuid, task.processor_type.as_str()).await; + let (cancel_tx, cancel_rx) = mpsc::channel(1); let job_id = task.job.id; let processor_type = task.processor_type; @@ -144,6 +193,13 @@ impl ProcessorPool { } tokio::spawn(async move { + // Guard 的 Drop 確保 panic 時也清理 pool state + let _guard = ProcessorCleanupGuard { + job_id, + running: running.clone(), + running_count: running_count.clone(), + }; + info!("Starting processor {} for job {}", processor_name, job.uuid); let _ = db @@ -171,19 +227,6 @@ impl ProcessorPool { let result = Self::run_processor(&db, &redis, &job, processor_type, cancel_rx).await; - // Store child PID for stability - { - let mut pid_lock = child_pid.write().await; - *pid_lock = Some(0); - } - - { - let mut running_guard = running.write().await; - running_guard.remove(&job_id); - let mut count_guard = running_count.write().await; - *count_guard -= 1; - } - match result { Ok(output) => { info!( @@ -747,6 +790,12 @@ impl ProcessorPool { "_face" ); + // 確保 collection 存在（dim=512 for FaceNet） + if let Err(e) = qdrant.ensure_collection(&collection, 512).await { + tracing::error!("Failed to ensure Qdrant face collection: {}", e); + return Ok(()); + } + let mut count = 0; for frame in &face_result.frames { for face in &frame.faces { @@ -807,6 +856,12 @@ impl ProcessorPool { "_voice" ); + // 確保 collection 存在（dim=192 for ASRX voice） + if let Err(e) = qdrant.ensure_collection(&collection, 192).await { + tracing::error!("Failed to ensure Qdrant voice collection: {}", e); + return Ok(()); + } + let embeddings = match &asrx_result.embeddings { Some(e) => e, None => return Ok(()), @@ -958,6 +1013,24 @@ impl ProcessorPool { db.store_scene_pre_chunks_batch(uuid, &scenes).await?; + for (i, scene) in scene_result.scenes.iter().enumerate() { + let chk_id = format!("scene_{}", i + 1); + let meta = serde_json::json!({ + "scene_type": scene.scene_type, + "scene_type_zh": scene.scene_type_zh, + "confidence": scene.confidence, + "top_5": scene.top_5, + }); + let _ = sqlx::query( + "UPDATE dev.chunks SET metadata = metadata || $1::jsonb WHERE file_uuid=$2 AND chunk_id=$3" + ) + .bind(&meta) + .bind(uuid) + .bind(&chk_id) + .execute(db.pool()) + .await; + } + tracing::info!( "Stored {} Scene pre-chunks for video {}", scenes.len(),