fix: ASRX duplication, TKG edges, trace ingest, and add pipeline progress publishing
- ASRX handler no longer stores duplicate 'asr' pre_chunks - Pre_chunks storage made idempotent (delete-before-insert) - Rule 1 + trace_ingest changed to query 'asrx' not 'asr' - Trace chunks removed (dynamic from TKG/Qdrant) - TKG scroll_face_points fixed: trace_id >= 1 (not == 1) - TKG AsrxSegmentEntry: start/end -> start_time/end_time (match ASRX JSON) - Unregister error handling: log instead of silent discard - Add publish_pipeline_progress calls at each pipeline stage (processors, rule1, face_trace, identity_agent, TKG, rule2, completion)
This commit is contained in:
Accusys
469
src/api/scan.rs
469
src/api/scan.rs
@@ -10,6 +10,83 @@ use serde::{Deserialize, Serialize};
|
||||
use super::types::AppState;
|
||||
use crate::core::db::schema;
|
||||
|
||||
/// Comprehensive file stats endpoint — provides all data sources for frontend transparency
|
||||
/// Combines: JSON file status + PostgreSQL counts + Qdrant collections + TKG stats + Identity Agent stats
|
||||
#[derive(Debug, Serialize)]
|
||||
struct FileStatsResponse {
|
||||
file_uuid: String,
|
||||
file_name: Option<String>,
|
||||
status: Option<String>,
|
||||
// Processor status
|
||||
processors: Vec<ProcessorStatus>,
|
||||
// PostgreSQL counts
|
||||
postgres: PostgresStats,
|
||||
// Qdrant collection counts
|
||||
qdrant: QdrantStats,
|
||||
// TKG stats
|
||||
tkg: TkgFileStats,
|
||||
// Identity Agent stats
|
||||
identity_agent: IdentityAgentStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ProcessorStatus {
|
||||
name: String,
|
||||
status: String,
|
||||
progress: u32,
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Default)]
|
||||
struct PostgresStats {
|
||||
sentence_chunks: i64,
|
||||
trace_chunks: i64,
|
||||
relationship_chunks: i64,
|
||||
identities: i64,
|
||||
file_identities: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct QdrantStats {
|
||||
faces: i64,
|
||||
face_traces: i64,
|
||||
face_identities: i64,
|
||||
text_chunks: i64,
|
||||
speakers: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Default)]
|
||||
struct TkgFileStats {
|
||||
total_nodes: i64,
|
||||
total_edges: i64,
|
||||
face_track_nodes: i64,
|
||||
gaze_track_nodes: i64,
|
||||
lip_track_nodes: i64,
|
||||
text_region_nodes: i64,
|
||||
appearance_nodes: i64,
|
||||
accessory_nodes: i64,
|
||||
object_nodes: i64,
|
||||
hand_nodes: i64,
|
||||
speaker_nodes: i64,
|
||||
co_occurrence_edges: i64,
|
||||
speaker_face_edges: i64,
|
||||
face_face_edges: i64,
|
||||
mutual_gaze_edges: i64,
|
||||
lip_sync_edges: i64,
|
||||
has_appearance_edges: i64,
|
||||
wears_edges: i64,
|
||||
hand_object_edges: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Default)]
|
||||
struct IdentityAgentStats {
|
||||
clusters: i64,
|
||||
identities_created: i64,
|
||||
tmdb_matches: i64,
|
||||
speaker_bindings: i64,
|
||||
confirmations: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ScannedFileInfo {
|
||||
file_name: String,
|
||||
@@ -372,9 +449,46 @@ async fn get_ingestion_status(
|
||||
) -> Result<Json<IngestionStatusResponse>, StatusCode> {
|
||||
let pool = state.db.pool();
|
||||
let chunk = schema::table_name("chunk");
|
||||
let fd = schema::table_name("face_detections");
|
||||
let identities = schema::table_name("identities");
|
||||
|
||||
// Get face counts from Qdrant _faces
|
||||
use crate::core::db::qdrant_db::QdrantDb;
|
||||
use serde_json::json;
|
||||
|
||||
let qdrant = QdrantDb::new();
|
||||
let face_filter = json!({
|
||||
"must": [
|
||||
{"key": "file_uuid", "match": {"value": file_uuid}}
|
||||
]
|
||||
});
|
||||
let points = qdrant.scroll_all_points("_faces", face_filter, 1000).await.unwrap_or_default();
|
||||
|
||||
let face_total = points.len() as i64;
|
||||
let mut trace_ids: std::collections::HashSet<i64> = std::collections::HashSet::new();
|
||||
let mut identity_ids: std::collections::HashSet<i64> = std::collections::HashSet::new();
|
||||
let mut stranger_traces: std::collections::HashSet<i64> = std::collections::HashSet::new();
|
||||
|
||||
for point in &points {
|
||||
let payload = &point["payload"];
|
||||
if let Some(tid) = payload["trace_id"].as_i64() {
|
||||
if tid > 0 {
|
||||
trace_ids.insert(tid);
|
||||
if payload["identity_id"].is_null() {
|
||||
stranger_traces.insert(tid);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(iid) = payload["identity_id"].as_i64() {
|
||||
if iid > 0 {
|
||||
identity_ids.insert(iid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let trace_count = trace_ids.len() as i64;
|
||||
let identity_count = identity_ids.len() as i64;
|
||||
let strangers = stranger_traces.len() as i64;
|
||||
|
||||
let scene_meta_path = format!(
|
||||
"{}/{}.scene_meta.json",
|
||||
crate::core::config::OUTPUT_DIR.as_str(),
|
||||
@@ -398,14 +512,12 @@ async fn get_ingestion_status(
|
||||
let scene_count = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut'"
|
||||
));
|
||||
let face_total = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {fd} WHERE file_uuid = '{file_uuid}'"
|
||||
));
|
||||
let trace_count = count_sql!(&format!("SELECT COUNT(DISTINCT trace_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL"));
|
||||
let face_total = face_total;
|
||||
let trace_count = trace_count;
|
||||
let trace_chunks = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'trace'"
|
||||
));
|
||||
let identity_count = count_sql!(&format!("SELECT COUNT(DISTINCT identity_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND identity_id IS NOT NULL"));
|
||||
let identity_count = identity_count;
|
||||
let tkg_nodes = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
|
||||
schema::table_name("tkg_nodes")
|
||||
@@ -414,12 +526,41 @@ async fn get_ingestion_status(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
|
||||
schema::table_name("tkg_edges")
|
||||
));
|
||||
let related_identities: Vec<IdentityRef> =
|
||||
|
||||
// Get individual node counts by type
|
||||
let tkg_nodes_table = schema::table_name("tkg_nodes");
|
||||
let face_track_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'face_track'"));
|
||||
let gaze_track_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'gaze_track'"));
|
||||
let lip_track_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'lip_track'"));
|
||||
let text_region_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'text_region'"));
|
||||
let appearance_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'appearance_trace'"));
|
||||
let accessory_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'accessory'"));
|
||||
let object_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'yolo_object'"));
|
||||
let hand_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'hand'"));
|
||||
let speaker_nodes: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = '{file_uuid}' AND node_type = 'speaker'"));
|
||||
|
||||
// Get individual edge counts by type
|
||||
let tkg_edges_table = schema::table_name("tkg_edges");
|
||||
let co_occurrence_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'CO_OCCURS_WITH'"));
|
||||
let speaker_face_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'SPEAKS_AS'"));
|
||||
let face_face_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'FACE_TO_FACE'"));
|
||||
let mutual_gaze_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'MUTUAL_GAZE'"));
|
||||
let lip_sync_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'LIP_SYNC'"));
|
||||
let has_appearance_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'HAS_APPEARANCE'"));
|
||||
let wears_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'WEARS'"));
|
||||
let hand_object_edges: i64 = count_sql!(&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = '{file_uuid}' AND edge_type = 'HAND_OBJECT'"));
|
||||
|
||||
// Rule 2 relationship chunks
|
||||
let rule2_chunks = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'relationship'"
|
||||
));
|
||||
// Get related identities from Qdrant _faces
|
||||
let related_identity_ids: Vec<i64> = identity_ids.into_iter().collect();
|
||||
let related_identities: Vec<IdentityRef> = if !related_identity_ids.is_empty() {
|
||||
let id_list: String = related_identity_ids.iter().map(|id| id.to_string()).collect::<Vec<_>>().join(",");
|
||||
match sqlx::query_as::<_, (String, String)>(&format!(
|
||||
"SELECT DISTINCT i.uuid::text, i.name FROM {identities} i \
|
||||
JOIN {fd} fd ON fd.identity_id = i.id \
|
||||
WHERE fd.file_uuid = '{file_uuid}' AND fd.identity_id IS NOT NULL \
|
||||
ORDER BY i.name"
|
||||
"SELECT DISTINCT uuid::text, name FROM {identities} \
|
||||
WHERE id IN ({id_list}) ORDER BY name"
|
||||
))
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
@@ -435,12 +576,12 @@ async fn get_ingestion_status(
|
||||
tracing::error!("related_identities query failed: {}", e);
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
}
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
let strangers = count_sql!(&format!(
|
||||
"SELECT COUNT(DISTINCT trace_id) FROM {fd} \
|
||||
WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL AND identity_id IS NULL"
|
||||
));
|
||||
let strangers = strangers;
|
||||
|
||||
macro_rules! step {
|
||||
($name:expr, $done:expr, $detail:expr) => {
|
||||
@@ -462,9 +603,9 @@ async fn get_ingestion_status(
|
||||
"auto_vectorize",
|
||||
sentence_embedded > 0,
|
||||
Some(format!("{sentence_embedded} embedded"))
|
||||
),
|
||||
step!(
|
||||
"face_track",
|
||||
),
|
||||
step!(
|
||||
"face_track",
|
||||
trace_count > 0,
|
||||
Some(format!("{trace_count} traces / {face_total} detections"))
|
||||
),
|
||||
@@ -473,11 +614,32 @@ step!(
|
||||
trace_chunks > 0,
|
||||
Some(format!("{trace_chunks} trace chunks"))
|
||||
),
|
||||
// TKG Nodes
|
||||
step!("tkg_face_track", face_track_nodes > 0, Some(format!("{face_track_nodes} nodes"))),
|
||||
step!("tkg_gaze_track", gaze_track_nodes > 0, Some(format!("{gaze_track_nodes} nodes"))),
|
||||
step!("tkg_lip_track", lip_track_nodes > 0, Some(format!("{lip_track_nodes} nodes"))),
|
||||
step!("tkg_text_region", text_region_nodes > 0, Some(format!("{text_region_nodes} nodes"))),
|
||||
step!("tkg_appearance", appearance_nodes > 0, Some(format!("{appearance_nodes} nodes"))),
|
||||
step!("tkg_accessory", accessory_nodes > 0, Some(format!("{accessory_nodes} nodes"))),
|
||||
step!("tkg_object", object_nodes > 0, Some(format!("{object_nodes} nodes"))),
|
||||
step!("tkg_hand", hand_nodes > 0, Some(format!("{hand_nodes} nodes"))),
|
||||
step!("tkg_speaker", speaker_nodes > 0, Some(format!("{speaker_nodes} nodes"))),
|
||||
// TKG Edges
|
||||
step!("tkg_co_occurrence", co_occurrence_edges > 0, Some(format!("{co_occurrence_edges} edges"))),
|
||||
step!("tkg_speaker_face", speaker_face_edges > 0, Some(format!("{speaker_face_edges} edges"))),
|
||||
step!("tkg_face_face", face_face_edges > 0, Some(format!("{face_face_edges} edges"))),
|
||||
step!("tkg_mutual_gaze", mutual_gaze_edges > 0, Some(format!("{mutual_gaze_edges} edges"))),
|
||||
step!("tkg_lip_sync", lip_sync_edges > 0, Some(format!("{lip_sync_edges} edges"))),
|
||||
step!("tkg_has_appearance", has_appearance_edges > 0, Some(format!("{has_appearance_edges} edges"))),
|
||||
step!("tkg_wears", wears_edges > 0, Some(format!("{wears_edges} edges"))),
|
||||
step!("tkg_hand_object", hand_object_edges > 0, Some(format!("{hand_object_edges} edges"))),
|
||||
// Rule 2
|
||||
step!(
|
||||
"tkg",
|
||||
tkg_nodes > 0 || tkg_edges > 0,
|
||||
Some(format!("{tkg_nodes} nodes, {tkg_edges} edges"))
|
||||
"rule2_relationship",
|
||||
rule2_chunks > 0,
|
||||
Some(format!("{rule2_chunks} relationship chunks"))
|
||||
),
|
||||
// Identity & Scene
|
||||
step!(
|
||||
"identity_match",
|
||||
identity_count > 0,
|
||||
@@ -494,6 +656,248 @@ step!(
|
||||
}))
|
||||
}
|
||||
|
||||
/// Comprehensive file stats endpoint — combines all data sources for frontend transparency
|
||||
async fn get_file_stats(
|
||||
State(state): State<AppState>,
|
||||
Path(file_uuid): Path<String>,
|
||||
) -> Result<Json<FileStatsResponse>, StatusCode> {
|
||||
let pool = state.db.pool();
|
||||
|
||||
// 1. Get file info from PostgreSQL
|
||||
let videos_table = schema::table_name("videos");
|
||||
let file_info: Option<(String, String, String)> = sqlx::query_as(&format!(
|
||||
"SELECT file_uuid, file_name, status FROM {} WHERE file_uuid = $1",
|
||||
videos_table
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let (file_uuid_str, file_name, status) = file_info
|
||||
.map(|(uuid, name, s)| (uuid, Some(name), Some(s)))
|
||||
.unwrap_or_else(|| (file_uuid.clone(), None, None));
|
||||
|
||||
// 2. Get processor status from processing_status JSONB
|
||||
let processing_status: serde_json::Value =
|
||||
sqlx::query_scalar(&format!(
|
||||
"SELECT processing_status FROM {} WHERE file_uuid = $1",
|
||||
videos_table
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.unwrap_or(serde_json::json!({}));
|
||||
|
||||
let processors: Vec<ProcessorStatus> = processing_status
|
||||
.get("progress")
|
||||
.and_then(|p| p.as_object())
|
||||
.map(|progress| {
|
||||
progress
|
||||
.iter()
|
||||
.filter_map(|(name, info)| {
|
||||
info.as_object().map(|obj| {
|
||||
let status = obj
|
||||
.get("status")
|
||||
.and_then(|s| s.as_str())
|
||||
.unwrap_or("pending")
|
||||
.to_string();
|
||||
let progress_val = obj
|
||||
.get("percentage")
|
||||
.and_then(|p| p.as_u64())
|
||||
.unwrap_or(0) as u32;
|
||||
let message = obj
|
||||
.get("message")
|
||||
.and_then(|m| m.as_str())
|
||||
.map(|s| s.to_string());
|
||||
ProcessorStatus {
|
||||
name: name.clone(),
|
||||
status,
|
||||
progress: progress_val,
|
||||
message,
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
// 3. Get PostgreSQL counts
|
||||
let chunk_table = schema::table_name("chunk");
|
||||
let identities_table = schema::table_name("identities");
|
||||
let file_identities_table = schema::table_name("file_identities");
|
||||
|
||||
let postgres = PostgresStats {
|
||||
sentence_chunks: sqlx::query_scalar::<_, i64>(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'sentence'"
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0),
|
||||
trace_chunks: sqlx::query_scalar::<_, i64>(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'trace'"
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0),
|
||||
relationship_chunks: sqlx::query_scalar::<_, i64>(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'relationship'"
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0),
|
||||
identities: sqlx::query_scalar::<_, i64>(&format!(
|
||||
"SELECT COUNT(DISTINCT i.id) FROM {identities_table} i \
|
||||
JOIN {file_identities_table} fi ON fi.identity_id = i.id \
|
||||
WHERE fi.file_uuid = $1"
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0),
|
||||
file_identities: sqlx::query_scalar::<_, i64>(&format!(
|
||||
"SELECT COUNT(*) FROM {file_identities_table} WHERE file_uuid = $1"
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0),
|
||||
};
|
||||
|
||||
// 4. Get Qdrant stats
|
||||
use crate::core::db::qdrant_db::QdrantDb;
|
||||
use serde_json::json;
|
||||
|
||||
let qdrant_db = QdrantDb::new();
|
||||
|
||||
// Face stats
|
||||
let face_filter = json!({
|
||||
"must": [{"key": "file_uuid", "match": {"value": file_uuid}}]
|
||||
});
|
||||
let face_points = qdrant_db
|
||||
.scroll_all_points("_faces", face_filter.clone(), 500)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut face_traces = std::collections::HashSet::new();
|
||||
let mut face_identities = std::collections::HashSet::new();
|
||||
for point in &face_points {
|
||||
let payload = &point["payload"];
|
||||
if let Some(tid) = payload["trace_id"].as_i64() {
|
||||
if tid > 0 {
|
||||
face_traces.insert(tid);
|
||||
}
|
||||
}
|
||||
if let Some(iid) = payload["identity_id"].as_i64() {
|
||||
if iid > 0 {
|
||||
face_identities.insert(iid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Text chunk stats (rule1 collection)
|
||||
let schema = std::env::var("DATABASE_SCHEMA").unwrap_or_else(|_| "dev".to_string());
|
||||
let rule1_collection = format!("momentry_{}_rule1_v2", schema);
|
||||
let text_filter = json!({
|
||||
"must": [{"key": "file_uuid", "match": {"value": file_uuid}}]
|
||||
});
|
||||
let text_points = qdrant_db
|
||||
.scroll_all_points(&rule1_collection, text_filter, 500)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
// Speaker stats
|
||||
let speaker_collection = format!("momentry_{}_speaker", schema);
|
||||
let speaker_filter = json!({
|
||||
"must": [{"key": "file_uuid", "match": {"value": file_uuid}}]
|
||||
});
|
||||
let speaker_points = qdrant_db
|
||||
.scroll_all_points(&speaker_collection, speaker_filter, 500)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let qdrant_stats = QdrantStats {
|
||||
faces: face_points.len() as i64,
|
||||
face_traces: face_traces.len() as i64,
|
||||
face_identities: face_identities.len() as i64,
|
||||
text_chunks: text_points.len() as i64,
|
||||
speakers: speaker_points.len() as i64,
|
||||
};
|
||||
|
||||
// 5. Get TKG stats from PostgreSQL
|
||||
let tkg_nodes_table = schema::table_name("tkg_nodes");
|
||||
let tkg_edges_table = schema::table_name("tkg_edges");
|
||||
|
||||
let tkg = TkgFileStats {
|
||||
face_track_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "face_track").await,
|
||||
gaze_track_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "gaze_track").await,
|
||||
lip_track_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "lip_track").await,
|
||||
text_region_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "text_region").await,
|
||||
appearance_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "appearance_trace").await,
|
||||
accessory_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "accessory").await,
|
||||
object_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "yolo_object").await,
|
||||
hand_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "hand").await,
|
||||
speaker_nodes: count_by_type(pool, &tkg_nodes_table, &file_uuid, "speaker").await,
|
||||
co_occurrence_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "CO_OCCURS_WITH").await,
|
||||
speaker_face_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "SPEAKS_AS").await,
|
||||
face_face_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "FACE_TO_FACE").await,
|
||||
mutual_gaze_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "MUTUAL_GAZE").await,
|
||||
lip_sync_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "LIP_SYNC").await,
|
||||
has_appearance_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "HAS_APPEARANCE").await,
|
||||
wears_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "WEARS").await,
|
||||
hand_object_edges: count_by_type(pool, &tkg_edges_table, &file_uuid, "HAND_OBJECT").await,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 6. Get Identity Agent stats from Qdrant _seeds
|
||||
let seeds_filter = json!({
|
||||
"must": [
|
||||
{"key": "file_uuid", "match": {"value": file_uuid}}
|
||||
]
|
||||
});
|
||||
let seed_points = qdrant_db
|
||||
.scroll_all_points("_seeds", seeds_filter, 500)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let identity_agent = IdentityAgentStats {
|
||||
clusters: 0, // From face_clustered.json if available
|
||||
identities_created: face_identities.len() as i64,
|
||||
tmdb_matches: seed_points.iter()
|
||||
.filter(|p| p["payload"]["source"].as_str() == Some("tmdb"))
|
||||
.count() as i64,
|
||||
speaker_bindings: speaker_points.len() as i64,
|
||||
confirmations: 0, // From identity_bindings table
|
||||
};
|
||||
|
||||
Ok(Json(FileStatsResponse {
|
||||
file_uuid: file_uuid_str,
|
||||
file_name,
|
||||
status,
|
||||
processors,
|
||||
postgres,
|
||||
qdrant: qdrant_stats,
|
||||
tkg,
|
||||
identity_agent,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn count_by_type(pool: &sqlx::PgPool, table: &str, file_uuid: &str, type_val: &str) -> i64 {
|
||||
sqlx::query_scalar::<_, i64>(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND (node_type = $2 OR edge_type = $2)",
|
||||
table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.bind(type_val)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn scan_routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/v1/files/scan", get(scan_files))
|
||||
@@ -502,4 +906,25 @@ pub fn scan_routes() -> Router<AppState> {
|
||||
"/api/v1/stats/ingestion-status/:file_uuid",
|
||||
get(get_ingestion_status),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/stats/file/:file_uuid",
|
||||
get(get_file_stats),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/stats/pipeline/:file_uuid",
|
||||
get(get_pipeline_progress_handler),
|
||||
)
|
||||
}
|
||||
|
||||
/// Get segmented pipeline progress with weighted stages
|
||||
async fn get_pipeline_progress_handler(
|
||||
State(state): State<AppState>,
|
||||
Path(file_uuid): Path<String>,
|
||||
) -> Result<Json<crate::core::progress::PipelineProgress>, StatusCode> {
|
||||
let redis_lock = state.redis_cache.get_client().await;
|
||||
let redis_guard = redis_lock.read().await;
|
||||
let pipeline = crate::core::progress::get_pipeline_progress(&*redis_guard, &file_uuid)
|
||||
.await
|
||||
.unwrap_or_else(|| crate::core::progress::PipelineProgress::new(&file_uuid));
|
||||
Ok(Json(pipeline))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user