admin/momentry_core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ASRX hybrid pipeline, identity history, worker fixes, checkpoint system

Browse Source
This commit is contained in:
Accusys
2026-06-02 07:13:23 +08:00
parent e3066c3f49
commit e1572907ae
198 changed files with 43705 additions and 8910 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
src/api/media_api.rs
View File

@@ -57,6 +57,10 @@ pub fn bbox_routes() -> Router<crate::api::types::AppState> {
"/api/v1/file/:file_uuid/trace/:trace_id/video",
get(trace_video),
)
.route(
"/api/v1/file/:file_uuid/stranger/:stranger_id/video",
get(stranger_video),
)
.route("/api/v1/file/:file_uuid/video", get(stream_video))
.route("/api/v1/file/:file_uuid/thumbnail", get(face_thumbnail))
.route("/api/v1/file/:file_uuid/clip", get(video_clip))
@@ -210,8 +214,9 @@ async fn bbox_overlay_video(
let start_sec = start_f as f64 / fps;
// Get face bboxes
// frame_number is BIGINT (i64) in database
let face_table = schema::table_name("face_detections");
let rows: Vec<(i32, i32, i32, i32, i32, Option<i32>, Option<String>)> = sqlx::query_as(
let rows: Vec<(i64, i32, i32, i32, i32, Option<i32>, Option<String>)> = sqlx::query_as(
&format!("SELECT frame_number, x, y, width, height, trace_id, face_id FROM {} WHERE file_uuid = $1 AND frame_number BETWEEN $2 AND $3 ORDER BY frame_number", face_table)
)
.bind(face_fuid).bind(start_f).bind(end_f)
@@ -222,7 +227,7 @@ async fn bbox_overlay_video(
let mut parts: Vec<String> = Vec::new();
for (frame, x, y, w, h, trace_id, _) in &rows {
let text = format!("t{}", trace_id.unwrap_or(0));
let offset = frame - start_f;
let offset = (*frame as i32) - start_f;
parts.push(format!(
"drawbox=x={}:y={}:w={}:h={}:color=red@0.8:thickness=4:enable='eq(n,{})'",
x, y, w, h, offset
@@ -300,6 +305,15 @@ async fn trace_video(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
Query(params): Query<std::collections::HashMap<String, String>>,
) -> Result<impl IntoResponse, StatusCode> {
trace_video_inner(&state, &file_uuid, trace_id, &params).await
}
async fn trace_video_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
trace_id: i32,
params: &std::collections::HashMap<String, String>,
) -> Result<impl IntoResponse, StatusCode> {
use axum::http::header;
@@ -317,8 +331,9 @@ async fn trace_video(
let (video_path, fps, _width, _height) = row.ok_or(StatusCode::NOT_FOUND)?;
// Query face detections to find frame range for target trace
// frame_number is BIGINT (i64) in database
let face_table = schema::table_name("face_detections");
let rows: Vec<(i32, i32, i32, i32, i32)> = sqlx::query_as(&format!(
let rows: Vec<(i64, i32, i32, i32, i32)> = sqlx::query_as(&format!(
"SELECT frame_number, x, y, width, height FROM {} WHERE file_uuid = $1 AND trace_id = $2 ORDER BY frame_number",
face_table
))
@@ -371,11 +386,12 @@ async fn trace_video(
// === DEBUG MODE: text overlay, list all traces in frame range ===
let start_fn = (start_sec * fps) as i32;
let end_fn = ((start_sec + duration) * fps) as i32;
let end_fn = ((start_sec + duration) * fps) as i64;
// Query all traces with identity names and bbox positions in the visible frame range
// frame_number is BIGINT (i64) in database
let identities_table = schema::table_name("identities");
let all_rows: Vec<(i32, i32, i32, i32, i32, i32, Option<String>)> = sqlx::query_as(&format!(
let all_rows: Vec<(i32, i64, i32, i32, i32, i32, Option<String>)> = sqlx::query_as(&format!(
"SELECT fd.trace_id, fd.frame_number, fd.x, fd.y, fd.width, fd.height, i.name \
FROM {} fd \
LEFT JOIN {} i ON fd.identity_id = i.id \
@@ -391,9 +407,10 @@ async fn trace_video(
.unwrap_or_default();
// Group frames by trace_id, compute start_frame per trace; collect bbox per frame
let mut trace_frames: HashMap<i32, Vec<i32>> = HashMap::new();
// frame_number is i64 (BIGINT), so HashMaps need i64 for frame values
let mut trace_frames: HashMap<i32, Vec<i64>> = HashMap::new();
let mut trace_identity: HashMap<i32, String> = HashMap::new();
let mut bbox_per_frame: HashMap<(i32, i32), (i32, i32, i32, i32)> = HashMap::new(); // (tid, fn) -> (x, y, w, h)
let mut bbox_per_frame: HashMap<(i32, i64), (i32, i32, i32, i32)> = HashMap::new(); // (tid, fn) -> (x, y, w, h)
for (tid, fn_, x, y, w, h, name_opt) in &all_rows {
trace_frames.entry(*tid).or_default().push(*fn_);
bbox_per_frame.insert((*tid, *fn_), (*x, *y, *w, *h));
@@ -417,7 +434,7 @@ async fn trace_video(
.unwrap_or_else(|| "-".to_string());
// Sort traces for consistent ordering
let mut sorted_traces: Vec<(i32, &Vec<i32>)> =
let mut sorted_traces: Vec<(i32, &Vec<i64>)> =
trace_frames.iter().map(|(k, v)| (*k, v)).collect();
sorted_traces.sort_by_key(|(tid, _)| *tid);
@@ -695,6 +712,7 @@ struct ThumbQuery {
y: Option<i32>,
w: Option<i32>,
h: Option<i32>,
trace_id: Option<i32>,
}
async fn face_thumbnail(
@@ -717,15 +735,70 @@ async fn face_thumbnail(
}
};
let row: Option<(String,)> = sqlx::query_as(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1",
// Step 1: Check for pre-stored face crop if trace_id is provided
if let Some(trace_id) = q.trace_id {
let output_dir = crate::core::config::OUTPUT_DIR.as_str();
let cached_path = std::path::PathBuf::from(output_dir)
.join(".faces")
.join(&file_uuid)
.join(trace_id.to_string())
.join(format!("{}.jpg", frame));
if cached_path.exists() {
tracing::debug!("[thumbnail] Using cached face crop: {}", cached_path.display());
let bytes = tokio::fs::read(&cached_path)
.await
.map_err(|e| {
tracing::warn!("[thumbnail] Failed to read cached file: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
// Validate cached JPEG
crate::core::thumbnail::validator::validate_jpeg(&bytes).map_err(|e| {
tracing::warn!("[thumbnail] Cached JPEG validation failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
return Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(bytes))
.unwrap());
}
// Cached file not found, fallback to ffmpeg
tracing::debug!("[thumbnail] Cached file not found, falling back to ffmpeg");
}
// Step 2: Fallback to ffmpeg on-demand extraction
let row: Option<(String, Option<i64>, Option<i32>, Option<i32>)> = sqlx::query_as(&format!(
"SELECT file_path, total_frames, width, height FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let (file_path,) = row.ok_or(StatusCode::NOT_FOUND)?;
let (file_path, total_frames, video_width, video_height) = row.ok_or(StatusCode::NOT_FOUND)?;
if let Some(total) = total_frames {
if total > 0 {
crate::core::thumbnail::validator::validate_frame(frame, total).map_err(|e| {
tracing::warn!("[thumbnail] Frame validation failed: {}", e);
StatusCode::BAD_REQUEST
})?;
}
}
if let (Some(x), Some(y), Some(w), Some(h)) = (q.x, q.y, q.w, q.h) {
if let (Some(vw), Some(vh)) = (video_width, video_height) {
crate::core::thumbnail::validator::validate_crop(x, y, w, h, vw, vh).map_err(|e| {
tracing::warn!("[thumbnail] Crop validation failed: {}", e);
StatusCode::BAD_REQUEST
})?;
}
}
let select = format!("select=eq(n\\,{})", frame);
let vf = if let (Some(x), Some(y), Some(w), Some(h)) = (q.x, q.y, q.w, q.h) {
@@ -755,6 +828,11 @@ async fn face_thumbnail(
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
crate::core::thumbnail::validator::validate_jpeg(&output.stdout).map_err(|e| {
tracing::warn!("[thumbnail] JPEG validation failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
@@ -849,3 +927,127 @@ async fn video_clip(
.body(Body::from(output.stdout))
.unwrap())
}
async fn stranger_video(
State(state): State<crate::api::types::AppState>,
Path((file_uuid, stranger_id)): Path<(String, i32)>,
Query(params): Query<std::collections::HashMap<String, String>>,
) -> Result<impl IntoResponse, StatusCode> {
stranger_video_inner(&state, &file_uuid, stranger_id, &params).await
}
async fn stranger_video_inner(
state: &crate::api::types::AppState,
file_uuid: &str,
stranger_id: i32,
params: &std::collections::HashMap<String, String>,
) -> Result<impl IntoResponse, StatusCode> {
use axum::http::header;
use uuid::Uuid;
tracing::info!("[stranger_video] Starting for file={}, stranger={}", file_uuid, stranger_id);
let (mode, audio) = parse_video_params(&params);
let videos_table = schema::table_name("videos");
tracing::debug!("[stranger_video] videos_table: {}", videos_table);
let row: Option<(String, f64, i32, i32)> = sqlx::query_as(&format!(
"SELECT file_path, COALESCE(fps, 24.0), COALESCE(width, 0), COALESCE(height, 0) FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[stranger_video] Video query error: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let (video_path, fps, _width, _height) = row.ok_or_else(|| {
tracing::error!("[stranger_video] Video not found for uuid={}", file_uuid);
StatusCode::NOT_FOUND
})?;
tracing::info!("[stranger_video] Found video: path={}, fps={}", video_path, fps);
// Query face detections by stranger_id directly
let face_table = schema::table_name("face_detections");
tracing::debug!("[stranger_video] face_table: {}", face_table);
// frame_number is BIGINT (i64) in database
let rows: Vec<(i64, i32, i32, i32, i32)> = sqlx::query_as(&format!(
"SELECT frame_number, x, y, width, height FROM {} WHERE file_uuid = $1 AND stranger_id = $2 ORDER BY frame_number",
face_table
))
.bind(&file_uuid).bind(stranger_id)
.fetch_all(state.db.pool()).await
.unwrap_or_else(|e| {
tracing::error!("[stranger_video] Face query error: {}", e);
vec![]
});
tracing::info!("[stranger_video] Found {} faces", rows.len());
if rows.is_empty() {
tracing::error!("[stranger_video] No faces found for stranger_id={}", stranger_id);
return Err(StatusCode::NOT_FOUND);
}
let first_frame = rows[0].0;
let last_frame = rows[rows.len() - 1].0;
let start_sec = first_frame as f64 / fps;
let padding = params
.get("padding")
.and_then(|s| s.parse().ok())
.unwrap_or(2.0);
let duration = (last_frame - first_frame) as f64 / fps + padding * 2.0;
let seek = (start_sec - padding).max(0.0);
tracing::info!("[stranger_video] Frame range: {} - {}, time: {:.2}s - {:.2}s",
first_frame, last_frame, seek, seek + duration);
// Only support normal mode for stranger video
let tmp = std::env::temp_dir().join(format!("stranger_{}.mp4", Uuid::new_v4()));
let tmp_str = tmp.to_str().unwrap_or("").to_string();
let sk = seek.to_string();
let du = duration.to_string();
let mut cmd_args = vec!["-ss", &sk, "-i", &video_path, "-t", &du, "-c", "copy"];
if audio == "off" {
cmd_args.push("-an");
}
cmd_args.extend_from_slice(&["-y", &tmp_str]);
tracing::debug!("[stranger_video] ffmpeg args: {:?}", cmd_args);
let result = ffmpeg_cmd()
.args(&cmd_args)
.output()
.map_err(|e| {
tracing::error!("[stranger_video] ffmpeg spawn error: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
if !result.status.success() {
tracing::error!("[stranger_video] ffmpeg failed: {}", String::from_utf8_lossy(&result.stderr));
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
tracing::info!("[stranger_video] ffmpeg success, output size: {} bytes", result.stdout.len());
let data = tokio::fs::read(&tmp)
.await
.map_err(|e| {
tracing::error!("[stranger_video] Read output error: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let _ = std::fs::remove_file(&tmp);
tracing::info!("[stranger_video] Returning video, size: {} bytes", data.len());
Ok(Response::builder()
.header(header::CONTENT_TYPE, "video/mp4")
.header(header::CONTENT_LENGTH, data.len())
.body(Body::from(data))
.unwrap())
}