Files
momentry_core/src/api/media_api.rs
Accusys 8f013cbdbc fix: trace debug mode — show all traces in frame range with interpolation
Debug overlay now lists every trace visible in the current frame range,
including interpolated frames (continuous from first to last detection).
Format per trace line:
  Trace {id}: start_frame={n}  Identity={name}
2026-05-14 15:09:34 +08:00

604 lines
21 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use axum::{
body::Body,
extract::{Path, Query, State},
http::{header, StatusCode},
response::{IntoResponse, Response},
routing::get,
Router,
};
use once_cell::sync::Lazy;
use crate::core::db::{schema, PostgresDb};
static FFMPEG: Lazy<String> = Lazy::new(|| {
std::env::var("MOMENTRY_FFMPEG").unwrap_or_else(|_| {
let full = "/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg";
if std::path::Path::new(full).exists() {
full.to_string()
} else {
"ffmpeg".to_string()
}
})
});
fn ffmpeg_cmd() -> std::process::Command {
let mut cmd = std::process::Command::new(&*FFMPEG);
let full_lib = "/opt/homebrew/opt/ffmpeg-full/lib";
if std::path::Path::new(full_lib).exists() {
cmd.env("DYLD_LIBRARY_PATH", full_lib);
}
cmd
}
pub fn bbox_routes() -> Router<crate::api::server::AppState> {
Router::new()
.route(
"/api/v1/file/:file_uuid/video/bbox",
get(bbox_overlay_video),
)
.route(
"/api/v1/file/:file_uuid/trace/:trace_id/video",
get(trace_video),
)
.route("/api/v1/file/:file_uuid/video", get(stream_video))
.route("/api/v1/file/:file_uuid/thumbnail", get(face_thumbnail))
}
/// 5×7 bitmap font — each character 5 wide × 7 tall
/// Encoding: col 0=0x10, col 1=0x08, col 2=0x04, col 3=0x02, col 4=0x01
fn bitmap_char(c: char) -> [u8; 7] {
match c.to_ascii_lowercase() {
'0' => [0x0E, 0x11, 0x13, 0x15, 0x19, 0x11, 0x0E],
'1' => [0x04, 0x0C, 0x04, 0x04, 0x04, 0x04, 0x0E],
'2' => [0x0E, 0x11, 0x01, 0x02, 0x04, 0x08, 0x1F],
'3' => [0x0E, 0x11, 0x01, 0x06, 0x01, 0x11, 0x0E],
'4' => [0x02, 0x06, 0x0A, 0x12, 0x1F, 0x02, 0x02],
'5' => [0x1F, 0x10, 0x1E, 0x01, 0x01, 0x11, 0x0E],
'6' => [0x0E, 0x10, 0x1E, 0x11, 0x11, 0x11, 0x0E],
'7' => [0x1F, 0x01, 0x02, 0x04, 0x08, 0x10, 0x10],
'8' => [0x0E, 0x11, 0x11, 0x0E, 0x11, 0x11, 0x0E],
'9' => [0x0E, 0x11, 0x11, 0x0F, 0x01, 0x11, 0x0E],
'a' => [0x0E, 0x11, 0x11, 0x1F, 0x11, 0x11, 0x11],
'b' => [0x1E, 0x11, 0x11, 0x1E, 0x11, 0x11, 0x1E],
'c' => [0x0E, 0x11, 0x10, 0x10, 0x10, 0x11, 0x0E],
'd' => [0x1C, 0x12, 0x11, 0x11, 0x11, 0x12, 0x1C],
'e' => [0x1F, 0x10, 0x10, 0x1E, 0x10, 0x10, 0x1F],
'f' => [0x1F, 0x10, 0x10, 0x1E, 0x10, 0x10, 0x10],
't' => [0x04, 0x04, 0x1F, 0x04, 0x04, 0x04, 0x06],
'_' => [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1F],
' ' => [0x00; 7],
_ => [0x00; 7],
}
}
/// Width of one character in pixels (5 cols × 3px/dot = 15px)
const CHAR_W: i32 = 5 * 3;
/// Spacing between characters (px)
const CHAR_GAP: i32 = 4;
/// Total advance per character
const CHAR_ADVANCE: i32 = CHAR_W + CHAR_GAP;
fn render_text(
parts: &mut Vec<String>,
text: &str,
origin_x: i32,
origin_y: i32,
enable: Option<i32>,
) -> i32 {
let mut px = origin_x;
for ch in text.chars() {
let bm = bitmap_char(ch);
for (row, bits) in bm.iter().enumerate() {
for col in 0..5 {
if bits & (1 << (4 - col)) != 0 {
let x = px + col as i32 * 3;
let y = origin_y + row as i32 * 3;
if let Some(offset) = enable {
parts.push(format!(
"drawbox=x={}:y={}:w=3:h=3:color=white@1.0:t=fill:enable='eq(n,{})'",
x, y, offset
));
} else {
parts.push(format!(
"drawbox=x={}:y={}:w=3:h=3:color=white@1.0:t=fill",
x, y
));
}
}
}
}
px += CHAR_ADVANCE;
}
px
}
#[derive(Debug, serde::Deserialize)]
struct BboxParams {
start: Option<i32>,
end: Option<i32>,
face_uuid: Option<String>,
duration: Option<f64>,
}
async fn bbox_overlay_video(
State(state): State<crate::api::server::AppState>,
Path(file_uuid): Path<String>,
Query(p): Query<BboxParams>,
) -> Result<impl IntoResponse, StatusCode> {
let videos_table = schema::table_name("videos");
let row: Option<(String,)> = sqlx::query_as(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let (video_path,) = row.ok_or(StatusCode::NOT_FOUND)?;
let start_f = p.start.unwrap_or(0);
let end_f = p.end.unwrap_or(i32::MAX);
let face_fuid = p.face_uuid.as_deref().unwrap_or(&file_uuid);
let duration = p.duration.unwrap_or(10.0);
// Get FPS
let fps: f64 = sqlx::query_scalar(&format!(
"SELECT COALESCE(fps, 24.0) FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.unwrap_or(24.0);
let start_sec = start_f as f64 / fps;
// Get face bboxes
let face_table = schema::table_name("face_detections");
let rows: Vec<(i32, i32, i32, i32, i32, Option<i32>, Option<String>)> = sqlx::query_as(
&format!("SELECT frame_number, x, y, width, height, trace_id, face_id FROM {} WHERE file_uuid = $1 AND frame_number BETWEEN $2 AND $3 ORDER BY frame_number", face_table)
)
.bind(face_fuid).bind(start_f).bind(end_f)
.fetch_all(state.db.pool()).await
.unwrap_or_else(|e| { tracing::error!("bbox query error: {}", e); vec![] });
// Build filters
let mut parts: Vec<String> = Vec::new();
let mut is_first = true;
for (frame, x, y, w, h, trace_id, _) in &rows {
let text = format!("t{}", trace_id.unwrap_or(0));
if is_first {
is_first = false;
// Persistent bbox: thin pale red border
parts.push(format!(
"drawbox=x={}:y={}:w={}:h={}:color=red@0.3:thickness=4",
x, y, w, h
));
// Always-on text: top-left of bbox with padding
let tx = *x + 6;
let ty = *y + 6;
render_text(&mut parts, &text, tx, ty, None);
} else {
let offset = frame - start_f;
// Per-frame bbox: thick bright red
parts.push(format!(
"drawbox=x={}:y={}:w={}:h={}:color=red@0.8:thickness=8:enable='eq(n,{})'",
x, y, w, h, offset
));
// Per-frame text
let tx = *x + 6;
let ty = *y + 6;
render_text(&mut parts, &text, tx, ty, Some(offset));
}
}
let vf = if parts.is_empty() {
"null".to_string()
} else {
parts.join(",")
};
let tmp = std::env::temp_dir().join(format!("bbox_{}.mp4", uuid::Uuid::new_v4()));
let tmp_str = tmp.to_str().unwrap_or("").to_string();
let status = ffmpeg_cmd()
.args([
"-ss",
&start_sec.to_string(),
"-i",
&video_path,
"-t",
&duration.to_string(),
"-vf",
&vf,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"28",
"-an",
"-movflags",
"+faststart",
"-y",
&tmp_str,
])
.status()
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if !status.success() {
let _ = std::fs::remove_file(&tmp);
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
let data = tokio::fs::read(&tmp)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let _ = std::fs::remove_file(&tmp);
Ok(Response::builder()
.header(header::CONTENT_TYPE, "video/mp4")
.header(header::CONTENT_LENGTH, data.len())
.body(Body::from(data))
.unwrap())
}
fn parse_range(range: &str, file_size: u64) -> (u64, u64) {
let r = range.trim_start_matches("bytes=");
let parts: Vec<&str> = r.split('-').collect();
let start = parts[0].parse::<u64>().unwrap_or(0);
let end = if parts.len() > 1 && !parts[1].is_empty() {
parts[1].parse::<u64>().unwrap_or(file_size - 1)
} else {
file_size - 1
};
(start.min(file_size - 1), end.min(file_size - 1))
}
async fn trace_video(
State(state): State<crate::api::server::AppState>,
Path((file_uuid, trace_id)): Path<(String, i32)>,
Query(params): Query<std::collections::HashMap<String, String>>,
) -> Result<impl IntoResponse, StatusCode> {
use axum::http::header;
let mode = params.get("mode").map(|s| s.as_str()).unwrap_or("normal");
let videos_table = schema::table_name("videos");
let row: Option<(String, f64, i32, i32)> = sqlx::query_as(&format!(
"SELECT file_path, COALESCE(fps, 24.0), COALESCE(width, 0), COALESCE(height, 0) FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let (video_path, fps, _width, _height) = row.ok_or(StatusCode::NOT_FOUND)?;
// Query face detections to find frame range for target trace
let face_table = schema::table_name("face_detections");
let rows: Vec<(i32, i32, i32, i32, i32)> = sqlx::query_as(&format!(
"SELECT frame_number, x, y, width, height FROM {} WHERE file_uuid = $1 AND trace_id = $2 ORDER BY frame_number",
face_table
))
.bind(&file_uuid).bind(trace_id)
.fetch_all(state.db.pool()).await
.unwrap_or_else(|e| { tracing::error!("trace query error: {}", e); vec![] });
if rows.is_empty() {
return Err(StatusCode::NOT_FOUND);
}
let first_frame = rows[0].0;
let last_frame = rows[rows.len() - 1].0;
let start_sec = first_frame as f64 / fps;
let padding = params
.get("padding")
.and_then(|s| s.parse().ok())
.unwrap_or(2.0);
let duration = (last_frame - first_frame) as f64 / fps + padding * 2.0;
let seek = (start_sec - padding).max(0.0);
// === NORMAL MODE: raw video without overlays ===
if mode == "normal" {
let tmp = std::env::temp_dir().join(format!("trace_{}.mp4", uuid::Uuid::new_v4()));
let tmp_str = tmp.to_str().unwrap_or("").to_string();
let result = ffmpeg_cmd()
.args(["-ss", &seek.to_string(), "-i", &video_path, "-t", &duration.to_string(),
"-c", "copy", "-an", "-y", &tmp_str])
.output()
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if !result.status.success() {
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
let data = tokio::fs::read(&tmp).await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let _ = std::fs::remove_file(&tmp);
return Ok(Response::builder()
.header(header::CONTENT_TYPE, "video/mp4")
.header(header::CONTENT_LENGTH, data.len())
.body(Body::from(data))
.unwrap());
}
// === DEBUG MODE: text overlay, list all traces in frame range ===
let start_fn = (start_sec * fps) as i32;
let end_fn = (start_sec + duration) as i32;
// Query all traces with identity names in the visible frame range
let identities_table = schema::table_name("identities");
let all_rows: Vec<(i32, i32, Option<String>)> = sqlx::query_as(&format!(
"SELECT fd.trace_id, fd.frame_number, i.name \
FROM {} fd \
LEFT JOIN {} i ON fd.identity_id = i.id \
WHERE fd.file_uuid = $1 AND fd.frame_number BETWEEN $2 AND $3 AND fd.trace_id IS NOT NULL \
ORDER BY fd.trace_id, fd.frame_number",
face_table, identities_table
))
.bind(&file_uuid).bind(start_fn).bind(end_fn)
.fetch_all(state.db.pool()).await
.unwrap_or_default();
// Group frames by trace_id, compute start_frame per trace
use std::collections::HashMap;
let mut trace_frames: HashMap<i32, Vec<i32>> = HashMap::new();
let mut trace_identity: HashMap<i32, String> = HashMap::new();
for (tid, fn_, name_opt) in &all_rows {
trace_frames.entry(*tid).or_default().push(*fn_);
if let Some(name) = name_opt {
trace_identity.entry(*tid).or_insert_with(|| name.clone());
}
}
// Query cut_id for this segment
let cut_table = schema::table_name("cut");
let cut_id: i32 = sqlx::query_scalar(
&format!("SELECT scene_number FROM {} WHERE file_uuid = $1 AND start_frame <= $2 AND end_frame >= $2 LIMIT 1", cut_table)
)
.bind(&file_uuid).bind(first_frame)
.fetch_optional(state.db.pool()).await
.unwrap_or(None)
.unwrap_or(0);
// Sort traces for consistent ordering
let mut sorted_traces: Vec<(i32, &Vec<i32>)> = trace_frames.iter().map(|(k, v)| (*k, v)).collect();
sorted_traces.sort_by_key(|(tid, _)| *tid);
let frame_offset = first_frame as i64 - (padding * fps) as i64;
let fps_str = &fps.to_string();
// Build drawtext entries
let mut parts: Vec<String> = Vec::new();
// Static header
parts.push(format!(
"drawtext=text='File UUID: {}':fontsize=14:fontcolor=white:box=1:boxcolor=black@0.6:x=10:y=12", file_uuid
));
parts.push(format!(
"drawtext=text='Cut: {}':fontsize=14:fontcolor=white:box=1:boxcolor=black@0.6:x=10:y=34", cut_id
));
parts.push(format!(
"drawtext=text='Frame: %{{eif:n+{}:d}} Time: %{{eif:(n+{})*100/{}:d}}s':fontsize=14:fontcolor=white:box=1:boxcolor=black@0.6:x=10:y=56",
frame_offset, frame_offset, fps_str
));
// Per-trace entries: show trace_id, start_frame, identity name
// Position starts at y=78, increments by 22 per trace
let mut y_pos = 78;
for (tid, frames) in &sorted_traces {
let start = frames.iter().min().unwrap_or(&first_frame);
let identity = trace_identity.get(tid).map(|s| s.as_str()).unwrap_or("unknown");
let label = format!("Trace {}: start={} {}", tid, start, identity);
// Continuous range (interpolated): visible from first to last frame
let enable = format!("between(n,{},{})", frames[0] as i64 - frame_offset, frames[frames.len() - 1] as i64 - frame_offset);
parts.push(format!(
"drawtext=text='{}':fontsize=14:fontcolor=white:box=1:boxcolor=black@0.6:x=10:y={}:enable='{}'",
label, y_pos, enable
));
y_pos += 22;
}
let filter_text = parts.join(",");
let filter_file = std::env::temp_dir().join(format!("vf_{}.txt", uuid::Uuid::new_v4()));
let _ = std::fs::write(&filter_file, &filter_text);
let filter_path = filter_file.to_str().unwrap_or("");
let tmp = std::env::temp_dir().join(format!("trace_{}.mp4", uuid::Uuid::new_v4()));
let tmp_str = tmp.to_str().unwrap_or("").to_string();
let result = ffmpeg_cmd()
.args(["-ss", &seek.to_string(), "-i", &video_path, "-t", &duration.to_string(),
"-/filter_complex", &filter_path,
"-c:v", "libx264", "-preset", "ultrafast", "-crf", "28",
"-c:a", "aac", "-movflags", "+faststart", "-y", &tmp_str])
.output()
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if !result.status.success() {
let stderr = String::from_utf8_lossy(&result.stderr);
tracing::error!("ffmpeg failed: {}", &stderr[..stderr.len().min(300)]);
let _ = std::fs::remove_file(&filter_file);
let _ = std::fs::remove_file(&tmp);
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
let data = tokio::fs::read(&tmp).await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let _ = std::fs::remove_file(&filter_file);
let _ = std::fs::remove_file(&tmp);
Ok(Response::builder()
.header(header::CONTENT_TYPE, "video/mp4")
.header(header::CONTENT_LENGTH, data.len())
.body(Body::from(data))
.unwrap())
}
async fn stream_video(
State(state): State<crate::api::server::AppState>,
Path(file_uuid): Path<String>,
Query(params): Query<std::collections::HashMap<String, String>>,
request: axum::http::Request<Body>,
) -> Result<impl IntoResponse, StatusCode> {
use tokio::io::{AsyncReadExt, AsyncSeekExt};
let videos_table = schema::table_name("videos");
let row: Option<(String,)> = sqlx::query_as(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let (file_path,) = row.ok_or(StatusCode::NOT_FOUND)?;
let src = std::path::PathBuf::from(&file_path);
if !src.exists() {
return Err(StatusCode::NOT_FOUND);
}
// Chunk extraction with start/end params
if let (Some(s), Some(e)) = (params.get("start"), params.get("end")) {
let start: f64 = s.parse().unwrap_or(0.0);
let end: f64 = e.parse().unwrap_or(0.0);
let dur = end - start;
if dur <= 0.0 {
return Err(StatusCode::BAD_REQUEST);
}
let tmp = std::env::temp_dir().join(format!("chunk_{}.mp4", uuid::Uuid::new_v4()));
let tmp_str = tmp.to_str().unwrap_or("").to_string();
let status = ffmpeg_cmd()
.args([
"-ss",
&start.to_string(),
"-i",
&file_path,
"-t",
&dur.to_string(),
"-c",
"copy",
"-movflags",
"+faststart",
"-y",
&tmp_str,
])
.status()
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if !status.success() {
let _ = std::fs::remove_file(&tmp);
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
let data = tokio::fs::read(&tmp)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let _ = std::fs::remove_file(&tmp);
return Ok(Response::builder()
.header(header::CONTENT_TYPE, "video/mp4")
.header(header::CONTENT_LENGTH, data.len())
.body(Body::from(data))
.unwrap());
}
let file_size = src.metadata().map(|m| m.len()).unwrap_or(0);
let content_type = "video/mp4";
let range_hdr = request
.headers()
.get(header::RANGE)
.and_then(|v| v.to_str().ok());
if let Some(range_str) = range_hdr {
let (start, end) = parse_range(range_str, file_size);
let length = end - start + 1;
let mut file = tokio::fs::File::open(&src)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
file.seek(std::io::SeekFrom::Start(start))
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let take = file.take(length);
let stream = tokio_util::io::ReaderStream::new(take);
let body = Body::from_stream(stream);
Ok(Response::builder()
.status(StatusCode::PARTIAL_CONTENT)
.header(header::CONTENT_TYPE, content_type)
.header(
header::CONTENT_RANGE,
format!("bytes {}-{}/{}", start, end, file_size),
)
.header(header::CONTENT_LENGTH, length)
.body(body)
.unwrap())
} else {
let file = tokio::fs::File::open(&src)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let stream = tokio_util::io::ReaderStream::new(file);
let body = Body::from_stream(stream);
Ok(Response::builder()
.header(header::CONTENT_TYPE, content_type)
.header(header::CONTENT_LENGTH, file_size)
.header(header::ACCEPT_RANGES, "bytes")
.body(body)
.unwrap())
}
}
#[derive(Debug, serde::Deserialize)]
struct ThumbQuery {
frame: i64,
x: Option<i32>,
y: Option<i32>,
w: Option<i32>,
h: Option<i32>,
}
async fn face_thumbnail(
State(state): State<crate::api::server::AppState>,
Path(file_uuid): Path<String>,
Query(q): Query<ThumbQuery>,
) -> Result<impl IntoResponse, StatusCode> {
let videos_table = schema::table_name("videos");
let row: Option<(String,)> = sqlx::query_as(&format!(
"SELECT file_path FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let (file_path,) = row.ok_or(StatusCode::NOT_FOUND)?;
let select = format!("select=eq(n\\,{})", q.frame);
let vf = if let (Some(x), Some(y), Some(w), Some(h)) = (q.x, q.y, q.w, q.h) {
format!("{},crop={}:{}:{}:{}", select, w, h, x, y)
} else {
select
};
let output = ffmpeg_cmd()
.args([
"-i",
&file_path,
"-vf",
&vf,
"-frames:v",
"1",
"-f",
"image2pipe",
"-vcodec",
"mjpeg",
"-",
])
.output()
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if !output.status.success() {
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "image/jpeg")
.header(header::CACHE_CONTROL, "public, max-age=86400")
.body(Body::from(output.stdout))
.unwrap())
}