cleanup: remove dead code and duplicate docs
- Remove session-ses_2f27.md (161KB raw session log) - Remove 49 ROOT_* duplicate files across REFERENCE/ - Remove 14 duplicate files between REFERENCE/ root and history/ - Remove asr_legacy.rs (dead code, replaced by asr.rs) - Remove src/core/worker/ (duplicate JobWorker) - Remove src/core/layers/ (empty directory) - Remove 4 .bak files in src/ - Remove 7 dead private methods in worker/processor.rs - Remove backup directory from git tracking
This commit is contained in:
Warren
@@ -228,6 +228,11 @@ impl From<VideoRow> for VideoRecord {
|
||||
registration_time: row.registration_time,
|
||||
total_frames: row.total_frames.unwrap_or(0) as u64,
|
||||
parent_uuid: row.parent_uuid,
|
||||
cut_done: false,
|
||||
cut_count: 0,
|
||||
cut_max_duration: 0.0,
|
||||
scene_done: false,
|
||||
audio_tracks: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -254,6 +259,11 @@ pub struct VideoRecord {
|
||||
pub registration_time: Option<String>,
|
||||
pub total_frames: u64,
|
||||
pub parent_uuid: Option<String>,
|
||||
pub cut_done: bool,
|
||||
pub cut_count: i32,
|
||||
pub cut_max_duration: f64,
|
||||
pub scene_done: bool,
|
||||
pub audio_tracks: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -332,9 +342,9 @@ pub struct MonitorJob {
|
||||
pub progress_current: i32,
|
||||
pub error_count: i32,
|
||||
pub last_error: Option<String>,
|
||||
pub started_at: Option<chrono::NaiveDateTime>,
|
||||
pub updated_at: Option<chrono::NaiveDateTime>,
|
||||
pub created_at: chrono::NaiveDateTime,
|
||||
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub updated_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub processors: Vec<String>,
|
||||
pub completed_processors: Vec<String>,
|
||||
pub failed_processors: Vec<String>,
|
||||
@@ -393,17 +403,80 @@ impl ProcessorType {
|
||||
}
|
||||
}
|
||||
|
||||
/// 預估 CPU 使用率（0.0 ~ 1.0, 1.0 = 一個完整核心）
|
||||
pub fn estimated_cpu(&self) -> f64 {
|
||||
match self {
|
||||
ProcessorType::Asr => 1.0,
|
||||
ProcessorType::Cut => 0.5,
|
||||
ProcessorType::Yolo => 0.3,
|
||||
ProcessorType::Ocr => 0.8,
|
||||
ProcessorType::Face => 0.6,
|
||||
ProcessorType::Pose => 0.4,
|
||||
ProcessorType::Asrx => 0.8,
|
||||
ProcessorType::VisualChunk => 0.3,
|
||||
ProcessorType::Scene => 0.3,
|
||||
}
|
||||
}
|
||||
|
||||
/// 是否使用 GPU
|
||||
pub fn uses_gpu(&self) -> bool {
|
||||
match self {
|
||||
ProcessorType::Yolo | ProcessorType::Face | ProcessorType::Pose => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 預估記憶體使用量 (MB)
|
||||
pub fn estimated_memory_mb(&self) -> u64 {
|
||||
match self {
|
||||
ProcessorType::Asr => 2048,
|
||||
ProcessorType::Cut => 512,
|
||||
ProcessorType::Yolo => 1024,
|
||||
ProcessorType::Ocr => 1024,
|
||||
ProcessorType::Face => 1536,
|
||||
ProcessorType::Pose => 1024,
|
||||
ProcessorType::Asrx => 2048,
|
||||
ProcessorType::VisualChunk => 512,
|
||||
ProcessorType::Scene => 512,
|
||||
}
|
||||
}
|
||||
|
||||
/// 使用的模型名稱（如有）
|
||||
pub fn model_name(&self) -> Option<&'static str> {
|
||||
match self {
|
||||
ProcessorType::Asr => Some("faster-whisper"),
|
||||
ProcessorType::Cut => None,
|
||||
ProcessorType::Yolo => Some("yolov8n"),
|
||||
ProcessorType::Ocr => Some("paddleocr"),
|
||||
ProcessorType::Face => Some("insightface/buffalo_l"),
|
||||
ProcessorType::Pose => Some("mediapipe/pose"),
|
||||
ProcessorType::Asrx => Some("speechbrain/ecapa-tdnn"),
|
||||
ProcessorType::VisualChunk => None,
|
||||
ProcessorType::Scene => Some("places365"),
|
||||
}
|
||||
}
|
||||
|
||||
/// 依賴的其他 Processor（需先完成才能執行）
|
||||
pub fn dependencies(&self) -> Vec<ProcessorType> {
|
||||
match self {
|
||||
ProcessorType::Asrx => vec![ProcessorType::Asr],
|
||||
ProcessorType::VisualChunk => vec![ProcessorType::Yolo],
|
||||
ProcessorType::Scene => vec![ProcessorType::Cut],
|
||||
_ => vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn all() -> Vec<ProcessorType> {
|
||||
vec![
|
||||
ProcessorType::Asr,
|
||||
ProcessorType::Cut,
|
||||
ProcessorType::Scene,
|
||||
ProcessorType::Asr,
|
||||
ProcessorType::Asrx,
|
||||
ProcessorType::Yolo,
|
||||
ProcessorType::Ocr,
|
||||
ProcessorType::Face,
|
||||
ProcessorType::Pose,
|
||||
ProcessorType::Asrx,
|
||||
ProcessorType::VisualChunk,
|
||||
ProcessorType::Scene,
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -701,8 +774,8 @@ impl PostgresDb {
|
||||
.await?;
|
||||
|
||||
// Chunks
|
||||
sqlx::query("CREATE TABLE IF NOT EXISTS chunks (id SERIAL PRIMARY KEY, uuid VARCHAR(32) NOT NULL, chunk_id VARCHAR(64) NOT NULL, chunk_index INTEGER NOT NULL, chunk_type VARCHAR(32) NOT NULL, start_time DOUBLE PRECISION NOT NULL, end_time DOUBLE PRECISION NOT NULL, fps DOUBLE PRECISION DEFAULT 24.0, start_frame BIGINT DEFAULT 0, end_frame BIGINT DEFAULT 0, content JSONB NOT NULL, metadata JSONB, vector_id VARCHAR(64), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(uuid, chunk_id))").execute(pool).await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_chunks_uuid ON chunks(uuid)")
|
||||
sqlx::query("CREATE TABLE IF NOT EXISTS chunks (id SERIAL PRIMARY KEY, file_uuid VARCHAR(32) NOT NULL, chunk_id VARCHAR(64) NOT NULL, chunk_index INTEGER NOT NULL, chunk_type VARCHAR(32) NOT NULL, start_time DOUBLE PRECISION NOT NULL, end_time DOUBLE PRECISION NOT NULL, fps DOUBLE PRECISION DEFAULT 24.0, start_frame BIGINT DEFAULT 0, end_frame BIGINT DEFAULT 0, content JSONB NOT NULL, metadata JSONB, vector_id VARCHAR(64), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(file_uuid, chunk_id))").execute(pool).await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_uuid)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(chunk_type)")
|
||||
@@ -765,15 +838,13 @@ impl PostgresDb {
|
||||
.await?;
|
||||
|
||||
// Chunks Rule 1
|
||||
sqlx::query("CREATE TABLE IF NOT EXISTS chunks_rule1 (id UUID PRIMARY KEY DEFAULT gen_random_uuid(), asset_uuid VARCHAR(32) NOT NULL REFERENCES videos(uuid) ON DELETE CASCADE, start_frame BIGINT NOT NULL, end_frame BIGINT NOT NULL, content TEXT NOT NULL, speaker_id VARCHAR(50), created_at TIMESTAMPTZ DEFAULT NOW())").execute(pool).await?;
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_chunks_rule1_asset ON chunks_rule1(asset_uuid)",
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
sqlx::query("CREATE TABLE IF NOT EXISTS chunks_rule1 (id UUID PRIMARY KEY DEFAULT gen_random_uuid(), file_uuid VARCHAR(32) NOT NULL REFERENCES videos(uuid) ON DELETE CASCADE, start_frame BIGINT NOT NULL, end_frame BIGINT NOT NULL, content TEXT NOT NULL, speaker_id VARCHAR(50), created_at TIMESTAMPTZ DEFAULT NOW())").execute(pool).await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_chunks_rule1_asset ON chunks_rule1(file_uuid)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
// Jobs (Legacy/P0)
|
||||
sqlx::query("CREATE TABLE IF NOT EXISTS jobs (id UUID PRIMARY KEY, asset_uuid VARCHAR(32) NOT NULL REFERENCES videos(uuid) ON DELETE CASCADE, processor_list TEXT[], assigned_processor_id UUID, rule VARCHAR(20), status VARCHAR(20) DEFAULT 'QUEUED', total_frames BIGINT DEFAULT 0, processed_frames BIGINT DEFAULT 0, error_message TEXT, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW())").execute(pool).await?;
|
||||
sqlx::query("CREATE TABLE IF NOT EXISTS jobs (id UUID PRIMARY KEY, file_uuid VARCHAR(32) NOT NULL REFERENCES videos(uuid) ON DELETE CASCADE, processor_list TEXT[], assigned_processor_id UUID, rule VARCHAR(20), status VARCHAR(20) DEFAULT 'QUEUED', total_frames BIGINT DEFAULT 0, processed_frames BIGINT DEFAULT 0, error_message TEXT, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW())").execute(pool).await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
@@ -1162,8 +1233,8 @@ impl PostgresDb {
|
||||
.await?;
|
||||
|
||||
sqlx::query(&format!(
|
||||
"DELETE FROM {} WHERE video_id IN (SELECT id FROM {} WHERE uuid = $1)",
|
||||
processor_results, videos
|
||||
"DELETE FROM {} WHERE file_uuid = $1",
|
||||
processor_results
|
||||
))
|
||||
.bind(uuid)
|
||||
.execute(&self.pool)
|
||||
@@ -2026,21 +2097,19 @@ impl PostgresDb {
|
||||
r#"
|
||||
INSERT INTO {} (
|
||||
file_uuid, processor_type, coordinate_type, coordinate_index,
|
||||
timestamp, data, identity_id, confidence
|
||||
) VALUES ($1, $2, 'frame', $3, $4, $5, $6, $7)
|
||||
start_frame, end_frame, start_time, data
|
||||
) VALUES ($1, $2, 'frame', $3, $3, $3, $4, $5)
|
||||
"#,
|
||||
table
|
||||
);
|
||||
|
||||
for (coord_idx, ts, data, id, conf) in chunks {
|
||||
for (coord_idx, ts, data, _id, _conf) in chunks {
|
||||
sqlx::query(&query)
|
||||
.bind(file_uuid)
|
||||
.bind(processor_type)
|
||||
.bind(*coord_idx)
|
||||
.bind(*ts)
|
||||
.bind(data)
|
||||
.bind(*id)
|
||||
.bind(*conf)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
@@ -2060,7 +2129,7 @@ impl PostgresDb {
|
||||
let query = format!(
|
||||
r#"
|
||||
INSERT INTO {} (
|
||||
file_uuid, processor_type, coordinate_type, coordinate_index,
|
||||
file_uuid, processor_type, coordinate_type, coordinate_index,
|
||||
start_frame, end_frame, start_time, end_time, data
|
||||
) VALUES ($1, 'asr', 'time', $2, $3, $4, $5, $6, $7)
|
||||
"#,
|
||||
@@ -2402,10 +2471,10 @@ impl PostgresDb {
|
||||
offset: i64,
|
||||
) -> Result<Vec<IdentityChunkRecord>> {
|
||||
let query = r#"
|
||||
SELECT c.id, c.uuid as file_uuid, c.chunk_id, c.chunk_type,
|
||||
SELECT c.id, c.file_uuid, c.chunk_id, c.chunk_type,
|
||||
c.start_time, c.end_time, c.text_content, c.content
|
||||
FROM chunks c
|
||||
WHERE c.uuid IN (
|
||||
WHERE c.file_uuid IN (
|
||||
SELECT DISTINCT fi.file_uuid
|
||||
FROM file_identities fi
|
||||
JOIN identities i ON fi.identity_id = i.id
|
||||
@@ -2504,9 +2573,9 @@ impl PostgresDb {
|
||||
|
||||
sqlx::query(&format!(
|
||||
r#"
|
||||
INSERT INTO {} (file_id, uuid, chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids)
|
||||
INSERT INTO {} (file_id, file_uuid, chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15, $16, $17, $18)
|
||||
ON CONFLICT (uuid, chunk_id) DO UPDATE SET
|
||||
ON CONFLICT (file_uuid, chunk_id) DO UPDATE SET
|
||||
start_time = EXCLUDED.start_time,
|
||||
end_time = EXCLUDED.end_time,
|
||||
fps = EXCLUDED.fps,
|
||||
@@ -2579,9 +2648,9 @@ impl PostgresDb {
|
||||
|
||||
sqlx::query(&format!(
|
||||
r#"
|
||||
INSERT INTO {} (file_id, uuid, chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids)
|
||||
INSERT INTO {} (file_id, file_uuid, chunk_id, chunk_index, chunk_type, start_time, end_time, fps, start_frame, end_frame, text_content, content, metadata, vector_id, frame_count, pre_chunk_ids, parent_chunk_id, child_chunk_ids)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15, $16, $17, $18)
|
||||
ON CONFLICT (uuid, chunk_id) DO UPDATE SET
|
||||
ON CONFLICT (file_uuid, chunk_id) DO UPDATE SET
|
||||
start_time = EXCLUDED.start_time,
|
||||
end_time = EXCLUDED.end_time,
|
||||
fps = EXCLUDED.fps,
|
||||
@@ -2626,7 +2695,7 @@ impl PostgresDb {
|
||||
pub async fn get_chunks_by_uuid(&self, uuid: &str) -> Result<Vec<Chunk>> {
|
||||
let table = schema::table_name("chunks");
|
||||
let rows = sqlx::query(&format!(
|
||||
"SELECT COALESCE(file_id, 0) as file_id, uuid, chunk_id, chunk_index, chunk_type, COALESCE(fps, 24.0) as fps, COALESCE(start_frame, 0) as start_frame, COALESCE(end_frame, 0) as end_frame, text_content, content, metadata, vector_id, COALESCE(frame_count, 0) as frame_count, pre_chunk_ids, parent_chunk_id::text as parent_chunk_id, child_chunk_ids, visual_stats FROM {} WHERE uuid = $1 ORDER BY chunk_index",
|
||||
"SELECT COALESCE(file_id, 0) as file_id, file_uuid as uuid, chunk_id, chunk_index, chunk_type, COALESCE(fps, 24.0) as fps, COALESCE(start_frame, 0) as start_frame, COALESCE(end_frame, 0) as end_frame, text_content, content, metadata, vector_id, COALESCE(frame_count, 0) as frame_count, pre_chunk_ids, parent_chunk_id::text as parent_chunk_id, child_chunk_ids, visual_stats FROM {} WHERE file_uuid = $1 ORDER BY chunk_index",
|
||||
table
|
||||
))
|
||||
.bind(uuid)
|
||||
@@ -3264,36 +3333,40 @@ impl PostgresDb {
|
||||
let sql = match uuid {
|
||||
Some(_) => &format!(
|
||||
r#"
|
||||
SELECT c.chunk_id, c.uuid, c.chunk_index, c.chunk_type, c.start_frame, c.end_frame, c.fps, c.start_time, c.end_time,
|
||||
SELECT c.chunk_id, c.file_uuid, c.chunk_index, c.chunk_type, c.start_frame, c.end_frame, c.fps, c.start_time, c.end_time,
|
||||
c.text_content, GREATEST(ts_rank_cd(c.search_vector, to_tsquery('english', $1)), ts_rank_cd(pc.summary_tsvector, to_tsquery('english', $1))) as bm25_score,
|
||||
c.visual_stats,
|
||||
pc.metadata->'structured_summary' as scene_summary,
|
||||
c.parent_chunk_id::integer
|
||||
FROM {} c
|
||||
LEFT JOIN parent_chunks pc ON c.parent_chunk_id = pc.id::varchar
|
||||
WHERE (c.search_vector @@ to_tsquery('english', $1) OR pc.summary_tsvector @@ to_tsquery('english', $1)) AND c.uuid = $2
|
||||
WHERE (c.search_vector @@ to_tsquery('english', $1) OR pc.summary_tsvector @@ to_tsquery('english', $1) OR c.text_content ILIKE $3) AND c.file_uuid = $2
|
||||
ORDER BY bm25_score DESC
|
||||
LIMIT $3
|
||||
LIMIT $4
|
||||
"#,
|
||||
table
|
||||
),
|
||||
None => &format!(
|
||||
r#"
|
||||
SELECT c.chunk_id, c.uuid, c.chunk_index, c.chunk_type, c.start_frame, c.end_frame, c.fps, c.start_time, c.end_time,
|
||||
SELECT c.chunk_id, c.file_uuid, c.chunk_index, c.chunk_type, c.start_frame, c.end_frame, c.fps, c.start_time, c.end_time,
|
||||
c.text_content, GREATEST(ts_rank_cd(c.search_vector, to_tsquery('english', $1)), ts_rank_cd(pc.summary_tsvector, to_tsquery('english', $1))) as bm25_score,
|
||||
c.visual_stats,
|
||||
pc.metadata->'structured_summary' as scene_summary,
|
||||
c.parent_chunk_id::integer
|
||||
FROM {} c
|
||||
LEFT JOIN parent_chunks pc ON c.parent_chunk_id = pc.id::varchar
|
||||
WHERE (c.search_vector @@ to_tsquery('english', $1) OR pc.summary_tsvector @@ to_tsquery('english', $1))
|
||||
WHERE (c.search_vector @@ to_tsquery('english', $1) OR pc.summary_tsvector @@ to_tsquery('english', $1) OR c.text_content ILIKE $2)
|
||||
ORDER BY bm25_score DESC
|
||||
LIMIT $2
|
||||
LIMIT $3
|
||||
"#,
|
||||
table
|
||||
),
|
||||
};
|
||||
|
||||
// 使用 pg_trgm 支援中英文模糊搜尋
|
||||
// ILIKE 支援中文 LIKE 匹配，pg_trgm 的 similarity() 可做更精確的排名
|
||||
let ilike_pattern = format!("%{}%", query);
|
||||
|
||||
let rows: Vec<(
|
||||
String,
|
||||
String,
|
||||
@@ -3310,10 +3383,11 @@ impl PostgresDb {
|
||||
Option<serde_json::Value>,
|
||||
Option<i32>,
|
||||
)> = match uuid {
|
||||
Some(_) => {
|
||||
Some(u) => {
|
||||
sqlx::query_as(sql)
|
||||
.bind(&tsquery)
|
||||
.bind(uuid)
|
||||
.bind(u)
|
||||
.bind(&ilike_pattern)
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await?
|
||||
@@ -3321,6 +3395,7 @@ impl PostgresDb {
|
||||
None => {
|
||||
sqlx::query_as(sql)
|
||||
.bind(&tsquery)
|
||||
.bind(&ilike_pattern)
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await?
|
||||
@@ -3809,6 +3884,54 @@ impl PostgresDb {
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn get_all_running_jobs(&self, limit: i32) -> Result<Vec<MonitorJob>> {
|
||||
let monitor_jobs = schema::table_name("monitor_jobs");
|
||||
let rows = sqlx::query(&format!(
|
||||
r#"
|
||||
SELECT id, uuid, video_path, status, current_processor, progress_total, progress_current,
|
||||
error_count, last_error, started_at, updated_at, created_at,
|
||||
processors, completed_processors, failed_processors, video_id
|
||||
FROM {}
|
||||
WHERE status = 'running'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $1
|
||||
"#,
|
||||
monitor_jobs
|
||||
))
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
let jobs: Vec<MonitorJob> = rows
|
||||
.into_iter()
|
||||
.map(|r| {
|
||||
let status_str: String = r.get(3);
|
||||
let status =
|
||||
MonitorJobStatus::from_db_str(&status_str).unwrap_or(MonitorJobStatus::Running);
|
||||
MonitorJob {
|
||||
id: r.get(0),
|
||||
uuid: r.get(1),
|
||||
video_path: r.get(2),
|
||||
status,
|
||||
current_processor: r.get(4),
|
||||
progress_total: r.get(5),
|
||||
progress_current: r.get(6),
|
||||
error_count: r.get(7),
|
||||
last_error: r.get(8),
|
||||
started_at: r.get(9),
|
||||
updated_at: r.get(10),
|
||||
created_at: r.get(11),
|
||||
processors: r.get::<Option<Vec<String>>, _>(12).unwrap_or_default(),
|
||||
completed_processors: r.get::<Option<Vec<String>>, _>(13).unwrap_or_default(),
|
||||
failed_processors: r.get::<Option<Vec<String>>, _>(14).unwrap_or_default(),
|
||||
video_id: r.get(15),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
pub async fn get_pending_jobs(&self, limit: i32) -> Result<Vec<MonitorJob>> {
|
||||
let monitor_jobs = schema::table_name("monitor_jobs");
|
||||
let rows = sqlx::query(&format!(
|
||||
@@ -3817,7 +3940,7 @@ impl PostgresDb {
|
||||
error_count, last_error, started_at, updated_at, created_at,
|
||||
processors, completed_processors, failed_processors, video_id
|
||||
FROM {}
|
||||
WHERE status IN ('pending', 'running')
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $1
|
||||
"#,
|
||||
@@ -4322,7 +4445,7 @@ impl PostgresDb {
|
||||
name: &str,
|
||||
) -> Result<crate::core::person_identity::Identity> {
|
||||
let identity = sqlx::query_as::<_, crate::core::person_identity::Identity>(
|
||||
r#"INSERT INTO identities (name) VALUES ($1) ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id, name, embedding::text, metadata, created_at"#,
|
||||
r#"INSERT INTO identities (name) VALUES ($1) ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id, name, identity_embedding::text as embedding, metadata, created_at"#,
|
||||
)
|
||||
.bind(name)
|
||||
.fetch_one(&self.pool)
|
||||
@@ -4371,7 +4494,7 @@ impl PostgresDb {
|
||||
binding_value: &str,
|
||||
) -> Result<Option<crate::core::person_identity::Identity>> {
|
||||
let identity = sqlx::query_as::<_, crate::core::person_identity::Identity>(
|
||||
"SELECT i.id, i.name, i.embedding::text, i.metadata, i.created_at FROM identities i JOIN identity_bindings b ON i.id = b.identity_id WHERE b.identity_type = $1 AND b.identity_value = $2",
|
||||
"SELECT i.id, i.name, i.identity_embedding::text as embedding, i.metadata, i.created_at FROM identities i JOIN identity_bindings b ON i.id = b.identity_id WHERE b.identity_type = $1 AND b.identity_value = $2",
|
||||
)
|
||||
.bind(binding_type)
|
||||
.bind(binding_value)
|
||||
@@ -4389,12 +4512,12 @@ impl PostgresDb {
|
||||
) -> Result<Vec<crate::core::person_identity::Identity>> {
|
||||
let query = if !search.is_empty() {
|
||||
sqlx::query_as::<_, crate::core::person_identity::Identity>(
|
||||
"SELECT id, name, embedding::text, metadata, created_at FROM identities WHERE name ILIKE $1 ORDER BY id LIMIT $2 OFFSET $3",
|
||||
"SELECT id, name, identity_embedding::text as embedding, metadata, created_at FROM identities WHERE name ILIKE $1 ORDER BY id LIMIT $2 OFFSET $3",
|
||||
)
|
||||
.bind(format!("%{}%", search))
|
||||
} else {
|
||||
sqlx::query_as::<_, crate::core::person_identity::Identity>(
|
||||
"SELECT id, name, embedding::text, metadata, created_at FROM identities ORDER BY id LIMIT $1 OFFSET $2",
|
||||
"SELECT id, name, identity_embedding::text as embedding, metadata, created_at FROM identities ORDER BY id LIMIT $1 OFFSET $2",
|
||||
)
|
||||
};
|
||||
let identities = query.bind(limit).bind(offset).fetch_all(&self.pool).await?;
|
||||
@@ -4407,7 +4530,7 @@ impl PostgresDb {
|
||||
id: i64,
|
||||
) -> Result<Option<crate::core::person_identity::Identity>> {
|
||||
let identity = sqlx::query_as::<_, crate::core::person_identity::Identity>(
|
||||
"SELECT id, name, embedding::text, metadata, created_at FROM identities WHERE id = $1",
|
||||
"SELECT id, name, identity_embedding::text as embedding, metadata, created_at FROM identities WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
@@ -4716,7 +4839,7 @@ impl PostgresDb {
|
||||
"speaker_ids"
|
||||
};
|
||||
let query = format!(
|
||||
"SELECT id, start_frame, end_frame, content FROM chunks WHERE uuid = $1 AND $2::text = ANY({}::text[]) ORDER BY start_frame",
|
||||
"SELECT id, start_frame, end_frame, content FROM chunks WHERE file_uuid = $1 AND $2::text = ANY({}::text[]) ORDER BY start_frame",
|
||||
column
|
||||
);
|
||||
|
||||
@@ -4836,7 +4959,7 @@ mod tests {
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
fps: 30.0,
|
||||
probe_json: Some("{}".to_string()),
|
||||
probe_json: Some(serde_json::from_str("{}").unwrap()),
|
||||
storage: StorageStatus::default(),
|
||||
status: VideoStatus::Pending,
|
||||
processing_status: None,
|
||||
@@ -4847,6 +4970,11 @@ mod tests {
|
||||
registration_time: None,
|
||||
total_frames: 0,
|
||||
parent_uuid: None,
|
||||
cut_done: false,
|
||||
cut_count: 0,
|
||||
cut_max_duration: 0.0,
|
||||
scene_done: false,
|
||||
audio_tracks: None,
|
||||
};
|
||||
|
||||
let json = serde_json::to_string(&record).unwrap();
|
||||
@@ -4935,13 +5063,18 @@ mod tests {
|
||||
error_count: 0,
|
||||
last_error: None,
|
||||
started_at: Some(
|
||||
NaiveDateTime::parse_from_str("2024-01-01 10:00:00", "%Y-%m-%d %H:%M:%S").unwrap(),
|
||||
chrono::DateTime::parse_from_rfc3339("2024-01-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&chrono::Utc),
|
||||
),
|
||||
updated_at: Some(
|
||||
NaiveDateTime::parse_from_str("2024-01-01 10:05:00", "%Y-%m-%d %H:%M:%S").unwrap(),
|
||||
chrono::DateTime::parse_from_rfc3339("2024-01-01T10:05:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&chrono::Utc),
|
||||
),
|
||||
created_at: NaiveDateTime::parse_from_str("2024-01-01 09:55:00", "%Y-%m-%d %H:%M:%S")
|
||||
.unwrap(),
|
||||
created_at: chrono::DateTime::parse_from_rfc3339("2024-01-01T09:55:00Z")
|
||||
.unwrap()
|
||||
.into(),
|
||||
processors: vec!["asr".to_string(), "cut".to_string()],
|
||||
completed_processors: vec!["asr".to_string()],
|
||||
failed_processors: vec![],
|
||||
@@ -4968,7 +5101,7 @@ mod tests {
|
||||
"last_error": null,
|
||||
"started_at": null,
|
||||
"updated_at": null,
|
||||
"created_at": "2024-01-01T00:00:00",
|
||||
"created_at": "2024-01-01T00:00:00Z",
|
||||
"processors": ["asr", "cut"],
|
||||
"completed_processors": [],
|
||||
"failed_processors": [],
|
||||
|
||||
@@ -88,6 +88,62 @@ impl QdrantDb {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 將向量寫入指定 collection（支援多 collection）
|
||||
pub async fn upsert_vector_to_collection(
|
||||
&self,
|
||||
collection: &str,
|
||||
point_id: u64,
|
||||
vector: &[f32],
|
||||
payload: Option<serde_json::Value>,
|
||||
) -> Result<()> {
|
||||
let url = format!(
|
||||
"{}/collections/{}/points?wait=true",
|
||||
self.base_url, collection
|
||||
);
|
||||
|
||||
tracing::debug!("Qdrant upsert URL: {}, collection: {}", url, collection);
|
||||
|
||||
let points = if let Some(p) = payload {
|
||||
serde_json::json!({
|
||||
"points": [{
|
||||
"id": point_id,
|
||||
"vector": vector,
|
||||
"payload": p,
|
||||
}]
|
||||
})
|
||||
} else {
|
||||
serde_json::json!({
|
||||
"points": [{
|
||||
"id": point_id,
|
||||
"vector": vector,
|
||||
}]
|
||||
})
|
||||
};
|
||||
|
||||
let response = self
|
||||
.client
|
||||
.put(&url)
|
||||
.header("api-key", &self.api_key)
|
||||
.json(&points)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to send upsert request to Qdrant")?;
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let response_text = response.text().await.unwrap_or_default();
|
||||
tracing::error!("Qdrant upsert failed: {} - {}", status, response_text);
|
||||
anyhow::bail!(
|
||||
"Qdrant upsert failed with status {}: {}",
|
||||
status,
|
||||
response_text
|
||||
);
|
||||
}
|
||||
|
||||
tracing::debug!("Successfully upserted vector for point: {}", point_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn upsert_vector(
|
||||
&self,
|
||||
chunk_id: &str,
|
||||
|
||||
@@ -371,6 +371,11 @@ impl RedisClient {
|
||||
processor: &str,
|
||||
status: &str,
|
||||
error: Option<&str>,
|
||||
frames_processed: i32,
|
||||
chunks_produced: i32,
|
||||
total_frames: i32,
|
||||
retry_count: i32,
|
||||
pid: i32,
|
||||
) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
@@ -378,13 +383,24 @@ impl RedisClient {
|
||||
|
||||
let now = chrono::Utc::now().to_rfc3339();
|
||||
|
||||
let mut fields: Vec<(&str, &str)> = vec![("status", status), ("updated_at", &now)];
|
||||
let mut fields: Vec<(&str, String)> = vec![
|
||||
("status", status.to_string()),
|
||||
("updated_at", now),
|
||||
("current", frames_processed.to_string()),
|
||||
("total", total_frames.to_string()),
|
||||
("frames_processed", frames_processed.to_string()),
|
||||
("chunks_produced", chunks_produced.to_string()),
|
||||
("retry_count", retry_count.to_string()),
|
||||
("pid", pid.to_string()),
|
||||
];
|
||||
|
||||
if let Some(err) = error {
|
||||
fields.push(("error", err));
|
||||
fields.push(("error", err.to_string()));
|
||||
}
|
||||
|
||||
let _: Option<String> = conn.hset_multiple(&key, &fields).await?;
|
||||
let field_refs: Vec<(&str, &str)> = fields.iter().map(|(k, v)| (*k, v.as_str())).collect();
|
||||
|
||||
let _: Option<String> = conn.hset_multiple(&key, &field_refs).await?;
|
||||
let _: bool = conn.expire(&key, 86400).await?;
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user