M4 handover: coordinate fixes, detector registry, deploy v2, YOLOv8s, identity lifecycle
- Fix swift_pose/swift_ocr Y-flip bugs (BUG-003~006) - Add heuristic_scene module + post-processing trigger (replaces Places365) - YOLOv5nu → YOLOv8s CoreML (+33% detections, +390% scene indicators) - Per-table SQL export (split 4.7GB single file → 478MB max per table) - Version/build check in deploy.sh (compare /health vs file_info.json) - Add file_uuid column to identities table + backfill - Identity pre-clean step in deploy (avoids UNIQUE conflicts on re-deploy) - Stranger_xxx naming fix with UUID context - Add DETECTOR_REGISTRY.md (25 detectors), DETECTOR_SELECTION_SOP.md - Update SPATIAL_COORDINATE_REGISTRY.md (P layer, 6-layer architecture) - New IDENTITY_LIFECYCLE.md - M4 response docs for deploy_script_fix and 111614 test report
This commit is contained in:
Accusys
@@ -12,6 +12,10 @@ use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
fn dir_size(path: &Path) -> u64 {
|
||||
path.read_dir().map(|d| d.filter_map(|e| e.ok()).filter_map(|e| e.metadata().ok()).map(|m| m.len()).sum()).unwrap_or(0)
|
||||
}
|
||||
|
||||
const DEMO_DIR: &str = "/Users/accusys/momentry/var/sftpgo/data/demo";
|
||||
const OUTPUT_DIR: &str = "/Users/accusys/momentry/output_dev";
|
||||
const RELEASE_DIR: &str = "/Users/accusys/momentry_core_0.1/release/files";
|
||||
@@ -353,77 +357,133 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"status": "completed",
|
||||
"momentry_version": env!("CARGO_PKG_VERSION"),
|
||||
"momentry_build": env!("BUILD_GIT_HASH"),
|
||||
});
|
||||
fs::write(outdir.join("file_info.json"), serde_json::to_string_pretty(&info)?)?;
|
||||
|
||||
// Export data.sql
|
||||
let sql_path = outdir.join("data.sql");
|
||||
// Export per-table .sql files (avoid single 4.7GB psql load)
|
||||
let sql_dir = outdir.join("sql");
|
||||
fs::create_dir_all(&sql_dir)?;
|
||||
let tables = [
|
||||
("dev.videos", "file_uuid"),
|
||||
("dev.chunk", "file_uuid"),
|
||||
("dev.chunk_vectors", "uuid"),
|
||||
("dev.face_detections", "file_uuid"),
|
||||
("dev.tkg_nodes", "file_uuid"),
|
||||
("dev.tkg_edges", "file_uuid"),
|
||||
];
|
||||
|
||||
{
|
||||
let mut f = fs::File::create(&sql_path)?;
|
||||
writeln!(f, "-- Release package: {}", uuid)?;
|
||||
writeln!(f, "BEGIN;")?;
|
||||
writeln!(f)?;
|
||||
let mut import_order = vec!["master.sql"];
|
||||
|
||||
for (tbl, col) in &tables {
|
||||
writeln!(f, "-- {} WHERE {} = '{}'", tbl, col, uuid)?;
|
||||
// Get columns
|
||||
let parts: Vec<&str> = tbl.split('.').collect();
|
||||
let cols = psql_exec(&format!(
|
||||
"SELECT string_agg(column_name, ', ' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='{}' AND table_name='{}' AND is_updatable='YES'",
|
||||
parts[0], parts[1]
|
||||
))?;
|
||||
|
||||
// COPY
|
||||
let data = psql_exec(&format!(
|
||||
"COPY (SELECT * FROM {} WHERE {} = '{}') TO STDOUT WITH CSV HEADER",
|
||||
tbl, col, uuid
|
||||
))?;
|
||||
|
||||
if !data.is_empty() {
|
||||
writeln!(f, "COPY {} ({}) FROM STDIN WITH CSV HEADER;", tbl, cols)?;
|
||||
writeln!(f, "{}", data)?;
|
||||
writeln!(f, "\\.")?;
|
||||
writeln!(f)?;
|
||||
}
|
||||
}
|
||||
// Export identities referenced by this file
|
||||
writeln!(f, "-- dev.identities (referenced by face_detections)")?;
|
||||
let cols = psql_exec("SELECT string_agg(column_name, ', ' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='dev' AND table_name='identities' AND is_updatable='YES'")?;
|
||||
fn write_table_sql(outdir: &Path, tbl: &str, col: &str, uuid: &str, psql_exec: &dyn Fn(&str) -> Result<String>) -> Result<()> {
|
||||
let safe_name = tbl.replace('.', "_");
|
||||
let path = outdir.join(format!("{}.sql", safe_name));
|
||||
let parts: Vec<&str> = tbl.split('.').collect();
|
||||
let cols = psql_exec(&format!(
|
||||
"SELECT string_agg(column_name, ', ' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='{}' AND table_name='{}' AND is_updatable='YES'",
|
||||
parts[0], parts[1]
|
||||
))?;
|
||||
let data = psql_exec(&format!(
|
||||
"COPY (SELECT DISTINCT i.* FROM dev.identities i INNER JOIN dev.face_detections fd ON fd.identity_id = i.id WHERE fd.file_uuid = '{}') TO STDOUT WITH CSV HEADER", uuid
|
||||
"COPY (SELECT * FROM {} WHERE {} = '{}') TO STDOUT WITH CSV HEADER",
|
||||
tbl, col, uuid
|
||||
))?;
|
||||
if !data.is_empty() {
|
||||
let mut f = fs::File::create(&path)?;
|
||||
writeln!(f, "-- {} WHERE {} = '{}'", tbl, col, uuid)?;
|
||||
writeln!(f, "COPY {} ({}) FROM STDIN WITH CSV HEADER;", tbl, cols)?;
|
||||
writeln!(f, "{}", data)?;
|
||||
writeln!(f, "\\.")?;
|
||||
let sz = fs::metadata(&path)?.len();
|
||||
println!(" sql/{} ({} MB)", safe_name, sz / 1024 / 1024);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
for (tbl, col) in &tables {
|
||||
write_table_sql(&sql_dir, tbl, col, uuid, &|q| psql_exec(q))?;
|
||||
}
|
||||
|
||||
// Export identities with file_uuid (direct column, no JOIN needed)
|
||||
// FILE LOCAL: file_uuid = '{uuid}'
|
||||
// GLOBAL (cross-file): tmdb identities + user-defined (exclude inactive auto)
|
||||
let idents_name = "dev_identities";
|
||||
let idents_path = sql_dir.join(format!("{}.sql", idents_name));
|
||||
{
|
||||
let idents_query = format!(
|
||||
"COPY (SELECT * FROM dev.identities WHERE file_uuid = '{}' OR (file_uuid IS NULL AND source IN ('tmdb', 'merged', 'user_defined'))) TO STDOUT WITH CSV HEADER", uuid
|
||||
);
|
||||
let cols = psql_exec(&format!(
|
||||
"SELECT string_agg(column_name, ', ' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='dev' AND table_name='identities' AND is_updatable='YES'"
|
||||
))?;
|
||||
let data = psql_exec(&idents_query)?;
|
||||
if !data.is_empty() {
|
||||
let mut f = fs::File::create(&idents_path)?;
|
||||
writeln!(f, "-- dev.identities WHERE file_uuid = '{}' OR global (tmdb/merged/user_defined)", uuid)?;
|
||||
writeln!(f, "COPY dev.identities ({}) FROM STDIN WITH CSV HEADER;", cols)?;
|
||||
writeln!(f, "{}", data)?;
|
||||
writeln!(f, "\\.")?;
|
||||
writeln!(f)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Export identity_bindings for identities referenced by this file
|
||||
writeln!(f, "-- dev.identity_bindings (for identities in face_detections)")?;
|
||||
let cols = psql_exec("SELECT string_agg(column_name, ', ' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='dev' AND table_name='identity_bindings' AND is_updatable='YES'")?;
|
||||
let data = psql_exec(&format!(
|
||||
"COPY (SELECT DISTINCT ib.* FROM dev.identity_bindings ib INNER JOIN dev.face_detections fd ON fd.identity_id = ib.identity_id WHERE fd.file_uuid = '{}') TO STDOUT WITH CSV HEADER", uuid
|
||||
// Export identity_bindings with custom query
|
||||
let binds_name = "dev_identity_bindings";
|
||||
let binds_path = sql_dir.join(format!("{}.sql", binds_name));
|
||||
{
|
||||
let binds_query = format!(
|
||||
"COPY (SELECT DISTINCT ib.* FROM dev.identity_bindings ib INNER JOIN dev.face_detections fd ON fd.identity_id = ib.identity_id AND fd.trace_id IS NOT NULL WHERE fd.file_uuid = '{}' AND ib.identity_value IN (SELECT DISTINCT trace_id::text FROM dev.face_detections WHERE file_uuid = '{}' AND trace_id IS NOT NULL)) TO STDOUT WITH CSV HEADER", uuid, uuid
|
||||
);
|
||||
let cols = psql_exec(&format!(
|
||||
"SELECT string_agg(column_name, ', ' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema='dev' AND table_name='identity_bindings' AND is_updatable='YES'"
|
||||
))?;
|
||||
let data = psql_exec(&binds_query)?;
|
||||
if !data.is_empty() {
|
||||
let mut f = fs::File::create(&binds_path)?;
|
||||
writeln!(f, "-- dev.identity_bindings (from face_detections JOIN)")?;
|
||||
writeln!(f, "COPY dev.identity_bindings ({}) FROM STDIN WITH CSV HEADER;", cols)?;
|
||||
writeln!(f, "{}", data)?;
|
||||
writeln!(f, "\\.")?;
|
||||
writeln!(f)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Write master.sql (import order, runs BEGIN/COMMIT around all)
|
||||
let master_path = sql_dir.join("master.sql");
|
||||
{
|
||||
let mut f = fs::File::create(&master_path)?;
|
||||
writeln!(f, "BEGIN;")?;
|
||||
writeln!(f)?;
|
||||
|
||||
writeln!(f, "\\i sql/dev_videos.sql")?;
|
||||
writeln!(f, "\\i sql/dev_chunk.sql")?;
|
||||
writeln!(f, "\\i sql/dev_chunk_vectors.sql")?;
|
||||
writeln!(f, "\\i sql/dev_face_detections.sql")?;
|
||||
writeln!(f, "\\i sql/dev_identities.sql")?;
|
||||
writeln!(f, "\\i sql/dev_identity_bindings.sql")?;
|
||||
writeln!(f, "\\i sql/dev_tkg_nodes.sql")?;
|
||||
writeln!(f, "\\i sql/dev_tkg_edges.sql")?;
|
||||
|
||||
writeln!(f, "COMMIT;")?;
|
||||
}
|
||||
|
||||
let sql_size = fs::metadata(&sql_path)?.len();
|
||||
println!(" data.sql ({} MB)", sql_size / 1024 / 1024);
|
||||
// Write legacy data.sql that sources master via psql -f (backward compat)
|
||||
let sql_path = outdir.join("data.sql");
|
||||
{
|
||||
let mut f = fs::File::create(&sql_path)?;
|
||||
writeln!(f, "-- Release package: {} — see sql/ for per-table files", uuid)?;
|
||||
writeln!(f, "BEGIN;")?;
|
||||
writeln!(f, "\\i sql/dev_videos.sql")?;
|
||||
writeln!(f, "\\i sql/dev_chunk.sql")?;
|
||||
writeln!(f, "\\i sql/dev_chunk_vectors.sql")?;
|
||||
writeln!(f, "\\i sql/dev_face_detections.sql")?;
|
||||
writeln!(f, "\\i sql/dev_identities.sql")?;
|
||||
writeln!(f, "\\i sql/dev_identity_bindings.sql")?;
|
||||
writeln!(f, "\\i sql/dev_tkg_nodes.sql")?;
|
||||
writeln!(f, "\\i sql/dev_tkg_edges.sql")?;
|
||||
writeln!(f, "COMMIT;")?;
|
||||
}
|
||||
|
||||
let sql_dir_sz = dir_size(&sql_dir);
|
||||
println!(" sql/ directory ({} MB total)", sql_dir_sz / 1024 / 1024);
|
||||
|
||||
// Copy video file
|
||||
if !file_path.is_empty() {
|
||||
@@ -487,6 +547,39 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
|
||||
}
|
||||
let tsize = fs::metadata(&tarball)?.len();
|
||||
println!("\n Package: {} ({} MB)", tarball.display(), tsize / 1024 / 1024);
|
||||
|
||||
// Sanity check: warn if any sql file is suspiciously large
|
||||
println!(" Checking sql/ file sizes...");
|
||||
for entry in fs::read_dir(&sql_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|s| s.to_str()) == Some("sql") && path.is_file() {
|
||||
let sz = fs::metadata(&path)?.len() as f64 / 1024.0 / 1024.0;
|
||||
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("?");
|
||||
match name {
|
||||
"dev_videos" | "master" if sz > 1.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 1 MB", name, sz as u64),
|
||||
"dev_chunk" if sz > 2.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 2 MB for ~2.4K chunks", name, sz as u64),
|
||||
"dev_identities" if sz > 1.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 1 MB for ~428 identities", name, sz as u64),
|
||||
"dev_identity_bindings" if sz > 5.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 5 MB for ~7.6K bindings", name, sz as u64),
|
||||
"dev_tkg_nodes" if sz > 10.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 10 MB for ~6.4K nodes", name, sz as u64),
|
||||
"dev_tkg_edges" if sz > 20.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 20 MB for ~21K edges", name, sz as u64),
|
||||
"dev_face_detections" if sz > 1000.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 1000 MB for ~70K faces (512D emb)", name, sz as u64),
|
||||
"dev_chunk_vectors" if sz > 200.0 =>
|
||||
println!(" ⚠️ {} is {} MB, expected < 200 MB for ~2.4K chunks (768D emb)", name, sz as u64),
|
||||
_ => {}
|
||||
}
|
||||
if sz > 2000.0 {
|
||||
println!(" ⚠️ {} is {:.0} MB — unusually large, verify query", name, sz);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -646,7 +739,9 @@ fn cmd_stats() -> Result<()> {
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
dotenv::from_filename(".env.development").ok();
|
||||
if dotenv::from_filename("/Users/accusys/momentry_core_0.1/.env.development").is_err() {
|
||||
let _ = dotenv::from_filename(".env.development");
|
||||
}
|
||||
let cli = Cli::parse();
|
||||
let db = PostgresDb::new(&config::DATABASE_URL).await?;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user