admin/momentry_core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: trace-level matching, health watcher/worker status, timezone config

Browse Source
This commit is contained in:
Accusys
2026-05-21 01:08:30 +08:00
parent 8ede4be159
commit bebaa743ed
60 changed files with 6110 additions and 1586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
src/bin/release.rs
View File

@@ -13,7 +13,14 @@ use std::path::{Path, PathBuf};
use std::process::Command;
fn dir_size(path: &Path) -> u64 {
path.read_dir().map(|d| d.filter_map(|e| e.ok()).filter_map(|e| e.metadata().ok()).map(|m| m.len()).sum()).unwrap_or(0)
path.read_dir()
.map(|d| {
d.filter_map(|e| e.ok())
.filter_map(|e| e.metadata().ok())
.map(|m| m.len())
.sum()
})
.unwrap_or(0)
}
const DEMO_DIR: &str = "/Users/accusys/momentry/var/sftpgo/data/demo";
@@ -22,7 +29,10 @@ const RELEASE_DIR: &str = "/Users/accusys/momentry_core_0.1/release/files";
const PG_BIN: &str = "/Users/accusys/pgsql/18.3/bin";
#[derive(Parser)]
#[command(name = "release", about = "Release Manager — deploy/undeploy video packages")]
#[command(
name = "release",
about = "Release Manager — deploy/undeploy video packages"
)]
struct Cli {
#[command(subcommand)]
command: Commands,
@@ -107,7 +117,12 @@ fn extract_tarball(tarball: &Path) -> Result<PathBuf> {
fs::create_dir_all(&tmpdir)?;
let status = Command::new("tar")
.args(["-xzf", tarball.to_str().unwrap(), "-C", tmpdir.to_str().unwrap()])
.args([
"-xzf",
tarball.to_str().unwrap(),
"-C",
tmpdir.to_str().unwrap(),
])
.status()
.context("tar extraction failed")?;
if !status.success() {
@@ -127,8 +142,8 @@ fn extract_tarball(tarball: &Path) -> Result<PathBuf> {
/// Get file_info.json from package directory
fn read_file_info(pkg_dir: &Path) -> Result<serde_json::Value> {
let info_path = pkg_dir.join("file_info.json");
let content = fs::read_to_string(&info_path)
.with_context(|| format!("Cannot read {:?}", info_path))?;
let content =
fs::read_to_string(&info_path).with_context(|| format!("Cannot read {:?}", info_path))?;
serde_json::from_str(&content).context("Invalid file_info.json")
}
@@ -140,7 +155,10 @@ async fn cmd_deploy(db: &PostgresDb, tarball: &str) -> Result<()> {
anyhow::bail!("File not found: {}", tarball);
}
println!("=== Deploy: {} ===", tarball_path.file_name().unwrap().to_str().unwrap());
println!(
"=== Deploy: {} ===",
tarball_path.file_name().unwrap().to_str().unwrap()
);
// Extract
let pkg_dir = extract_tarball(tarball_path)?;
@@ -148,7 +166,9 @@ async fn cmd_deploy(db: &PostgresDb, tarball: &str) -> Result<()> {
// Read file_info
let info = read_file_info(&pkg_dir)?;
let uuid = info["file_uuid"].as_str().context("Missing file_uuid in file_info.json")?;
let uuid = info["file_uuid"]
.as_str()
.context("Missing file_uuid in file_info.json")?;
let file_name = info["file_name"].as_str().unwrap_or("?");
println!("UUID: {}\nVideo: {}", uuid, file_name);
@@ -168,7 +188,8 @@ async fn cmd_deploy(db: &PostgresDb, tarball: &str) -> Result<()> {
let entry = entry?;
let fname = entry.file_name();
let fname_str = fname.to_str().unwrap_or("");
if fname_str.ends_with(".mp4") || fname_str.ends_with(".mov") || fname_str.ends_with(".avi") {
if fname_str.ends_with(".mp4") || fname_str.ends_with(".mov") || fname_str.ends_with(".avi")
{
let dest = Path::new(DEMO_DIR).join(&fname);
if !dest.exists() {
fs::copy(entry.path(), &dest)?;
@@ -192,12 +213,15 @@ async fn cmd_deploy(db: &PostgresDb, tarball: &str) -> Result<()> {
println!("Output files copied to {}", OUTPUT_DIR);
// Verify
let chunk_count: (i64,) = sqlx::query_as(
"SELECT COUNT(*) FROM dev.chunk WHERE file_uuid = $1"
).bind(uuid).fetch_one(db.pool()).await?;
let face_count: (i64,) = sqlx::query_as(
"SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid = $1"
).bind(uuid).fetch_one(db.pool()).await?;
let chunk_count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM dev.chunk WHERE file_uuid = $1")
.bind(uuid)
.fetch_one(db.pool())
.await?;
let face_count: (i64,) =
sqlx::query_as("SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid = $1")
.bind(uuid)
.fetch_one(db.pool())
.await?;
// Cleanup
fs::remove_dir_all(&pkg_dir.parent().unwrap_or(&pkg_dir))?;
@@ -213,9 +237,11 @@ async fn cmd_deploy(db: &PostgresDb, tarball: &str) -> Result<()> {
async fn cmd_undeploy(db: &PostgresDb, uuid: &str, skip_confirm: bool) -> Result<()> {
// Get video info
let rows: Vec<(String, String)> = sqlx::query_as(
"SELECT file_name, file_path FROM dev.videos WHERE file_uuid = $1"
).bind(uuid).fetch_all(db.pool()).await?;
let rows: Vec<(String, String)> =
sqlx::query_as("SELECT file_name, file_path FROM dev.videos WHERE file_uuid = $1")
.bind(uuid)
.fetch_all(db.pool())
.await?;
if rows.is_empty() {
anyhow::bail!("UUID {} not found in DB", uuid);
@@ -252,7 +278,9 @@ async fn cmd_undeploy(db: &PostgresDb, uuid: &str, skip_confirm: bool) -> Result
println!(" {}: {} rows deleted", tbl, result.rows_affected());
}
sqlx::query("DELETE FROM dev.videos WHERE file_uuid = $1")
.bind(uuid).execute(db.pool()).await?;
.bind(uuid)
.execute(db.pool())
.await?;
println!(" dev.videos: removed");
// Delete output files
@@ -270,7 +298,10 @@ async fn cmd_undeploy(db: &PostgresDb, uuid: &str, skip_confirm: bool) -> Result
let vp = Path::new(file_path);
if vp.exists() {
fs::remove_file(vp)?;
println!(" Video file: removed ({})", vp.file_name().unwrap().to_str().unwrap_or("?"));
println!(
" Video file: removed ({})",
vp.file_name().unwrap().to_str().unwrap_or("?")
);
}
}
@@ -292,11 +323,15 @@ async fn cmd_list(db: &PostgresDb) -> Result<()> {
"SELECT file_uuid, file_name, duration, status,
(SELECT COUNT(*) FROM dev.chunk WHERE file_uuid = v.file_uuid) as chunks,
(SELECT COUNT(*) FROM dev.face_detections WHERE file_uuid = v.file_uuid) as faces
FROM dev.videos v ORDER BY id DESC"
).fetch_all(db.pool()).await?;
FROM dev.videos v ORDER BY id DESC",
)
.fetch_all(db.pool())
.await?;
println!("{:<36} {:<44} {:>8} {:>10} {:>6} {:>6}",
"UUID", "Name", "Duration", "Status", "Chunks", "Faces");
println!(
"{:<36} {:<44} {:>8} {:>10} {:>6} {:>6}",
"UUID", "Name", "Duration", "Status", "Chunks", "Faces"
);
println!("{}", "-".repeat(116));
for row in &rows {
@@ -318,10 +353,15 @@ async fn cmd_list(db: &PostgresDb) -> Result<()> {
name.clone()
};
println!("{:<36} {:<44} {:>8} {:>10} {:>6} {:>6}",
uuid, short_name, dur_str,
println!(
"{:<36} {:<44} {:>8} {:>10} {:>6} {:>6}",
uuid,
short_name,
dur_str,
status.as_deref().unwrap_or("?"),
chunks.unwrap_or(0), faces.unwrap_or(0));
chunks.unwrap_or(0),
faces.unwrap_or(0)
);
}
Ok(())
}
@@ -336,9 +376,23 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
"SELECT file_uuid, file_name, file_path, duration, fps, width, height FROM dev.videos WHERE file_uuid = $1"
).bind(uuid).fetch_optional(db.pool()).await?;
let (_, file_name, file_path, duration, fps, width, height): (
String, String, String, Option<f64>, Option<f64>, Option<i32>, Option<i32>
String,
String,
String,
Option<f64>,
Option<f64>,
Option<i32>,
Option<i32>,
) = match row {
Some(r) => (r.get(0), r.get(1), r.get(2), r.get(3), r.get(4), r.get(5), r.get(6)),
Some(r) => (
r.get(0),
r.get(1),
r.get(2),
r.get(3),
r.get(4),
r.get(5),
r.get(6),
),
None => anyhow::bail!("UUID {} not found", uuid),
};
@@ -360,7 +414,10 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
"momentry_version": env!("CARGO_PKG_VERSION"),
"momentry_build": env!("BUILD_GIT_HASH"),
});
fs::write(outdir.join("file_info.json"), serde_json::to_string_pretty(&info)?)?;
fs::write(
outdir.join("file_info.json"),
serde_json::to_string_pretty(&info)?,
)?;
// Export per-table .sql files (avoid single 4.7GB psql load)
let sql_dir = outdir.join("sql");
@@ -376,7 +433,13 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
let mut import_order = vec!["master.sql"];
fn write_table_sql(outdir: &Path, tbl: &str, col: &str, uuid: &str, psql_exec: &dyn Fn(&str) -> Result<String>) -> Result<()> {
fn write_table_sql(
outdir: &Path,
tbl: &str,
col: &str,
uuid: &str,
psql_exec: &dyn Fn(&str) -> Result<String>,
) -> Result<()> {
let safe_name = tbl.replace('.', "_");
let path = outdir.join(format!("{}.sql", safe_name));
let parts: Vec<&str> = tbl.split('.').collect();
@@ -419,8 +482,16 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
let data = psql_exec(&idents_query)?;
if !data.is_empty() {
let mut f = fs::File::create(&idents_path)?;
writeln!(f, "-- dev.identities WHERE file_uuid = '{}' OR global (tmdb/merged/user_defined)", uuid)?;
writeln!(f, "COPY dev.identities ({}) FROM STDIN WITH CSV HEADER;", cols)?;
writeln!(
f,
"-- dev.identities WHERE file_uuid = '{}' OR global (tmdb/merged/user_defined)",
uuid
)?;
writeln!(
f,
"COPY dev.identities ({}) FROM STDIN WITH CSV HEADER;",
cols
)?;
writeln!(f, "{}", data)?;
writeln!(f, "\\.")?;
}
@@ -440,7 +511,11 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
if !data.is_empty() {
let mut f = fs::File::create(&binds_path)?;
writeln!(f, "-- dev.identity_bindings (from face_detections JOIN)")?;
writeln!(f, "COPY dev.identity_bindings ({}) FROM STDIN WITH CSV HEADER;", cols)?;
writeln!(
f,
"COPY dev.identity_bindings ({}) FROM STDIN WITH CSV HEADER;",
cols
)?;
writeln!(f, "{}", data)?;
writeln!(f, "\\.")?;
}
@@ -469,7 +544,11 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
let sql_path = outdir.join("data.sql");
{
let mut f = fs::File::create(&sql_path)?;
writeln!(f, "-- Release package: {} — see sql/ for per-table files", uuid)?;
writeln!(
f,
"-- Release package: {} — see sql/ for per-table files",
uuid
)?;
writeln!(f, "BEGIN;")?;
writeln!(f, "\\i sql/dev_videos.sql")?;
writeln!(f, "\\i sql/dev_chunk.sql")?;
@@ -492,7 +571,11 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
let dest = outdir.join(vp.file_name().unwrap());
fs::copy(vp, &dest)?;
let vsize = fs::metadata(&dest)?.len();
println!(" {} ({} MB)", vp.file_name().unwrap().to_str().unwrap_or("?"), vsize / 1024 / 1024);
println!(
" {} ({} MB)",
vp.file_name().unwrap().to_str().unwrap_or("?"),
vsize / 1024 / 1024
);
}
}
@@ -541,11 +624,18 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
let vec0_src = "/Users/accusys/momentry_core_0.1/scripts/vec0.dylib";
if Path::new(vec0_src).exists() {
fs::copy(vec0_src, outdir.join("vec0.dylib"))?;
println!(" vec0.dylib ({} KB)", fs::metadata(outdir.join("vec0.dylib"))?.len() / 1024);
println!(
" vec0.dylib ({} KB)",
fs::metadata(outdir.join("vec0.dylib"))?.len() / 1024
);
}
// Create tar.gz
let tarball = Path::new(RELEASE_DIR).join(format!("{}_v{}.tar.gz", uuid, Utc::now().format("%Y%m%d_%H%M%S")));
let tarball = Path::new(RELEASE_DIR).join(format!(
"{}_v{}.tar.gz",
uuid,
Utc::now().format("%Y%m%d_%H%M%S")
));
let status = Command::new("tar")
.args(["-czf", tarball.to_str().unwrap(), "-C", RELEASE_DIR, uuid])
.status()?;
@@ -553,7 +643,11 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
anyhow::bail!("tar creation failed");
}
let tsize = fs::metadata(&tarball)?.len();
println!("\n Package: {} ({} MB)", tarball.display(), tsize / 1024 / 1024);
println!(
"\n Package: {} ({} MB)",
tarball.display(),
tsize / 1024 / 1024
);
// Sanity check: warn if any sql file is suspiciously large
println!(" Checking sql/ file sizes...");
@@ -564,33 +658,55 @@ async fn cmd_package(db: &PostgresDb, uuid: &str) -> Result<()> {
let sz = fs::metadata(&path)?.len() as f64 / 1024.0 / 1024.0;
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("?");
match name {
"dev_videos" | "master" if sz > 1.0 =>
println!(" ⚠️ {} is {} MB, expected < 1 MB", name, sz as u64),
"dev_chunk" if sz > 2.0 =>
println!(" ⚠️ {} is {} MB, expected < 2 MB for ~2.4K chunks", name, sz as u64),
"dev_identities" if sz > 1.0 =>
println!(" ⚠️ {} is {} MB, expected < 1 MB for ~428 identities", name, sz as u64),
"dev_identity_bindings" if sz > 5.0 =>
println!(" ⚠️ {} is {} MB, expected < 5 MB for ~7.6K bindings", name, sz as u64),
"dev_tkg_nodes" if sz > 10.0 =>
println!(" ⚠️ {} is {} MB, expected < 10 MB for ~6.4K nodes", name, sz as u64),
"dev_tkg_edges" if sz > 20.0 =>
println!(" ⚠️ {} is {} MB, expected < 20 MB for ~21K edges", name, sz as u64),
"dev_face_detections" if sz > 1000.0 =>
println!(" ⚠️ {} is {} MB, expected < 1000 MB for ~70K faces (512D emb)", name, sz as u64),
"dev_chunk_vectors" if sz > 200.0 =>
println!(" ⚠️ {} is {} MB, expected < 200 MB for ~2.4K chunks (768D emb)", name, sz as u64),
"dev_videos" | "master" if sz > 1.0 => {
println!(" ⚠️ {} is {} MB, expected < 1 MB", name, sz as u64)
}
"dev_chunk" if sz > 2.0 => println!(
" ⚠️ {} is {} MB, expected < 2 MB for ~2.4K chunks",
name, sz as u64
),
"dev_identities" if sz > 1.0 => println!(
" ⚠️ {} is {} MB, expected < 1 MB for ~428 identities",
name, sz as u64
),
"dev_identity_bindings" if sz > 5.0 => println!(
" ⚠️ {} is {} MB, expected < 5 MB for ~7.6K bindings",
name, sz as u64
),
"dev_tkg_nodes" if sz > 10.0 => println!(
" ⚠️ {} is {} MB, expected < 10 MB for ~6.4K nodes",
name, sz as u64
),
"dev_tkg_edges" if sz > 20.0 => println!(
" ⚠️ {} is {} MB, expected < 20 MB for ~21K edges",
name, sz as u64
),
"dev_face_detections" if sz > 1000.0 => println!(
" ⚠️ {} is {} MB, expected < 1000 MB for ~70K faces (512D emb)",
name, sz as u64
),
"dev_chunk_vectors" if sz > 200.0 => println!(
" ⚠️ {} is {} MB, expected < 200 MB for ~2.4K chunks (768D emb)",
name, sz as u64
),
_ => {}
}
if sz > 2000.0 {
println!(" ⚠️ {} is {:.0} MB — unusually large, verify query", name, sz);
println!(
" ⚠️ {} is {:.0} MB — unusually large, verify query",
name, sz
);
}
}
}
Ok(())
}
fn cmd_visualize_offline(sqlite_path: &str, output: Option<&str>, identity: Option<i64>) -> Result<()> {
fn cmd_visualize_offline(
sqlite_path: &str,
output: Option<&str>,
identity: Option<i64>,
) -> Result<()> {
let outpath = match output {
Some(p) => p.to_string(),
None => sqlite_path.replace(".sqlite", "_report.html"),
@@ -606,7 +722,10 @@ fn cmd_visualize_offline(sqlite_path: &str, output: Option<&str>, identity: Opti
.output()
.context("Offline report script failed")?;
if !output.status.success() {
anyhow::bail!("Offline report: {}", String::from_utf8_lossy(&output.stderr));
anyhow::bail!(
"Offline report: {}",
String::from_utf8_lossy(&output.stderr)
);
}
println!("{}", String::from_utf8_lossy(&output.stdout));
println!("\n Open: {}", outpath);
@@ -624,7 +743,10 @@ fn cmd_visualize(uuid: &str, typ: &str, output: Option<&str>, identity: Option<i
match typ {
"heatmap" | "density" => generate_face_heatmap(uuid, &outpath, identity)?,
"timeline" => generate_face_timeline(uuid, &outpath, identity)?,
_ => anyhow::bail!("Unknown visualization type: {}. Try: heatmap, density, timeline", typ),
_ => anyhow::bail!(
"Unknown visualization type: {}. Try: heatmap, density, timeline",
typ
),
}
Ok(())
}
@@ -698,16 +820,28 @@ fn cmd_stats() -> Result<()> {
for line in listing.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.ends_with('/') { continue; }
if trimmed.is_empty() || trimmed.ends_with('/') {
continue;
}
// tar -tvzf format: perms link owner group size date_month date_day time path...
// Fields are space-separated; size is 5th field, path starts at 8th field
let parts: Vec<&str> = trimmed.split_whitespace().collect();
if parts.len() < 8 { continue; }
if parts.len() < 8 {
continue;
}
let fsize = parts[4].parse::<u64>().unwrap_or(0);
let fpath = parts[8..].join(" ");
let fname = Path::new(&fpath).file_name().unwrap_or_default().to_str().unwrap_or("?");
let ext = Path::new(&fpath).extension().unwrap_or_default().to_str().unwrap_or("");
let fname = Path::new(&fpath)
.file_name()
.unwrap_or_default()
.to_str()
.unwrap_or("?");
let ext = Path::new(&fpath)
.extension()
.unwrap_or_default()
.to_str()
.unwrap_or("");
match ext {
"sql" => {
@@ -732,10 +866,26 @@ fn cmd_stats() -> Result<()> {
}
println!(" ─────────────────────────────");
println!(" SQL: {} files, {:.0} MB", sql_count, total_sql as f64 / 1048576.0);
println!(" Video: {} files, {:.0} MB", video_count, total_video as f64 / 1048576.0);
println!(" JSON: {} files, {:.0} MB", json_count, total_json as f64 / 1048576.0);
println!(" Total: {:.0} MB (compressed: {:.0} MB)", (total_sql + total_video + total_json) as f64 / 1048576.0, pkg_size as f64 / 1048576.0);
println!(
" SQL: {} files, {:.0} MB",
sql_count,
total_sql as f64 / 1048576.0
);
println!(
" Video: {} files, {:.0} MB",
video_count,
total_video as f64 / 1048576.0
);
println!(
" JSON: {} files, {:.0} MB",
json_count,
total_json as f64 / 1048576.0
);
println!(
" Total: {:.0} MB (compressed: {:.0} MB)",
(total_sql + total_video + total_json) as f64 / 1048576.0,
pkg_size as f64 / 1048576.0
);
println!();
}
@@ -758,8 +908,17 @@ async fn main() -> Result<()> {
Commands::List => cmd_list(&db).await?,
Commands::Package { uuid } => cmd_package(&db, &uuid).await?,
Commands::Stats => cmd_stats()?,
Commands::Visualize { uuid, typ, output, identity } => cmd_visualize(&uuid, &typ, output.as_deref(), identity)?,
Commands::VisualizeOffline { sqlite_path, output, identity } => cmd_visualize_offline(&sqlite_path, output.as_deref(), identity)?,
Commands::Visualize {
uuid,
typ,
output,
identity,
} => cmd_visualize(&uuid, &typ, output.as_deref(), identity)?,
Commands::VisualizeOffline {
sqlite_path,
output,
identity,
} => cmd_visualize_offline(&sqlite_path, output.as_deref(), identity)?,
}
Ok(())
}

408
src/bin/service.rs
View File

@@ -16,7 +16,10 @@ const LOG_DIR: &str = "/Users/accusys/service_logs";
const LAUNCH_DIR: &str = "/Users/accusys/Library/LaunchAgents";
#[derive(Parser)]
#[command(name = "service", about = "Service Lifecycle Manager — source → build → install → config → launch → env")]
#[command(
name = "service",
about = "Service Lifecycle Manager — source → build → install → config → launch → env"
)]
struct Cli {
#[command(subcommand)]
command: Commands,
@@ -111,22 +114,54 @@ fn cmd_source_list() -> Result<()> {
("pyenv", "pyenv/", "git repo"),
("cmake", "cmake-4.2.0-macos-universal.tar.gz", "binary"),
("llama.cpp", "llama.cpp/", "git repo"),
("libreoffice (src)", "libreoffice-26.2.3.2.tar.xz", "source tarball"),
("libreoffice (dmg)", "LibreOffice_26.2.3_MacOS_aarch64.dmg", "binary (TDF)"),
("mermaid-cli", "mermaid-js-mermaid-cli-11.14.0.tgz", "npm package"),
(
"libreoffice (src)",
"libreoffice-26.2.3.2.tar.xz",
"source tarball",
),
(
"libreoffice (dmg)",
"LibreOffice_26.2.3_MacOS_aarch64.dmg",
"binary (TDF)",
),
(
"mermaid-cli",
"mermaid-js-mermaid-cli-11.14.0.tgz",
"npm package",
),
("librsvg", "librsvg/", "Rust source"),
("GroundingDINO", "GroundingDINO/", "git repo (IDEA-Research)"),
(
"GroundingDINO",
"GroundingDINO/",
"git repo (IDEA-Research)",
),
("PaliGemma", "paligemma/", "HuggingFace reference"),
("Odoo 19 CE", "odoo/", "git repo (LGPL-3.0)"),
("ERPNext v15", "erpnext/", "git repo (GPL-3.0)"),
("Frappe Framework", "frappe/", "git repo (MIT)"),
("Gitea v1.25", "gitea/", "git repo (MIT, Go)"),
("Go v1.26", "go/", "git repo (BSD)"),
("Rust/Cargo", "rustc-1.92.0-src.tar.xz", "source tarball (Apache 2.0 / MIT)"),
("rustup", "rustup-1.28.1.tar.gz", "source tarball (Apache 2.0)"),
("Swift v6.3", "swift-6.3.1-RELEASE.tar.gz", "source tarball (Apache 2.0)"),
(
"Rust/Cargo",
"rustc-1.92.0-src.tar.xz",
"source tarball (Apache 2.0 / MIT)",
),
(
"rustup",
"rustup-1.28.1.tar.gz",
"source tarball (Apache 2.0)",
),
(
"Swift v6.3",
"swift-6.3.1-RELEASE.tar.gz",
"source tarball (Apache 2.0)",
),
("yt-dlp", "yt-dlp/", "git repo (Unlicense)"),
("SQLite", "sqlite-amalgamation-3490100.zip", "amalgamation (Public Domain)"),
(
"SQLite",
"sqlite-amalgamation-3490100.zip",
"amalgamation (Public Domain)",
),
("sqlite-vec", "sqlite-vec/", "git repo (MIT)"),
];
@@ -164,7 +199,11 @@ fn cmd_source_verify() -> Result<()> {
("cmake", "cmake-4.2.0-macos-universal.tar.gz", false),
("llama.cpp", "llama.cpp/", true),
("libreoffice (src)", "libreoffice-26.2.3.2.tar.xz", false),
("libreoffice (dmg)", "LibreOffice_26.2.3_MacOS_aarch64.dmg", false),
(
"libreoffice (dmg)",
"LibreOffice_26.2.3_MacOS_aarch64.dmg",
false,
),
("mermaid-cli", "mermaid-js-mermaid-cli-11.14.0.tgz", false),
("librsvg", "librsvg/", true),
("GroundingDINO", "GroundingDINO/", true),
@@ -186,7 +225,11 @@ fn cmd_source_verify() -> Result<()> {
let mut missing = 0;
for (name, path, is_dir) in &checks {
let full = src_dir.join(path);
let exists = if *is_dir { full.is_dir() } else { full.is_file() };
let exists = if *is_dir {
full.is_dir()
} else {
full.is_file()
};
if exists {
println!(" ✅ {}", name);
ok += 1;
@@ -202,7 +245,10 @@ fn cmd_source_verify() -> Result<()> {
// ---- Build ----
fn cmd_build(service: &str) -> Result<()> {
let install_sh = Path::new(SERVICE_SRC).parent().unwrap().join("install_services.sh");
let install_sh = Path::new(SERVICE_SRC)
.parent()
.unwrap()
.join("install_services.sh");
if service == "all" {
// Run the full install script
@@ -224,8 +270,14 @@ fn cmd_build(service: &str) -> Result<()> {
"ffmpeg" => {
println!("Building ffmpeg (requires x264 + freetype)...");
// Simplified: run the install script which handles incremental builds
let status = Command::new("bash").arg(&install_sh).env("PREFIX", PREFIX).env("SRC_DIR", SERVICE_SRC).status()?;
if !status.success() { anyhow::bail!("Build failed"); }
let status = Command::new("bash")
.arg(&install_sh)
.env("PREFIX", PREFIX)
.env("SRC_DIR", SERVICE_SRC)
.status()?;
if !status.success() {
anyhow::bail!("Build failed");
}
}
"redis" => {
let src = format!("{}/redis-7.4.3.tar.gz", SERVICE_SRC);
@@ -236,37 +288,67 @@ fn cmd_build(service: &str) -> Result<()> {
run_build("postgresql", &src, &format!("cd /tmp && tar xzf {} && cd postgresql-18.3 && ./configure --prefix={}/pgsql/18.3 && make -j$(sysctl -n hw.ncpu) && make install", src, PREFIX))?;
}
"llama" => {
println!("Building llama.cpp from {}...", format!("{}/llama.cpp", SERVICE_SRC));
println!(
"Building llama.cpp from {}...",
format!("{}/llama.cpp", SERVICE_SRC)
);
let status = Command::new("cmake")
.args(["-B", "build", "-DCMAKE_INSTALL_PREFIX=/tmp/llama_install"])
.current_dir(format!("{}/llama.cpp", SERVICE_SRC))
.status()?;
if !status.success() { anyhow::bail!("cmake failed"); }
let status = Command::new("cmake").args(["--build", "build", "--config", "Release", "-j"]).current_dir(format!("{}/llama.cpp", SERVICE_SRC)).status()?;
if !status.success() { anyhow::bail!("build failed"); }
if !status.success() {
anyhow::bail!("cmake failed");
}
let status = Command::new("cmake")
.args(["--build", "build", "--config", "Release", "-j"])
.current_dir(format!("{}/llama.cpp", SERVICE_SRC))
.status()?;
if !status.success() {
anyhow::bail!("build failed");
}
}
"libreoffice" => {
let dmg = format!("{}/LibreOffice_26.2.3_MacOS_aarch64.dmg", SERVICE_SRC);
let mount = "/tmp/lo_mount";
println!("Extracting LibreOffice from DMG...");
// Mount
let status = Command::new("hdiutil").args(["attach", &dmg, "-nobrowse", "-quiet", "-mountpoint", mount]).status()?;
if !status.success() { anyhow::bail!("DMG mount failed"); }
let status = Command::new("hdiutil")
.args(["attach", &dmg, "-nobrowse", "-quiet", "-mountpoint", mount])
.status()?;
if !status.success() {
anyhow::bail!("DMG mount failed");
}
// Copy app
let lo_dir = format!("{}/libreoffice", PREFIX);
let _ = std::fs::remove_dir_all(format!("{}/LibreOffice.app", lo_dir));
std::fs::create_dir_all(&lo_dir)?;
let status = Command::new("cp").args(["-R", &format!("{}/LibreOffice.app", mount), &format!("{}/LibreOffice.app", lo_dir)]).status()?;
if !status.success() { anyhow::bail!("Copy failed"); }
let status = Command::new("cp")
.args([
"-R",
&format!("{}/LibreOffice.app", mount),
&format!("{}/LibreOffice.app", lo_dir),
])
.status()?;
if !status.success() {
anyhow::bail!("Copy failed");
}
// Create symlink
std::fs::create_dir_all(format!("{}/bin", lo_dir))?;
let _ = std::fs::remove_file(format!("{}/bin/soffice", lo_dir));
std::os::unix::fs::symlink("../LibreOffice.app/Contents/MacOS/soffice", format!("{}/bin/soffice", lo_dir))?;
std::os::unix::fs::symlink(
"../LibreOffice.app/Contents/MacOS/soffice",
format!("{}/bin/soffice", lo_dir),
)?;
// Unmount
let _ = Command::new("hdiutil").args(["detach", mount, "-quiet"]).status();
let _ = Command::new("hdiutil")
.args(["detach", mount, "-quiet"])
.status();
println!(" libreoffice installed to {}/bin/soffice", lo_dir);
}
_ => anyhow::bail!("Unknown service: {}. Try: all, ffmpeg, redis, postgres, llama, libreoffice, python", service),
_ => anyhow::bail!(
"Unknown service: {}. Try: all, ffmpeg, redis, postgres, llama, libreoffice, python",
service
),
}
Ok(())
}
@@ -274,7 +356,9 @@ fn cmd_build(service: &str) -> Result<()> {
fn run_build(name: &str, src: &str, cmd: &str) -> Result<()> {
println!("Building {} from {}...", name, src);
let status = Command::new("bash").arg("-c").arg(cmd).status()?;
if !status.success() { anyhow::bail!("{} build failed", name); }
if !status.success() {
anyhow::bail!("{} build failed", name);
}
println!(" {} build complete", name);
Ok(())
}
@@ -292,7 +376,10 @@ fn cmd_install(service: &str) -> Result<()> {
let rsvg_src = format!("{}/librsvg/bin/rsvg-convert", PREFIX);
let gitea_src = format!("{}/gitea/bin/gitea", PREFIX);
let go_src = format!("{}/go/bin/go", PREFIX);
let rustc_src = format!("{}/.rustup/toolchains/stable-aarch64-apple-darwin/bin/rustc", PREFIX);
let rustc_src = format!(
"{}/.rustup/toolchains/stable-aarch64-apple-darwin/bin/rustc",
PREFIX
);
let swift_src = "/usr/bin/swift".to_string();
let ytdlp_src = "/opt/homebrew/bin/yt-dlp".to_string();
@@ -313,7 +400,9 @@ fn cmd_install(service: &str) -> Result<()> {
];
for (name, src) in &installs {
if service != "all" && service != *name { continue; }
if service != "all" && service != *name {
continue;
}
if Path::new(src).exists() {
println!(" ✅ {} installed: {}", name, src);
} else {
@@ -370,12 +459,18 @@ fn cmd_config(service: &str) -> Result<()> {
println!("MOMENTRY_LLM_SUMMARY_URL=http://localhost:8082/v1/chat/completions");
println!("MOMENTRY_OUTPUT_DIR={}/momentry/output_dev", PREFIX);
println!("MOMENTRY_SCRIPTS_DIR={}/momentry_core_0.1/scripts", PREFIX);
println!("MOMENTRY_PYTHON_PATH={}/.pyenv/versions/3.11.15/bin/python3.11", PREFIX);
println!(
"MOMENTRY_PYTHON_PATH={}/.pyenv/versions/3.11.15/bin/python3.11",
PREFIX
);
}
if service == "all" || service == "embedding" {
println!("\n--- Embedding Server config ---");
println!("# Start: {} embeddinggemma_server.py --port 11436", format!("{}/momentry_core_0.1/scripts", PREFIX));
println!(
"# Start: {} embeddinggemma_server.py --port 11436",
format!("{}/momentry_core_0.1/scripts", PREFIX)
);
println!("MODEL=google/embeddinggemma-300m");
println!("PORT=11436");
println!("DEVICE=mps");
@@ -393,25 +488,58 @@ fn cmd_launch_generate() -> Result<()> {
let pg_args = format!("-D {}/pgsql/18.3/data", PREFIX);
let redis_bin = format!("{}/redis/bin/redis-server", PREFIX);
let redis_args = format!("{}/redis/redis.conf", PREFIX);
let qdrant_bin = format!("{}/momentry_core_0.1/services/qdrant/target/release/qdrant", PREFIX);
let qdrant_bin = format!(
"{}/momentry_core_0.1/services/qdrant/target/release/qdrant",
PREFIX
);
let embed_bin = format!("{}/.pyenv/versions/3.11.15/bin/python3.11", PREFIX);
let embed_args = format!("{}/momentry_core_0.1/scripts/embeddinggemma_server.py --port 11436", PREFIX);
let embed_args = format!(
"{}/momentry_core_0.1/scripts/embeddinggemma_server.py --port 11436",
PREFIX
);
let llama_bin = format!("{}/llama/bin/llama-server", PREFIX);
let llama_args = format!("-m {}/models/google_gemma-4-26B-A4B-it-Q5_K_M.gguf --port 8082 -ngl 99 -c 16384", PREFIX);
let play_bin = format!("{}/momentry_core_0.1/target/debug/momentry_playground", PREFIX);
let llama_args = format!(
"-m {}/models/google_gemma-4-26B-A4B-it-Q5_K_M.gguf --port 8082 -ngl 99 -c 16384",
PREFIX
);
let play_bin = format!(
"{}/momentry_core_0.1/target/debug/momentry_playground",
PREFIX
);
let services: Vec<(&str, &str, &str, &str)> = vec![
("com.momentry.postgres", &pg_bin, &pg_args, "PostgreSQL"),
("com.momentry.redis", &redis_bin, &redis_args, "Redis"),
("com.momentry.qdrant", &qdrant_bin, "", "Qdrant"),
("com.momentry.embedding", &embed_bin, &embed_args, "EmbeddingGemma"),
("com.momentry.llama", &llama_bin, &llama_args, "LLM (llama.cpp)"),
("com.momentry.playground", &play_bin, "server --port 3003", "Momentry Playground"),
("com.momentry.worker", &play_bin, "worker --max-concurrent 2 --poll-interval 5", "Momentry Worker"),
("com.momentry.postgres", &pg_bin, &pg_args, "PostgreSQL"),
("com.momentry.redis", &redis_bin, &redis_args, "Redis"),
("com.momentry.qdrant", &qdrant_bin, "", "Qdrant"),
(
"com.momentry.embedding",
&embed_bin,
&embed_args,
"EmbeddingGemma",
),
(
"com.momentry.llama",
&llama_bin,
&llama_args,
"LLM (llama.cpp)",
),
(
"com.momentry.playground",
&play_bin,
"server --port 3003",
"Momentry Playground",
),
(
"com.momentry.worker",
&play_bin,
"worker --max-concurrent 2 --poll-interval 5",
"Momentry Worker",
),
];
for (label, bin, args, _desc) in &services {
let plist = format!(r#"<?xml version="1.0" encoding="UTF-8"?>
let plist = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
@@ -451,7 +579,11 @@ fn cmd_launch_generate() -> Result<()> {
fs::write(&plist_path, plist)?;
println!(" 📝 {} → {:?}", label, plist_path.file_name().unwrap());
}
println!("\n Generated {} plist files in {}", services.len(), LAUNCH_DIR);
println!(
"\n Generated {} plist files in {}",
services.len(),
LAUNCH_DIR
);
Ok(())
}
@@ -461,7 +593,9 @@ fn cmd_launch_load() -> Result<()> {
let path = entry.path();
if path.extension().map_or(false, |e| e == "plist") {
let name = path.file_stem().unwrap().to_str().unwrap_or("?");
let status = Command::new("launchctl").args(["load", "-w", path.to_str().unwrap()]).status();
let status = Command::new("launchctl")
.args(["load", "-w", path.to_str().unwrap()])
.status();
match status {
Ok(s) if s.success() => println!(" ✅ loaded: {}", name),
Ok(_) => println!(" ⚠️ load failed: {}", name),
@@ -478,7 +612,9 @@ fn cmd_launch_unload() -> Result<()> {
let path = entry.path();
if path.extension().map_or(false, |e| e == "plist") {
let name = path.file_stem().unwrap().to_str().unwrap_or("?");
let status = Command::new("launchctl").args(["unload", path.to_str().unwrap()]).status();
let status = Command::new("launchctl")
.args(["unload", path.to_str().unwrap()])
.status();
match status {
Ok(s) if s.success() => println!(" ✅ unloaded: {}", name),
Ok(_) => println!(" ⚠️ unload failed: {}", name),
@@ -504,7 +640,11 @@ fn cmd_launch_status() -> Result<()> {
Ok(o) if o.status.success() => {
let stdout = String::from_utf8_lossy(&o.stdout);
if stdout.contains("PID") || stdout.lines().count() > 1 {
let pid = stdout.lines().nth(1).and_then(|l| l.split_whitespace().next()).unwrap_or("-");
let pid = stdout
.lines()
.nth(1)
.and_then(|l| l.split_whitespace().next())
.unwrap_or("-");
println!(" 🟢 {} (PID: {})", label, pid);
} else {
println!(" ⚪ {} (not running)", label);
@@ -519,7 +659,8 @@ fn cmd_launch_status() -> Result<()> {
// ---- Env ----
fn cmd_env(output: &Option<String>) -> Result<()> {
let env_content = format!(r#"# Momentry Core — Environment Configuration
let env_content = format!(
r#"# Momentry Core — Environment Configuration
# Generated: {}
# Service: {} env
@@ -601,8 +742,14 @@ fn cmd_test() -> Result<()> {
let rsvg_bin = format!("{}/librsvg/bin/rsvg-convert", PREFIX);
let gitea_bin = format!("{}/gitea/bin/gitea", PREFIX);
let go_bin = format!("{}/go/bin/go", PREFIX);
let rustc_bin = format!("{}/.rustup/toolchains/stable-aarch64-apple-darwin/bin/rustc", PREFIX);
let cargo_bin = format!("{}/.rustup/toolchains/stable-aarch64-apple-darwin/bin/cargo", PREFIX);
let rustc_bin = format!(
"{}/.rustup/toolchains/stable-aarch64-apple-darwin/bin/rustc",
PREFIX
);
let cargo_bin = format!(
"{}/.rustup/toolchains/stable-aarch64-apple-darwin/bin/cargo",
PREFIX
);
let swift_bin = "/usr/bin/swift".to_string();
let ytdlp_bin = "/opt/homebrew/bin/yt-dlp".to_string();
@@ -641,7 +788,11 @@ fn cmd_test() -> Result<()> {
let output = Command::new(bin).args(args).output();
match output {
Ok(o) if o.status.success() => {
let ver = String::from_utf8_lossy(&o.stdout).lines().next().unwrap_or("?").to_string();
let ver = String::from_utf8_lossy(&o.stdout)
.lines()
.next()
.unwrap_or("?")
.to_string();
println!("✅ {}", ver.chars().take(70).collect::<String>());
pass += 1;
}
@@ -666,14 +817,87 @@ fn cmd_test() -> Result<()> {
// Functional tests
println!("\n--- Functional Tests ---");
// Create test docx for libreoffice test
let _ = std::fs::write("/tmp/svc_test_func.docx", "Service test document for LibreOffice conversion");
let _ = std::fs::write(
"/tmp/svc_test_func.docx",
"Service test document for LibreOffice conversion",
);
let func_tests = [
("ffprobe probe", "ffprobe", vec!["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "/Users/accusys/momentry/var/sftpgo/data/demo/Charade_YouTube_24fps.mp4"]),
("ffmpeg audio extract", "ffmpeg", vec!["-y", "-v", "quiet", "-i", "/Users/accusys/momentry/var/sftpgo/data/demo/Charade_YouTube_24fps.mp4", "-t", "2", "-ar", "16000", "-ac", "1", "/tmp/svc_test_audio.wav"]),
("ffmpeg frame extract", "ffmpeg", vec!["-y", "-v", "quiet", "-i", "/Users/accusys/momentry/var/sftpgo/data/demo/Charade_YouTube_24fps.mp4", "-ss", "100", "-vframes", "1", "/tmp/svc_test_frame.jpg"]),
("libreoffice doc→txt", "libreoffice", vec!["--headless", "--convert-to", "txt", "/tmp/svc_test_func.docx", "--outdir", "/tmp/"]),
("rsvg-convert svg→png", "rsvg-convert", vec!["-o", "/tmp/svc_test_rsvg.png", "/tmp/test_rsvg.svg"]),
("mmdc mermaid→png", "mermaid-cli", vec!["-i", "/tmp/test_mermaid.mmd", "-o", "/tmp/svc_test_mmd.png", "-w", "200"]),
(
"ffprobe probe",
"ffprobe",
vec![
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"csv=p=0",
"/Users/accusys/momentry/var/sftpgo/data/demo/Charade_YouTube_24fps.mp4",
],
),
(
"ffmpeg audio extract",
"ffmpeg",
vec![
"-y",
"-v",
"quiet",
"-i",
"/Users/accusys/momentry/var/sftpgo/data/demo/Charade_YouTube_24fps.mp4",
"-t",
"2",
"-ar",
"16000",
"-ac",
"1",
"/tmp/svc_test_audio.wav",
],
),
(
"ffmpeg frame extract",
"ffmpeg",
vec![
"-y",
"-v",
"quiet",
"-i",
"/Users/accusys/momentry/var/sftpgo/data/demo/Charade_YouTube_24fps.mp4",
"-ss",
"100",
"-vframes",
"1",
"/tmp/svc_test_frame.jpg",
],
),
(
"libreoffice doc→txt",
"libreoffice",
vec![
"--headless",
"--convert-to",
"txt",
"/tmp/svc_test_func.docx",
"--outdir",
"/tmp/",
],
),
(
"rsvg-convert svg→png",
"rsvg-convert",
vec!["-o", "/tmp/svc_test_rsvg.png", "/tmp/test_rsvg.svg"],
),
(
"mmdc mermaid→png",
"mermaid-cli",
vec![
"-i",
"/tmp/test_mermaid.mmd",
"-o",
"/tmp/svc_test_mmd.png",
"-w",
"200",
],
),
];
for (desc, bin_name, args) in &func_tests {
@@ -689,8 +913,14 @@ fn cmd_test() -> Result<()> {
};
let output = Command::new(bin).args(args).output();
match output {
Ok(o) if o.status.success() => { println!("✅"); pass += 1; }
_ => { println!("❌"); fail += 1; }
Ok(o) if o.status.success() => {
println!("✅");
pass += 1;
}
_ => {
println!("❌");
fail += 1;
}
}
}
@@ -706,7 +936,10 @@ fn cmd_test() -> Result<()> {
fn cmd_report() -> Result<()> {
println!("=== Momentry Service Report ===");
println!("Generated: {}", chrono::Local::now().format("%Y-%m-%d %H:%M:%S"));
println!(
"Generated: {}",
chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
);
println!();
// 1. Source status
@@ -730,13 +963,25 @@ fn cmd_report() -> Result<()> {
println!("\n## 2. Binaries");
let binaries = [
("cmake", &format!("{}/bin/cmake", PREFIX)),
("python3.11", &format!("{}/.pyenv/versions/3.11.15/bin/python3.11", PREFIX)),
(
"python3.11",
&format!("{}/.pyenv/versions/3.11.15/bin/python3.11", PREFIX),
),
("ffmpeg", &format!("{}/ffmpeg_build/bin/ffmpeg", PREFIX)),
("ffprobe", &format!("{}/ffmpeg_build/bin/ffprobe", PREFIX)),
("redis-server", &format!("{}/redis/bin/redis-server", PREFIX)),
(
"redis-server",
&format!("{}/redis/bin/redis-server", PREFIX),
),
("postgres", &format!("{}/pgsql/18.3/bin/postgres", PREFIX)),
("llama-server", &format!("{}/llama/bin/llama-server", PREFIX)),
("libreoffice", &format!("{}/libreoffice/bin/soffice", PREFIX)),
(
"llama-server",
&format!("{}/llama/bin/llama-server", PREFIX),
),
(
"libreoffice",
&format!("{}/libreoffice/bin/soffice", PREFIX),
),
];
for (name, path) in &binaries {
let status = if Path::new(path).exists() {
@@ -772,9 +1017,18 @@ fn cmd_report() -> Result<()> {
// 4. Ports
println!("\n## 4. Port Status");
let ports = [(3003, "Playground"), (5432, "PostgreSQL"), (6379, "Redis"), (6333, "Qdrant"), (8082, "LLM"), (11436, "Embedding")];
let ports = [
(3003, "Playground"),
(5432, "PostgreSQL"),
(6379, "Redis"),
(6333, "Qdrant"),
(8082, "LLM"),
(11436, "Embedding"),
];
for (port, name) in &ports {
let output = Command::new("lsof").args(["-i", &format!(":{}", port)]).output();
let output = Command::new("lsof")
.args(["-i", &format!(":{}", port)])
.output();
match output {
Ok(o) if o.status.success() => println!(" 🟢 :{} ({})", port, name),
_ => println!(" ⚪ :{} ({})", port, name),
@@ -797,14 +1051,21 @@ fn cmd_report() -> Result<()> {
}
fn format_bytes(bytes: u64) -> String {
if bytes > 1024 * 1024 * 1024 { format!("{:.1}GB", bytes as f64 / 1_073_741_824.0) }
else if bytes > 1024 * 1024 { format!("{:.0}MB", bytes as f64 / 1_048_576.0) }
else if bytes > 1024 { format!("{:.0}KB", bytes as f64 / 1024.0) }
else { format!("{}B", bytes) }
if bytes > 1024 * 1024 * 1024 {
format!("{:.1}GB", bytes as f64 / 1_073_741_824.0)
} else if bytes > 1024 * 1024 {
format!("{:.0}MB", bytes as f64 / 1_048_576.0)
} else if bytes > 1024 {
format!("{:.0}KB", bytes as f64 / 1024.0)
} else {
format!("{}B", bytes)
}
}
fn format_dir_size(path: &Path) -> String {
let output = Command::new("du").args(["-sh", path.to_str().unwrap()]).output();
let output = Command::new("du")
.args(["-sh", path.to_str().unwrap()])
.output();
match output {
Ok(o) if o.status.success() => {
let s = String::from_utf8_lossy(&o.stdout);
@@ -824,7 +1085,10 @@ async fn main() -> Result<()> {
SourceAction::List => cmd_source_list()?,
SourceAction::Verify => cmd_source_verify()?,
SourceAction::Download { name } => {
println!("Downloading: {} (use install_services.sh for full download)", name);
println!(
"Downloading: {} (use install_services.sh for full download)",
name
);
println!("Source URLs:");
println!(" ffmpeg: https://ffmpeg.org/releases/ffmpeg-7.1.1.tar.xz");
println!(" redis: https://download.redis.io/releases/redis-7.4.3.tar.gz");