feat: Phase 2.6 edges migration to Qdrant (TKG-only architecture)
Phase 2.6.1: co_occurrence_edges migration - build_co_occurrence_edges_from_qdrant() - Qdrant embeddings → frame grouping → YOLO objects - Result: 6679 edges (vs 6701 PostgreSQL) Phase 2.6.2: face_face_edges migration - build_face_face_edges_from_qdrant() - Qdrant embeddings → frame grouping → face pairs - mutual_gaze detection preserved - Result: 6 edges (exact match) Phase 2.6.3: speaker_face_edges migration - build_speaker_face_edges_from_qdrant() - Qdrant embeddings → trace_id frame ranges - SPEAKS_AS edge creation Architecture: - All edges use Qdrant payload (no face_detections queries) - PostgreSQL fallback for empty Qdrant - Estimated 3.6x performance improvement Testing: - Playground (3003): ✓ All Phase 2.6 logs verified - Edge counts: ✓ Close match with PostgreSQL - Fallback: ✓ Working Docs: - docs_v1.0/DESIGN/TKG_PHASE2_6_EDGES_MIGRATION.md - docs_v1.0/M4_workspace/2026-06-21_phase2_6_test.md
This commit is contained in:
Accusys
471
v1.1/scripts/dashboard_v1.11.py
Normal file
471
v1.1/scripts/dashboard_v1.11.py
Normal file
@@ -0,0 +1,471 @@
|
||||
#!/opt/homebrew/bin/python3.11
|
||||
"""
|
||||
Momentry Dashboard v2 — Direct DB/Qdrant/Redis queries, no subprocess blocking
|
||||
"""
|
||||
|
||||
import json, os, platform, time
|
||||
from pathlib import Path
|
||||
from flask import Flask, jsonify, render_template_string
|
||||
import psycopg2
|
||||
import urllib.request
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
HOSTNAME = platform.node()
|
||||
IS_M5 = "MacBook" in HOSTNAME
|
||||
SYSTEM_ROLE = "M5 (MacBook Pro)" if IS_M5 else "M4 (Mac Mini)"
|
||||
SYSTEM_COLOR = "#58a6ff" if IS_M5 else "#f0883e"
|
||||
|
||||
DB_URL = "postgresql://accusys@localhost:5432/momentry?host=/tmp"
|
||||
QDRANT_URL = "http://localhost:6333"
|
||||
LLM_URL = "http://localhost:8082/v1/chat/completions"
|
||||
EMBED_URL = "http://localhost:11436/v1/embeddings"
|
||||
|
||||
COLLECTIONS = [
|
||||
"momentry_dev_v1", "momentry_dev_stories", "momentry_dev_voice",
|
||||
"momentry_dev_faces", "sentence_story", "sentence_summary",
|
||||
"momentry_dev_rule1_v2",
|
||||
]
|
||||
|
||||
UUID = "aeed71342a899fe4b4c57b7d41bcb692"
|
||||
|
||||
def db_query(sql, params=None):
|
||||
conn = psycopg2.connect(DB_URL)
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params or ())
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
def qdrant_get(path):
|
||||
try:
|
||||
resp = urllib.request.urlopen(f"{QDRANT_URL}{path}", timeout=5)
|
||||
return json.loads(resp.read())
|
||||
except:
|
||||
return None
|
||||
|
||||
def qdrant_count(col):
|
||||
r = qdrant_get(f"/collections/{col}")
|
||||
if r:
|
||||
return r.get("result", {}).get("points_count", 0)
|
||||
return -1
|
||||
|
||||
def qdrant_dim(col):
|
||||
r = qdrant_get(f"/collections/{col}")
|
||||
if r:
|
||||
cfg = r.get("result", {}).get("config", {}).get("params", {}).get("vectors", {})
|
||||
return cfg.get("size", "?")
|
||||
return "?"
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
return render_template_string(TEMPLATE, SYSTEM_ROLE=SYSTEM_ROLE)
|
||||
|
||||
@app.route("/api/all")
|
||||
def api_all():
|
||||
return jsonify({
|
||||
"system": {"hostname": HOSTNAME, "role": SYSTEM_ROLE, "is_m5": IS_M5},
|
||||
"status": get_status(),
|
||||
"qdrant": get_qdrant_info(),
|
||||
"db": get_db_info(),
|
||||
"processes": get_processes(),
|
||||
})
|
||||
|
||||
@app.route("/api/status")
|
||||
def api_status():
|
||||
return jsonify(get_status())
|
||||
|
||||
@app.route("/api/qdrant")
|
||||
def api_qdrant():
|
||||
return jsonify(get_qdrant_info())
|
||||
|
||||
@app.route("/api/db")
|
||||
def api_db():
|
||||
return jsonify(get_db_info())
|
||||
|
||||
@app.route("/api/processes")
|
||||
def api_processes():
|
||||
return jsonify(get_processes())
|
||||
|
||||
def get_status():
|
||||
"""Pipeline checklist — direct DB queries"""
|
||||
t0 = time.time()
|
||||
stages = []
|
||||
|
||||
# 1. ASR file
|
||||
asr_path = f"/Users/accusys/momentry/output_dev/{UUID}.asr.json"
|
||||
asr_segs = 0
|
||||
try:
|
||||
if os.path.exists(asr_path):
|
||||
d = json.load(open(asr_path))
|
||||
asr_segs = len(d.get("segments", []))
|
||||
except: pass
|
||||
stages.append({"name":"ASR","passed":asr_segs>0,"detail":f"{asr_segs} seg","elapsed":0.0})
|
||||
|
||||
# 2. ASRX file
|
||||
asrx_path = f"/Users/accusys/momentry/output_dev/{UUID}.asrx.json"
|
||||
asrx_segs = 0
|
||||
try:
|
||||
if os.path.exists(asrx_path):
|
||||
d = json.load(open(asrx_path))
|
||||
asrx_segs = len(d.get("segments", []))
|
||||
except: pass
|
||||
stages.append({"name":"ASRX","passed":asrx_segs>0,"detail":f"{asrx_segs} seg","elapsed":0.0})
|
||||
|
||||
# 3. Sentence chunks
|
||||
try:
|
||||
cnt = db_query("SELECT count(*) FROM dev.chunks WHERE file_uuid=%s AND chunk_type='sentence'", (UUID,))[0][0]
|
||||
except:
|
||||
cnt = 0
|
||||
stages.append({"name":"Sentence","passed":cnt>0,"detail":f"{cnt} chunks","elapsed":0.0})
|
||||
|
||||
# 4. Vectorization (Qdrant)
|
||||
v1 = qdrant_count("momentry_dev_v1")
|
||||
stages.append({"name":"Vectorize","passed":v1>0,"detail":f"{v1} Qdrant","elapsed":0.0})
|
||||
|
||||
# 5. Face traces
|
||||
try:
|
||||
traces = db_query("SELECT count(DISTINCT trace_id) FROM dev.face_detections WHERE file_uuid=%s AND trace_id IS NOT NULL", (UUID,))[0][0]
|
||||
faces = db_query("SELECT count(*) FROM dev.face_detections WHERE file_uuid=%s AND trace_id IS NOT NULL", (UUID,))[0][0]
|
||||
except:
|
||||
traces = faces = 0
|
||||
stages.append({"name":"FaceTrace","passed":traces>0,"detail":f"{traces} traces, {faces} faces","elapsed":0.0})
|
||||
|
||||
# 6. TKG
|
||||
try:
|
||||
nodes = db_query("SELECT count(*) FROM dev.tkg_nodes WHERE file_uuid=%s", (UUID,))[0][0]
|
||||
edges = db_query("SELECT count(*) FROM dev.tkg_edges WHERE file_uuid=%s", (UUID,))[0][0]
|
||||
except:
|
||||
nodes = edges = 0
|
||||
stages.append({"name":"TKG","passed":nodes>0,"detail":f"{nodes} nodes, {edges} edges","elapsed":0.0})
|
||||
|
||||
# 7. Trace chunks
|
||||
try:
|
||||
tc = db_query("SELECT count(*) FROM dev.chunks WHERE file_uuid=%s AND chunk_type='trace'", (UUID,))[0][0]
|
||||
except:
|
||||
tc = 0
|
||||
stages.append({"name":"TraceChunks","passed":tc>0,"detail":f"{tc} chunks","elapsed":0.0})
|
||||
|
||||
# 8. Phase 1 release
|
||||
p1 = PROJECT / "release" / "phase1" / "latest"
|
||||
p1_ok = p1.exists() and (p1 / "RELEASE_INFO.txt").exists()
|
||||
p1_size = sum(f.stat().st_size for f in p1.rglob("*") if f.is_file()) // (1024*1024) if p1.exists() else 0
|
||||
stages.append({"name":"Phase1","passed":p1_ok,"detail":f"{p1_size}MB","elapsed":0.0})
|
||||
|
||||
all_passed = all(s["passed"] for s in stages)
|
||||
return {
|
||||
"uuid": UUID,
|
||||
"passed": all_passed,
|
||||
"stages": stages,
|
||||
"checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"total_elapsed": round(time.time() - t0, 1),
|
||||
"health": get_health(),
|
||||
}
|
||||
|
||||
def get_health():
|
||||
h = {}
|
||||
try:
|
||||
import os
|
||||
load = os.getloadavg()
|
||||
h["cpu_load_1m"] = round(load[0], 1)
|
||||
h["cpu_load_5m"] = round(load[1], 1)
|
||||
except:
|
||||
h["cpu_load_1m"] = h["cpu_load_5m"] = -1
|
||||
|
||||
try:
|
||||
import subprocess
|
||||
rss = 0
|
||||
out = subprocess.run(["ps", "-A", "-o", "rss="], capture_output=True, text=True, timeout=5).stdout
|
||||
for line in out.strip().split("\n"):
|
||||
if line.strip():
|
||||
rss += int(line.strip())
|
||||
h["memory_used_mb"] = rss // 1024 if rss else 0
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
d = subprocess.run(["df", "-h", "/Users/accusys/momentry/output_dev"],
|
||||
capture_output=True, text=True, timeout=5).stdout.strip().split("\n")[-1].split()
|
||||
h["disk_use_pct"] = d[4] if len(d) > 4 else "?"
|
||||
h["disk_avail"] = d[3] if len(d) > 3 else "?"
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
import torch
|
||||
h["gpu_available"] = torch.backends.mps.is_available()
|
||||
except:
|
||||
h["gpu_available"] = False
|
||||
|
||||
services = {"postgresql": False, "qdrant": False, "embedding": False, "llm": False}
|
||||
try:
|
||||
conn = psycopg2.connect(DB_URL)
|
||||
conn.close()
|
||||
services["postgresql"] = True
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
r = qdrant_get("/collections")
|
||||
services["qdrant"] = r is not None
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
resp = urllib.request.urlopen("http://localhost:11436/health", timeout=3)
|
||||
services["embedding"] = resp.status == 200
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
req = urllib.request.Request(LLM_URL,
|
||||
data=json.dumps({"model":"google_gemma-4-26B-A4B-it-Q5_K_M.gguf","messages":[{"role":"user","content":"ping"}],"max_tokens":1}).encode(),
|
||||
headers={"Content-Type":"application/json"}, method="POST")
|
||||
resp = urllib.request.urlopen(req, timeout=3)
|
||||
services["llm"] = resp.status == 200
|
||||
except:
|
||||
pass
|
||||
|
||||
h["services"] = services
|
||||
return h
|
||||
|
||||
def get_qdrant_info():
|
||||
result = []
|
||||
for col in COLLECTIONS:
|
||||
r = qdrant_get(f"/collections/{col}")
|
||||
if r:
|
||||
info = r.get("result", {})
|
||||
cfg = info.get("config", {}).get("params", {}).get("vectors", {})
|
||||
result.append({
|
||||
"name": col,
|
||||
"points": info.get("points_count", 0),
|
||||
"dim": cfg.get("size", "?"),
|
||||
})
|
||||
else:
|
||||
result.append({"name": col, "points": -1, "dim": "?"})
|
||||
return result
|
||||
|
||||
def get_db_info():
|
||||
result = {}
|
||||
try:
|
||||
rows = db_query("""
|
||||
SELECT 'videos', count(*) FROM dev.videos
|
||||
UNION ALL SELECT 'chunks', count(*) FROM dev.chunks
|
||||
UNION ALL SELECT 'face_detections', count(*) FROM dev.face_detections
|
||||
UNION ALL SELECT 'identities', count(*) FROM dev.identities
|
||||
UNION ALL SELECT 'tkg_nodes', count(*) FROM dev.tkg_nodes
|
||||
UNION ALL SELECT 'tkg_edges', count(*) FROM dev.tkg_edges
|
||||
""")
|
||||
for r in rows:
|
||||
result[r[0]] = r[1]
|
||||
except:
|
||||
pass
|
||||
return result
|
||||
|
||||
def get_processes():
|
||||
import subprocess
|
||||
scripts = ["clean_sentence_text.py", "generate_sentence_summaries.py"]
|
||||
result = {}
|
||||
for s in scripts:
|
||||
try:
|
||||
r = subprocess.run(["pgrep", "-f", s], capture_output=True, text=True, timeout=3)
|
||||
pids = [p.strip() for p in r.stdout.strip().split("\n") if p.strip()]
|
||||
if pids:
|
||||
r2 = subprocess.run(["ps", "-o", "etime=", "-p", pids[0]], capture_output=True, text=True, timeout=3)
|
||||
result[s] = {"pid": int(pids[0]), "elapsed": r2.stdout.strip()}
|
||||
else:
|
||||
result[s] = None
|
||||
except:
|
||||
result[s] = None
|
||||
return result
|
||||
|
||||
TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh-TW">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Momentry Dashboard</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: #0d1117; color: #c9d1d9; padding: 20px; }
|
||||
.container { max-width: 1200px; margin: 0 auto; }
|
||||
h1 { font-size: 24px; margin-bottom: 20px; color: #58a6ff; }
|
||||
h2 { font-size: 16px; margin-bottom: 12px; color: #8b949e; text-transform: uppercase; letter-spacing: 1px; }
|
||||
.section { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 20px; margin-bottom: 20px; }
|
||||
.row { display: flex; gap: 16px; flex-wrap: wrap; }
|
||||
.col { flex: 1; min-width: 300px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 14px; }
|
||||
th, td { padding: 8px 12px; text-align: left; border-bottom: 1px solid #21262d; }
|
||||
th { color: #8b949e; font-weight: 600; }
|
||||
.pass { color: #3fb950; font-weight: bold; }
|
||||
.fail { color: #f85149; font-weight: bold; }
|
||||
.stat-value { font-size: 28px; font-weight: 700; }
|
||||
.stat-label { font-size: 12px; color: #8b949e; margin-top: 4px; }
|
||||
.stat-card { background: #0d1117; border: 1px solid #30363d; border-radius: 6px; padding: 16px; text-align: center; }
|
||||
.refresh-bar { display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px; }
|
||||
.last-updated { color: #8b949e; font-size: 13px; }
|
||||
button { background: #238636; color: white; border: none; padding: 8px 20px; border-radius: 6px; cursor: pointer; font-size: 14px; }
|
||||
button:hover { background: #2ea043; }
|
||||
#error { display: none; background: #3a1b1b; border: 1px solid #f85149; border-radius: 6px; padding: 12px; margin-bottom: 16px; color: #f85149; font-size: 13px; }
|
||||
@media (max-width: 768px) { .col { min-width: 100%; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="refresh-bar">
|
||||
<h1>Momentry Dashboard <span id="roleBadge" style="font-size:14px;background:#1f2937;padding:4px 12px;border-radius:12px;margin-left:8px">\U0001F4BB {{ SYSTEM_ROLE }}</span></h1>
|
||||
<div style="display:flex;align-items:center;gap:8px">
|
||||
<span class="last-updated" id="lastUpdated">\u2014</span>
|
||||
<button onclick="load()" style="background:#238636;padding:6px 14px;font-size:13px">\u27F3 Refresh</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="error"></div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<div class="section">
|
||||
<h2>\u2705 Pipeline Checklist</h2>
|
||||
<table id="checklist"><tr><td>Loading...</td></tr></table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="section">
|
||||
<h2>\U0001F4BB System Health</h2>
|
||||
<div id="health" style="font-size:14px">Loading...</div>
|
||||
</div>
|
||||
<div class="section">
|
||||
<h2>\U0001F6E0 Services</h2>
|
||||
<div id="services" style="font-size:14px">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<div class="section">
|
||||
<h2>\U0001F4CA Qdrant Collections</h2>
|
||||
<div id="qdrant" style="font-size:14px">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="section">
|
||||
<h2>\u2699\uFE0F Background Processes</h2>
|
||||
<div id="processes" style="font-size:14px">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<div class="section">
|
||||
<h2>\U0001F4DB Database</h2>
|
||||
<div id="db" style="font-size:14px">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function load() {
|
||||
const ts = new Date().toISOString().slice(11,19);
|
||||
document.getElementById("lastUpdated").textContent = "\U0001F504 " + ts;
|
||||
document.getElementById("error").style.display = "none";
|
||||
|
||||
try {
|
||||
const resp = await fetch("/api/all");
|
||||
if (!resp.ok) throw new Error("HTTP " + resp.status);
|
||||
const d = await resp.json();
|
||||
renderChecklist(d.status);
|
||||
renderHealth(d.status.health);
|
||||
renderQdrant(d.qdrant);
|
||||
renderProcesses(d.processes);
|
||||
renderDb(d.db);
|
||||
document.getElementById("lastUpdated").textContent = "\u2705 " + ts;
|
||||
} catch(e) {
|
||||
showError(e.message);
|
||||
document.getElementById("lastUpdated").textContent = "\u274C " + ts;
|
||||
}
|
||||
}
|
||||
|
||||
function showError(msg) {
|
||||
document.getElementById("error").innerHTML = "\u26A0\uFE0F " + msg;
|
||||
document.getElementById("error").style.display = "block";
|
||||
}
|
||||
|
||||
function renderChecklist(status) {
|
||||
const job = status || {};
|
||||
const stages = job.stages || [];
|
||||
let h = "<tr><th>Stage</th><th>Status</th><th>Detail</th></tr>";
|
||||
for (const s of stages) {
|
||||
h += "<tr><td>" + s.name + '</td><td class="' + (s.passed ? "pass" : "fail") + '">' + (s.passed ? "\u2705" : "\u274C") + "</td><td>" + s.detail + "</td></tr>";
|
||||
}
|
||||
h += '<tr style="font-weight:bold;border-top:2px solid #30363d"><td>TOTAL</td><td class="' + (job.passed ? "pass" : "fail") + '">' + (job.passed ? "\u2705" : "\u274C") + "</td><td></td></tr>";
|
||||
document.getElementById("checklist").innerHTML = h;
|
||||
}
|
||||
|
||||
function renderHealth(h) {
|
||||
if (!h) return;
|
||||
let cards = '<div class="row">';
|
||||
cards += '<div class="col"><div class="stat-card"><div class="stat-value">' + (h.cpu_load_1m ?? "?") + '</div><div class="stat-label">CPU Load (1m)</div></div></div>';
|
||||
const memPct = h.memory_used_mb ? (h.memory_used_mb / 49152 * 100).toFixed(1) : "?";
|
||||
cards += '<div class="col"><div class="stat-card"><div class="stat-value">' + memPct + '%</div><div class="stat-label">Memory</div></div></div>';
|
||||
cards += '<div class="col"><div class="stat-card"><div class="stat-value">' + (h.disk_use_pct ?? "?") + '</div><div class="stat-label">Disk</div></div></div>';
|
||||
cards += "</div>";
|
||||
document.getElementById("health").innerHTML = cards;
|
||||
|
||||
const svc = h.services || {};
|
||||
let svcHtml = "";
|
||||
for (const [k, v] of Object.entries(svc)) {
|
||||
svcHtml += '<span style="margin-right:16px">' + (v ? "\u2705" : "\u274C") + " " + k + "</span>";
|
||||
}
|
||||
document.getElementById("services").innerHTML = svcHtml;
|
||||
}
|
||||
|
||||
function renderQdrant(cols) {
|
||||
if (!cols) return;
|
||||
let h = "<table><tr><th>Collection</th><th>Points</th><th>Dim</th></tr>";
|
||||
for (let i = 0; i < cols.length; i++) {
|
||||
const c = cols[i];
|
||||
h += "<tr><td>" + c.name + "</td><td>" + (c.points >= 0 ? Number(c.points).toLocaleString() : "err") + "</td><td>" + c.dim + "</td></tr>";
|
||||
}
|
||||
h += "</table>";
|
||||
document.getElementById("qdrant").innerHTML = h;
|
||||
}
|
||||
|
||||
function renderProcesses(procs) {
|
||||
if (!procs) return;
|
||||
let h = "<table><tr><th>Script</th><th>Status</th></tr>";
|
||||
for (const name in procs) {
|
||||
const info = procs[name];
|
||||
if (info) {
|
||||
h += "<tr><td>" + name + "</td><td>\u25B6 running " + info.elapsed + "</td></tr>";
|
||||
} else {
|
||||
h += '<tr style="color:#8b949e"><td>' + name + "</td><td>\u23F3 idle</td></tr>";
|
||||
}
|
||||
}
|
||||
h += "</table>";
|
||||
document.getElementById("processes").innerHTML = h;
|
||||
}
|
||||
|
||||
function renderDb(d) {
|
||||
if (!d) return;
|
||||
const keys = ["videos","chunks","face_detections","identities","tkg_nodes","tkg_edges"];
|
||||
let h = '<div class="row">';
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const v = d[keys[i]] ?? 0;
|
||||
h += '<div class="col"><div class="stat-card"><div class="stat-value">' + Number(v).toLocaleString() + '</div><div class="stat-label">' + keys[i].replace(/_/g," ") + '</div></div></div>';
|
||||
}
|
||||
h += "</div>";
|
||||
document.getElementById("db").innerHTML = h;
|
||||
}
|
||||
|
||||
load();
|
||||
setInterval(load, 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
if __name__ == "__main__":
|
||||
port = int(os.environ.get("DASHBOARD_PORT", 5050))
|
||||
print(f"Momentry Dashboard v2: http://0.0.0.0:{port}")
|
||||
app.run(host="0.0.0.0", port=port, threaded=True)
|
||||
Reference in New Issue
Block a user