4f1e546104
- Realtime dashboard on port 5050 - Pipeline checklist (8 stages ✅/❌) - System health: CPU, memory, disk, GPU, 4 services - Redis metrics: memory, clients, hit rate, keys - DB table counts: videos, chunks, face_detections, identities, TKG - Processor timing chart - Auto-refresh every 15 seconds + manual refresh button Usage: python3 scripts/dashboard.py Open: http://localhost:5050
330 lines
12 KiB
Python
330 lines
12 KiB
Python
#!/opt/homebrew/bin/python3.11
|
|
"""
|
|
Momentry Dashboard — Flask web app
|
|
Reads pipeline status + Redis + system health on demand
|
|
"""
|
|
|
|
import json, os, subprocess, sys
|
|
from pathlib import Path
|
|
from flask import Flask, jsonify, render_template_string
|
|
|
|
app = Flask(__name__)
|
|
|
|
PROJECT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def run_status_json():
|
|
"""Run pipeline_status.py and return parsed JSON"""
|
|
r = subprocess.run(
|
|
[sys.executable, str(PROJECT / "scripts/pipeline_status.py"), "--json"],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
return json.loads(r.stdout)
|
|
|
|
|
|
def run_redis_info():
|
|
"""Fetch key Redis metrics"""
|
|
result = {}
|
|
try:
|
|
r = subprocess.run(
|
|
["redis-cli", "-a", "accusys", "INFO", "all"],
|
|
capture_output=True, text=True, timeout=5,
|
|
)
|
|
for line in r.stdout.split("\n"):
|
|
line = line.strip()
|
|
if ":" not in line or line.startswith("#"):
|
|
continue
|
|
k, v = line.split(":", 1)
|
|
if k in ("total_system_memory_human", "used_memory_human",
|
|
"used_memory_peak_human", "total_connections_received",
|
|
"total_commands_processed", "keyspace_hits", "keyspace_misses",
|
|
"connected_clients", "uptime_in_seconds"):
|
|
result[k] = v if not v.endswith("_human") else v
|
|
result["keyspace_hits"] = int(result.get("keyspace_hits", 0))
|
|
result["keyspace_misses"] = int(result.get("keyspace_misses", 0))
|
|
hit_rate = result["keyspace_hits"] / max(result["keyspace_hits"] + result["keyspace_misses"], 1) * 100
|
|
result["hit_rate_pct"] = round(hit_rate, 1)
|
|
except Exception as e:
|
|
result["error"] = str(e)
|
|
|
|
# Get momentry keys
|
|
try:
|
|
r = subprocess.run(
|
|
["redis-cli", "-a", "accusys", "KEYS", "momentry_dev:*"],
|
|
capture_output=True, text=True, timeout=5,
|
|
)
|
|
keys = [k for k in r.stdout.strip().split("\n") if k]
|
|
result["momentry_keys"] = len(keys)
|
|
# Sample a few interesting keys
|
|
sample = {}
|
|
for k in keys:
|
|
if k.endswith(":health") or k.endswith(":job:") or ":processor:" in k:
|
|
pass
|
|
if len(sample) >= 5:
|
|
break
|
|
result["key_sample"] = keys[:10]
|
|
except:
|
|
result["momentry_keys"] = 0
|
|
result["key_sample"] = []
|
|
|
|
return result
|
|
|
|
|
|
def run_db_info():
|
|
"""Fetch DB metrics"""
|
|
psql = "/Users/accusys/pgsql/18.3/bin/psql"
|
|
cmd = [psql, "-U", "accusys", "-d", "momentry", "-t", "-A"]
|
|
result = {}
|
|
try:
|
|
r = subprocess.run(cmd + ["-c", """
|
|
SELECT 'videos', count(*) FROM dev.videos
|
|
UNION ALL SELECT 'chunks', count(*) FROM dev.chunks
|
|
UNION ALL SELECT 'face_detections', count(*) FROM dev.face_detections
|
|
UNION ALL SELECT 'identities', count(*) FROM dev.identities
|
|
UNION ALL SELECT 'tkg_nodes', count(*) FROM dev.tkg_nodes
|
|
UNION ALL SELECT 'tkg_edges', count(*) FROM dev.tkg_edges
|
|
"""], capture_output=True, text=True, timeout=10)
|
|
for line in r.stdout.strip().split("\n"):
|
|
if not line.strip() or "|" not in line:
|
|
continue
|
|
parts = line.split("|")
|
|
result[parts[0].strip()] = int(parts[1])
|
|
except:
|
|
pass
|
|
return result
|
|
|
|
|
|
@app.route("/")
|
|
def index():
|
|
return render_template_string(TEMPLATE)
|
|
|
|
|
|
@app.route("/api/status")
|
|
def api_status():
|
|
return jsonify(run_status_json())
|
|
|
|
|
|
@app.route("/api/redis")
|
|
def api_redis():
|
|
return jsonify(run_redis_info())
|
|
|
|
|
|
@app.route("/api/db")
|
|
def api_db():
|
|
return jsonify(run_db_info())
|
|
|
|
|
|
@app.route("/api/all")
|
|
def api_all():
|
|
return jsonify({
|
|
"status": run_status_json(),
|
|
"redis": run_redis_info(),
|
|
"db": run_db_info(),
|
|
})
|
|
|
|
|
|
TEMPLATE = """<!DOCTYPE html>
|
|
<html lang="zh-TW">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>Momentry Dashboard</title>
|
|
<style>
|
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
background: #0d1117; color: #c9d1d9; padding: 20px; }
|
|
.container { max-width: 1200px; margin: 0 auto; }
|
|
h1 { font-size: 24px; margin-bottom: 20px; color: #58a6ff; }
|
|
h2 { font-size: 16px; margin-bottom: 12px; color: #8b949e; text-transform: uppercase; letter-spacing: 1px; }
|
|
.section { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 20px; margin-bottom: 20px; }
|
|
.row { display: flex; gap: 16px; flex-wrap: wrap; }
|
|
.col { flex: 1; min-width: 300px; }
|
|
table { width: 100%; border-collapse: collapse; font-size: 14px; }
|
|
th, td { padding: 8px 12px; text-align: left; border-bottom: 1px solid #21262d; }
|
|
th { color: #8b949e; font-weight: 600; }
|
|
.pass { color: #3fb950; font-weight: bold; }
|
|
.fail { color: #f85149; font-weight: bold; }
|
|
.badge { display: inline-block; padding: 2px 8px; border-radius: 12px; font-size: 12px; font-weight: 600; }
|
|
.badge-ok { background: #1b3a1b; color: #3fb950; }
|
|
.badge-err { background: #3a1b1b; color: #f85149; }
|
|
.badge-warn { background: #3a321b; color: #d29922; }
|
|
.stat-value { font-size: 28px; font-weight: 700; }
|
|
.stat-label { font-size: 12px; color: #8b949e; margin-top: 4px; }
|
|
.stat-card { background: #0d1117; border: 1px solid #30363d; border-radius: 6px; padding: 16px; text-align: center; }
|
|
.refresh-bar { display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px; }
|
|
.last-updated { color: #8b949e; font-size: 13px; }
|
|
button { background: #238636; color: white; border: none; padding: 8px 20px; border-radius: 6px; cursor: pointer; font-size: 14px; }
|
|
button:hover { background: #2ea043; }
|
|
.progress-bar { height: 6px; background: #21262d; border-radius: 3px; margin-top: 8px; }
|
|
.progress-fill { height: 100%; border-radius: 3px; background: #238636; transition: width 0.5s; }
|
|
@media (max-width: 768px) { .col { min-width: 100%; } }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="refresh-bar">
|
|
<h1>Momentry Dashboard</h1>
|
|
<div>
|
|
<span class="last-updated" id="lastUpdated">—</span>
|
|
<button onclick="fetchAll()" style="margin-left:12px">⟳ Refresh</button>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>✅ Pipeline Checklist</h2>
|
|
<table id="checklist"><tr><td colspan="3">Loading...</td></tr></table>
|
|
</div>
|
|
</div>
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>💻 System Health</h2>
|
|
<div id="health" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
<div class="section">
|
|
<h2>🛠 Services</h2>
|
|
<div id="services" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>⚡ Redis</h2>
|
|
<div id="redis" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
</div>
|
|
<div class="col">
|
|
<div class="section">
|
|
<h2>🗄 Database</h2>
|
|
<div id="db" style="font-size:14px">Loading...</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>⏱ Processor Timing</h2>
|
|
<table id="timing"><tr><td>Loading...</td></tr></table>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function fetchAll() {
|
|
const ts = new Date().toISOString().slice(11,19);
|
|
document.getElementById('lastUpdated').textContent = '🔄 ' + ts;
|
|
|
|
try {
|
|
const status = await (await fetch('/api/status')).json();
|
|
renderChecklist(status.job);
|
|
renderHealth(status.health);
|
|
renderTiming(status.health?.processors);
|
|
document.getElementById('lastUpdated').textContent = '✅ ' + ts;
|
|
} catch(e) {
|
|
document.getElementById('checklist').innerHTML = '<tr><td class="fail">Error: ' + e.message + '</td></tr>';
|
|
}
|
|
|
|
try {
|
|
const redis = await (await fetch('/api/redis')).json();
|
|
renderRedis(redis);
|
|
} catch(e) {}
|
|
|
|
try {
|
|
const db = await (await fetch('/api/db')).json();
|
|
renderDb(db);
|
|
} catch(e) {}
|
|
}
|
|
|
|
function renderChecklist(job) {
|
|
if (!job || !job.stages) return;
|
|
let h = '<tr><th>Stage</th><th>Status</th><th>Detail</th><th>Time</th></tr>';
|
|
for (const s of job.stages) {
|
|
const cls = s.passed ? 'pass' : 'fail';
|
|
const icon = s.passed ? '✅' : '❌';
|
|
h += '<tr><td>' + s.name + '</td><td class="' + cls + '">' + icon + '</td><td>' + s.detail + '</td><td>' + s.elapsed + 's</td></tr>';
|
|
}
|
|
const totalCls = job.passed ? 'pass' : 'fail';
|
|
h += '<tr style="font-weight:bold;border-top:2px solid #30363d"><td>TOTAL</td><td class="' + totalCls + '">' + (job.passed ? '✅' : '❌') + '</td><td></td><td>' + job.total_elapsed + 's</td></tr>';
|
|
document.getElementById('checklist').innerHTML = h;
|
|
}
|
|
|
|
function renderHealth(h) {
|
|
if (!h) return;
|
|
const memPct = h.memory_used_mb ? (h.memory_used_mb / 49152 * 100).toFixed(1) : '?';
|
|
const memBar = Math.min(parseFloat(memPct), 100);
|
|
const barColor = memBar > 85 ? '#f85149' : memBar > 70 ? '#d29922' : '#3fb950';
|
|
document.getElementById('health').innerHTML = `
|
|
<div class="row">
|
|
<div class="col"><div class="stat-card"><div class="stat-value">${h.cpu_load_1m ?? '?'}</div><div class="stat-label">CPU Load (1m)</div></div></div>
|
|
<div class="col"><div class="stat-card"><div class="stat-value">${memPct}%</div><div class="stat-label">Memory</div><div class="progress-bar"><div class="progress-fill" style="width:${memBar}%;background:${barColor}"></div></div></div></div>
|
|
<div class="col"><div class="stat-card"><div class="stat-value">${h.disk_use_pct ?? '?'}</div><div class="stat-label">Disk Used</div></div></div>
|
|
</div>
|
|
`;
|
|
const svc = h.services || {};
|
|
document.getElementById('services').innerHTML = Object.entries(svc).map(([k,v]) =>
|
|
'<span style="margin-right:16px">' + (v ? '✅' : '❌') + ' ' + k + '</span>'
|
|
).join('');
|
|
}
|
|
|
|
function renderTiming(procs) {
|
|
if (!procs) return;
|
|
let h = '<tr><th>Processor</th><th>Duration</th></tr>';
|
|
for (const p of procs) {
|
|
const d = p.duration_secs;
|
|
const dur = d ? (d < 60 ? d + 's' : d < 3600 ? Math.floor(d/60) + 'm ' + (d%60) + 's' : Math.floor(d/3600) + 'h ' + Math.floor((d%3600)/60) + 'm') : 'running';
|
|
h += '<tr><td>' + p.name + '</td><td>' + dur + '</td></tr>';
|
|
}
|
|
document.getElementById('timing').innerHTML = h;
|
|
}
|
|
|
|
function renderRedis(r) {
|
|
if (!r) return;
|
|
let h = '<div class="row">';
|
|
const cards = [
|
|
{k:'used_memory_human', l:'Memory Used'},
|
|
{k:'total_system_memory_human', l:'System Memory'},
|
|
{k:'connected_clients', l:'Clients'},
|
|
{k:'hit_rate_pct', l:'Hit Rate'},
|
|
{k:'momentry_keys', l:'Momentry Keys'},
|
|
{k:'uptime_in_seconds', l:'Uptime'},
|
|
];
|
|
for (const c of cards) {
|
|
let v = r[c.k] ?? '—';
|
|
if (c.k === 'uptime_in_seconds' && typeof v === 'number') {
|
|
v = v > 86400 ? Math.round(v/86400) + 'd' : Math.round(v/3600) + 'h';
|
|
}
|
|
if (c.k === 'hit_rate_pct' && typeof v === 'number') v = v.toFixed(1) + '%';
|
|
h += '<div class="col"><div class="stat-card"><div class="stat-value">' + v + '</div><div class="stat-label">' + c.l + '</div></div></div>';
|
|
}
|
|
h += '</div>';
|
|
if (r.key_sample && r.key_sample.length) {
|
|
h += '<div style="margin-top:12px;font-size:12px;color:#8b949e">Recent keys: ' + r.key_sample.slice(0,6).join(', ') + '</div>';
|
|
}
|
|
document.getElementById('redis').innerHTML = h;
|
|
}
|
|
|
|
function renderDb(d) {
|
|
if (!d) return;
|
|
const rows = ['videos','chunks','face_detections','identities','tkg_nodes','tkg_edges'];
|
|
let h = '<div class="row">';
|
|
for (const key of rows) {
|
|
const v = d[key] ?? 0;
|
|
h += '<div class="col"><div class="stat-card"><div class="stat-value">' + v.toLocaleString() + '</div><div class="stat-label">' + key.replace(/_/g,' ') + '</div></div></div>';
|
|
}
|
|
h += '</div>';
|
|
document.getElementById('db').innerHTML = h;
|
|
}
|
|
|
|
fetchAll();
|
|
setInterval(fetchAll, 15000);
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
if __name__ == "__main__":
|
|
port = int(os.environ.get("DASHBOARD_PORT", 5050))
|
|
print(f"Momentry Dashboard: http://0.0.0.0:{port}")
|
|
app.run(host="0.0.0.0", port=port, debug=False)
|