feat: Phase 2.6 edges migration to Qdrant (TKG-only architecture)
Phase 2.6.1: co_occurrence_edges migration - build_co_occurrence_edges_from_qdrant() - Qdrant embeddings → frame grouping → YOLO objects - Result: 6679 edges (vs 6701 PostgreSQL) Phase 2.6.2: face_face_edges migration - build_face_face_edges_from_qdrant() - Qdrant embeddings → frame grouping → face pairs - mutual_gaze detection preserved - Result: 6 edges (exact match) Phase 2.6.3: speaker_face_edges migration - build_speaker_face_edges_from_qdrant() - Qdrant embeddings → trace_id frame ranges - SPEAKS_AS edge creation Architecture: - All edges use Qdrant payload (no face_detections queries) - PostgreSQL fallback for empty Qdrant - Estimated 3.6x performance improvement Testing: - Playground (3003): ✓ All Phase 2.6 logs verified - Edge counts: ✓ Close match with PostgreSQL - Fallback: ✓ Working Docs: - docs_v1.0/DESIGN/TKG_PHASE2_6_EDGES_MIGRATION.md - docs_v1.0/M4_workspace/2026-06-21_phase2_6_test.md
This commit is contained in:
Accusys
208
v1.1/scripts/deploy_package_v1.11.sh
Normal file
208
v1.1/scripts/deploy_package_v1.11.sh
Normal file
@@ -0,0 +1,208 @@
|
||||
#!/bin/bash
|
||||
# Momentry Release Package — Deploy Script
|
||||
# Usage: SCHEMA=public bash deploy.sh [--force]
|
||||
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
UUID=$(basename "$DIR")
|
||||
PG_BIN="${PG_BIN:-/Users/accusys/pgsql/18.3/bin}"
|
||||
DB_NAME="${DB_NAME:-momentry}"
|
||||
DB_USER="${DB_USER:-accusys}"
|
||||
SCHEMA="${SCHEMA:-dev}"
|
||||
DEMO_DIR="${DEMO_DIR:-/Users/accusys/momentry/var/sftpgo/data/demo}"
|
||||
OUTPUT_DIR="${OUTPUT_DIR:-/Users/accusys/momentry/output_dev}"
|
||||
P="${SCHEMA}."
|
||||
|
||||
# Parse --force flag
|
||||
FORCE=false
|
||||
for arg in "$@"; do
|
||||
[ "$arg" = "--force" ] && FORCE=true
|
||||
done
|
||||
|
||||
echo "=== Momentry Package Deploy ==="
|
||||
echo "UUID: $UUID"
|
||||
echo "Time: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
# 0. Version & build compatibility check
|
||||
echo "[0/9] Checking system version and build..."
|
||||
PKG_VER=$(python3 -c "import json; f=json.load(open('$DIR/file_info.json')); print(f.get('momentry_version','?'))")
|
||||
PKG_BUILD=$(python3 -c "import json; f=json.load(open('$DIR/file_info.json')); print(f.get('momentry_build','?'))")
|
||||
SRV=$(curl -sf http://localhost:3003/health | python3 -c "
|
||||
import json,sys
|
||||
d=json.load(sys.stdin)
|
||||
print(d.get('version','unknown'), d.get('build_git_hash','unknown'))
|
||||
" 2>/dev/null || echo "down down")
|
||||
SRV_VER=$(echo "$SRV" | cut -d' ' -f1)
|
||||
SRV_BUILD=$(echo "$SRV" | cut -d' ' -f2)
|
||||
if [ "$SRV_VER" = "down" ]; then
|
||||
echo " ⚠️ Cannot reach server at localhost:3003, skipping version check"
|
||||
elif [ "$SRV_VER" != "$PKG_VER" ]; then
|
||||
echo " ❌ Version mismatch:"
|
||||
echo " Package Server"
|
||||
echo " Version: $PKG_VER $SRV_VER"
|
||||
echo ""
|
||||
echo " Please obtain the matching system upgrade package."
|
||||
exit 1
|
||||
elif [ "$SRV_BUILD" != "unknown" ] && [ "$SRV_BUILD" != "$PKG_BUILD" ]; then
|
||||
echo " ⚠️ Build mismatch (package=$PKG_BUILD, server=$SRV_BUILD) — version ok, continuing"
|
||||
else
|
||||
echo " ✅ Server v$SRV_VER (build $SRV_BUILD) matches package"
|
||||
fi
|
||||
|
||||
# 1. Verify package integrity
|
||||
echo "[1/9] Verifying package..."
|
||||
REQUIRED_FILES=("data.sql" "file_info.json")
|
||||
MISSING=0
|
||||
for f in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$DIR/$f" ]; then
|
||||
echo " ❌ Missing: $f"
|
||||
MISSING=1
|
||||
fi
|
||||
done
|
||||
if [ $MISSING -eq 1 ]; then
|
||||
echo "ERROR: Package incomplete"
|
||||
exit 1
|
||||
fi
|
||||
echo " ✅ Package verified"
|
||||
|
||||
# 1b. Check for existing data before pre-clean
|
||||
EXISTING=$("$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -t -A -c "SELECT COUNT(*) FROM ${P}videos WHERE file_uuid='$UUID'" 2>/dev/null || echo "0")
|
||||
if [ "$EXISTING" -gt 0 ] && [ "$FORCE" = false ]; then
|
||||
echo ""
|
||||
echo " ⚠️ Existing data found for UUID $UUID ($EXISTING videos table rows)"
|
||||
echo " This will DELETE all existing data for this UUID and re-import."
|
||||
echo " Continue? [y/N] "
|
||||
read -r CONFIRM
|
||||
if [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ]; then
|
||||
echo " ❌ Aborted by user"
|
||||
exit 1
|
||||
fi
|
||||
elif [ "$EXISTING" -gt 0 ] && [ "$FORCE" = true ]; then
|
||||
echo " ⚠️ Existing data will be overwritten (--force)"
|
||||
fi
|
||||
|
||||
# 2. Pre-clean: clear all data for this file (avoids PK/FK conflicts on COPY)
|
||||
echo "[2/9] Pre-cleaning existing data for this file..."
|
||||
"$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" <<-EOSQL > /dev/null 2>&1
|
||||
DELETE FROM ${P}tkg_edges WHERE file_uuid = '$UUID';
|
||||
DELETE FROM ${P}tkg_nodes WHERE file_uuid = '$UUID';
|
||||
DELETE FROM ${P}identity_bindings WHERE identity_id IN (
|
||||
SELECT id FROM ${P}identities WHERE file_uuid = '$UUID' OR file_uuid IS NULL OR name LIKE 'PERSON_${UUID:0:8}%'
|
||||
);
|
||||
DELETE FROM ${P}identities WHERE file_uuid = '$UUID';
|
||||
-- Also delete global identities that will be re-imported (TMDB + merged + user_defined)
|
||||
DELETE FROM ${P}identities WHERE file_uuid IS NULL AND source IN ('tmdb', 'merged', 'user_defined');
|
||||
DELETE FROM ${P}face_detections WHERE file_uuid = '$UUID';
|
||||
DELETE FROM ${P}chunk_vectors WHERE uuid = '$UUID';
|
||||
DELETE FROM ${P}chunk WHERE file_uuid = '$UUID';
|
||||
DELETE FROM ${P}videos WHERE file_uuid = '$UUID';
|
||||
-- Drop legacy constraint that conflicts with global identity re-import
|
||||
DO \$\$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'identities_name_key') THEN
|
||||
ALTER TABLE ${P}identities DROP CONSTRAINT identities_name_key;
|
||||
END IF;
|
||||
END
|
||||
\$\$;
|
||||
-- Ensure content_hash column exists (added in v1.0.0 V4.2)
|
||||
ALTER TABLE ${P}videos ADD COLUMN IF NOT EXISTS content_hash TEXT;
|
||||
EOSQL
|
||||
echo " ✅ Cleared existing data for $UUID (migration: content_hash column ensured)"
|
||||
|
||||
# 3. Import each table file in dependency order (FK constraints)
|
||||
echo "[3/9] Normalizing schema prefix and importing DB data..."
|
||||
# Normalize COPY schema prefix in per-table files and data.sql
|
||||
if [ "${SCHEMA}" != "dev" ]; then
|
||||
for f in "$DIR"/sql/dev_*.sql; do
|
||||
sed -i '' "s/COPY dev\./COPY ${SCHEMA}./g" "$f"
|
||||
done
|
||||
# Also normalize data.sql if present (full dump format)
|
||||
if [ -f "$DIR/data.sql" ]; then
|
||||
sed -i '' "s/COPY dev\./COPY ${SCHEMA}./g" "$DIR/data.sql"
|
||||
sed -i '' "s/Schema: dev/Schema: ${SCHEMA}/g" "$DIR/data.sql"
|
||||
fi
|
||||
echo " Schema prefix normalized: dev. → ${SCHEMA}."
|
||||
fi
|
||||
IMPORT_ORDER=(
|
||||
"sql/dev_videos.sql"
|
||||
"sql/dev_chunk.sql"
|
||||
"sql/dev_chunk_vectors.sql"
|
||||
"sql/dev_face_detections.sql"
|
||||
"sql/dev_identities.sql"
|
||||
"sql/dev_identity_bindings.sql"
|
||||
"sql/dev_tkg_nodes.sql"
|
||||
"sql/dev_tkg_edges.sql"
|
||||
)
|
||||
for tbl_sql in "${IMPORT_ORDER[@]}"; do
|
||||
tbl=$(basename "$tbl_sql" .sql)
|
||||
echo " Importing $tbl..."
|
||||
"$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -f "$DIR/$tbl_sql" 2>&1 | tail -1
|
||||
done
|
||||
echo " ✅ Data imported"
|
||||
|
||||
# 4. Copy video to demo dir (only this package's video, not scanning others)
|
||||
VIDEO_FILE=$(ls "$DIR"/*.mp4 "$DIR"/*.mov "$DIR"/*.avi "$DIR"/*.mkv 2>/dev/null | head -1)
|
||||
if [ -n "$VIDEO_FILE" ]; then
|
||||
VIDEO_NAME=$(basename "$VIDEO_FILE")
|
||||
DEST="$DEMO_DIR/$VIDEO_NAME"
|
||||
if [ ! -f "$DEST" ]; then
|
||||
cp "$VIDEO_FILE" "$DEST"
|
||||
echo "[4/9] Video copied: $VIDEO_NAME → $DEMO_DIR"
|
||||
else
|
||||
echo "[4/9] Video already in demo dir, skipping"
|
||||
fi
|
||||
else
|
||||
echo "[4/9] No video file in package, skipping"
|
||||
fi
|
||||
|
||||
# 5. Set video status to completed (package is fully processed)
|
||||
echo "[5/9] Setting deployment status..."
|
||||
"$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -c "UPDATE ${P}videos SET status = 'completed' WHERE file_uuid = '$UUID'" > /dev/null 2>&1
|
||||
echo " ✅ Status set to 'completed'"
|
||||
|
||||
# 6. Copy output files
|
||||
echo "[6/9] Copying output files..."
|
||||
COPIED=0
|
||||
for f in "$DIR"/*.json "$DIR"/*.sqlite "$DIR"/*.sqlite; do
|
||||
if [ -f "$f" ]; then
|
||||
FNAME=$(basename "$f")
|
||||
if [ "$FNAME" != "file_info.json" ] && [ "$FNAME" != "package.json" ]; then
|
||||
cp "$f" "$OUTPUT_DIR/$FNAME"
|
||||
COPIED=$((COPIED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo " ✅ $COPIED files copied to $OUTPUT_DIR"
|
||||
|
||||
# 7. Verify deployment
|
||||
echo "[7/9] Installing vec0.dylib..."
|
||||
if [ -f "$DIR/vec0.dylib" ]; then
|
||||
cp "$DIR/vec0.dylib" "/tmp/vec0.dylib"
|
||||
echo " ✅ vec0.dylib installed to /tmp/"
|
||||
else
|
||||
echo " ⚠️ vec0.dylib not found in package"
|
||||
fi
|
||||
|
||||
# 8. Verify deployment
|
||||
echo "[8/9] Verifying deployment..."
|
||||
CHUNKS=$("$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -t -A -c "SELECT COUNT(*) FROM ${P}chunk WHERE file_uuid='$UUID'" 2>/dev/null || echo "?")
|
||||
FACES=$("$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -t -A -c "SELECT COUNT(*) FROM ${P}face_detections WHERE file_uuid='$UUID'" 2>/dev/null || echo "?")
|
||||
IDENTS=$("$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -t -A -c "SELECT COUNT(*) FROM ${P}identities WHERE file_uuid='$UUID'" 2>/dev/null || echo "?")
|
||||
TKG_NODES=$("$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -t -A -c "SELECT COUNT(*) FROM ${P}tkg_nodes WHERE file_uuid='$UUID'" 2>/dev/null || echo "?")
|
||||
TKG_EDGES=$("$PG_BIN/psql" -U "$DB_USER" -d "$DB_NAME" -t -A -c "SELECT COUNT(*) FROM ${P}tkg_edges WHERE file_uuid='$UUID'" 2>/dev/null || echo "?")
|
||||
|
||||
echo ""
|
||||
echo "=== Deploy Complete ==="
|
||||
echo " UUID: $UUID"
|
||||
echo " Chunks: $CHUNKS"
|
||||
echo " Faces: $FACES"
|
||||
echo " Identities: $IDENTS"
|
||||
echo " TKG nodes: $TKG_NODES"
|
||||
echo " TKG edges: $TKG_EDGES"
|
||||
echo " Output: $OUTPUT_DIR/"
|
||||
echo ""
|
||||
echo "Package is self-contained — no further processing needed."
|
||||
echo ""
|
||||
echo "Offline report:"
|
||||
echo " python3 scripts/render_offline_report.py $OUTPUT_DIR/$UUID.sqlite"
|
||||
Reference in New Issue
Block a user