feat: schema version tracking, SHA256 integrity, setup scripts, bug fixes
This commit is contained in:
Accusys
11
release/migrate_add_schema_version.sql
Normal file
11
release/migrate_add_schema_version.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
-- Migration: Add schema version tracking table
|
||||
-- Date: 2026-05-15
|
||||
-- Usage: psql -U accusys -d momentry -f migrate_add_schema_version.sql
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
filename TEXT NOT NULL UNIQUE,
|
||||
checksum TEXT NOT NULL,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
duration_ms INTEGER DEFAULT 0
|
||||
);
|
||||
10
release/migrate_fix_chunk_id_format.sql
Normal file
10
release/migrate_fix_chunk_id_format.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
-- Migration: Normalize chunk_id format for all chunks
|
||||
-- Converts integer-format chunk_ids ('0', '1', '2', ...) to {file_uuid}_{id}
|
||||
-- Date: 2026-05-15
|
||||
-- Usage: psql -U accusys -d momentry -f migrate_fix_chunk_id_format.sql
|
||||
-- Note: runs with current search_path; use SET search_path TO <schema>; for target schema
|
||||
|
||||
UPDATE chunk
|
||||
SET chunk_id = file_uuid || '_' || id::text
|
||||
WHERE chunk_id ~ '^[0-9]+$'
|
||||
AND chunk_id != file_uuid || '_' || id::text;
|
||||
92
release/migrate_public_schema_v4.sql
Normal file
92
release/migrate_public_schema_v4.sql
Normal file
@@ -0,0 +1,92 @@
|
||||
-- ============================================================
|
||||
-- Public Schema Migration: V3.x → V4.0
|
||||
-- Purpose: Sync public schema with dev schema for v1.0.0 release
|
||||
-- Date: 2026-04-30
|
||||
-- ============================================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ============================================================
|
||||
-- 1. videos table: Add missing columns
|
||||
-- ============================================================
|
||||
|
||||
-- Add parent_uuid column (V4.0: for parent-child video relationships)
|
||||
ALTER TABLE public.videos
|
||||
ADD COLUMN IF NOT EXISTS parent_uuid VARCHAR(255);
|
||||
|
||||
-- ============================================================
|
||||
-- 2. chunks table: Add missing columns
|
||||
-- ============================================================
|
||||
|
||||
-- Add summary_text (V4.0: LLM-generated summaries)
|
||||
ALTER TABLE public.chunks
|
||||
ADD COLUMN IF NOT EXISTS summary_text TEXT;
|
||||
|
||||
-- Add metadata_version (V4.0: track metadata updates)
|
||||
ALTER TABLE public.chunks
|
||||
ADD COLUMN IF NOT EXISTS metadata_version INTEGER DEFAULT 0;
|
||||
|
||||
-- Add content_version (V4.0: track content updates)
|
||||
ALTER TABLE public.chunks
|
||||
ADD COLUMN IF NOT EXISTS content_version INTEGER DEFAULT 0;
|
||||
|
||||
-- ============================================================
|
||||
-- 3. face_detections table: Migrate V3.x → V4.0 structure
|
||||
-- ============================================================
|
||||
|
||||
-- Step 3a: Add new V4.0 columns
|
||||
ALTER TABLE public.face_detections
|
||||
ADD COLUMN IF NOT EXISTS file_uuid VARCHAR(255);
|
||||
|
||||
ALTER TABLE public.face_detections
|
||||
ADD COLUMN IF NOT EXISTS bbox JSONB;
|
||||
|
||||
-- Step 3b: Migrate video_uuid → file_uuid
|
||||
UPDATE public.face_detections
|
||||
SET file_uuid = video_uuid
|
||||
WHERE file_uuid IS NULL AND video_uuid IS NOT NULL;
|
||||
|
||||
-- Step 3c: Create bbox JSONB from x/y/width/height columns
|
||||
UPDATE public.face_detections
|
||||
SET bbox = jsonb_build_object(
|
||||
'x', x,
|
||||
'y', y,
|
||||
'width', width,
|
||||
'height', height
|
||||
)
|
||||
WHERE bbox IS NULL AND x IS NOT NULL;
|
||||
|
||||
-- Step 3d: Add NOT NULL constraint to file_uuid (after migration)
|
||||
ALTER TABLE public.face_detections
|
||||
ALTER COLUMN file_uuid SET NOT NULL;
|
||||
|
||||
-- Step 3e: Add index for new file_uuid column
|
||||
CREATE INDEX IF NOT EXISTS idx_face_detections_file_uuid
|
||||
ON public.face_detections(file_uuid);
|
||||
|
||||
-- Note: We keep old columns (video_uuid, x, y, width, height, timestamp_secs,
|
||||
-- embedding, identity_confidence, cluster_id) for backward compatibility.
|
||||
-- They can be removed in a future cleanup after verifying no dependencies.
|
||||
|
||||
-- ============================================================
|
||||
-- 4. Verify migration
|
||||
-- ============================================================
|
||||
|
||||
-- Check video count
|
||||
SELECT COUNT(*) AS total_videos FROM public.videos;
|
||||
|
||||
-- Check chunk columns
|
||||
SELECT column_name, data_type
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = 'chunks'
|
||||
AND column_name IN ('summary_text', 'metadata_version', 'content_version')
|
||||
ORDER BY column_name;
|
||||
|
||||
-- Check face_detections migration
|
||||
SELECT
|
||||
COUNT(*) AS total_faces,
|
||||
COUNT(CASE WHEN file_uuid IS NOT NULL THEN 1 END) AS with_file_uuid,
|
||||
COUNT(CASE WHEN bbox IS NOT NULL THEN 1 END) AS with_bbox
|
||||
FROM public.face_detections;
|
||||
|
||||
COMMIT;
|
||||
130
release/migrate_public_schema_v4_tables.sql
Normal file
130
release/migrate_public_schema_v4_tables.sql
Normal file
@@ -0,0 +1,130 @@
|
||||
-- ============================================================
|
||||
-- Public Schema Migration V4.0: Create Missing Tables
|
||||
-- Purpose: Sync public schema tables with dev schema
|
||||
-- Date: 2026-04-30
|
||||
-- ============================================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ============================================================
|
||||
-- 1. pre_chunks table (processor raw output)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.pre_chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
file_uuid VARCHAR(255) NOT NULL,
|
||||
processor_type VARCHAR(50) NOT NULL,
|
||||
coordinate_type VARCHAR(50) NOT NULL,
|
||||
coordinate_index BIGINT NOT NULL,
|
||||
start_frame BIGINT,
|
||||
end_frame BIGINT,
|
||||
start_time DOUBLE PRECISION,
|
||||
end_time DOUBLE PRECISION,
|
||||
fps DOUBLE PRECISION,
|
||||
data JSONB NOT NULL,
|
||||
identity_id UUID,
|
||||
confidence DOUBLE PRECISION,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pre_chunks_file_uuid ON public.pre_chunks(file_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_pre_chunks_processor ON public.pre_chunks(processor_type);
|
||||
|
||||
-- ============================================================
|
||||
-- 2. file_identities table (V4.0 identity binding)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.file_identities (
|
||||
id SERIAL PRIMARY KEY,
|
||||
file_uuid VARCHAR(255) NOT NULL,
|
||||
identity_id INTEGER NOT NULL REFERENCES public.identities(id),
|
||||
confidence DOUBLE PRECISION DEFAULT 1.0,
|
||||
metadata JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(file_uuid, identity_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_file_identities_file ON public.file_identities(file_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_file_identities_identity ON public.file_identities(identity_id);
|
||||
|
||||
-- ============================================================
|
||||
-- 3. jobs table (job queue)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.jobs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
asset_uuid VARCHAR(255) NOT NULL,
|
||||
rule VARCHAR(100),
|
||||
status VARCHAR(50) DEFAULT 'QUEUED',
|
||||
assigned_processor_id UUID,
|
||||
processed_frames BIGINT DEFAULT 0,
|
||||
total_frames BIGINT DEFAULT 0,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_asset ON public.jobs(asset_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_status ON public.jobs(status);
|
||||
|
||||
-- ============================================================
|
||||
-- 4. chunks_rule1 table (Rule1 sentence chunks)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.chunks_rule1 (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
uuid VARCHAR(255) NOT NULL,
|
||||
chunk_id VARCHAR(100) NOT NULL,
|
||||
chunk_index INTEGER NOT NULL,
|
||||
start_time DOUBLE PRECISION,
|
||||
end_time DOUBLE PRECISION,
|
||||
start_frame BIGINT,
|
||||
end_frame BIGINT,
|
||||
fps DOUBLE PRECISION,
|
||||
content JSONB,
|
||||
metadata JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_rule1_uuid ON public.chunks_rule1(uuid);
|
||||
|
||||
-- ============================================================
|
||||
-- 5. mac_allocations table (MAC-based UUID allocation)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.mac_allocations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
mac_address VARCHAR(17) NOT NULL UNIQUE,
|
||||
allocation_count INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- 6. resources table (resource registry)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.resources (
|
||||
id SERIAL PRIMARY KEY,
|
||||
resource_type VARCHAR(100) NOT NULL,
|
||||
resource_id VARCHAR(255) NOT NULL,
|
||||
metadata JSONB,
|
||||
status VARCHAR(50) DEFAULT 'active',
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(resource_type, resource_id)
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- 7. talents table (talent management)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.talents (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
metadata JSONB,
|
||||
status VARCHAR(50) DEFAULT 'active',
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- Verify
|
||||
-- ============================================================
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name IN ('pre_chunks', 'file_identities', 'jobs', 'chunks_rule1', 'mac_allocations', 'resources', 'talents')
|
||||
ORDER BY table_name;
|
||||
|
||||
COMMIT;
|
||||
96
release/migrate_public_v4_complete.sql
Normal file
96
release/migrate_public_v4_complete.sql
Normal file
@@ -0,0 +1,96 @@
|
||||
-- ============================================================
|
||||
-- Public Schema Migration V4.0 (Complete)
|
||||
-- Purpose: Sync public schema with dev schema for v1.0.0 release
|
||||
-- Date: 2026-04-30
|
||||
-- ============================================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ============================================================
|
||||
-- 1. videos table
|
||||
-- ============================================================
|
||||
ALTER TABLE public.videos ADD COLUMN IF NOT EXISTS parent_uuid VARCHAR(255);
|
||||
|
||||
-- ============================================================
|
||||
-- 2. chunks table
|
||||
-- ============================================================
|
||||
ALTER TABLE public.chunks ADD COLUMN IF NOT EXISTS summary_text TEXT;
|
||||
ALTER TABLE public.chunks ADD COLUMN IF NOT EXISTS metadata_version INTEGER DEFAULT 0;
|
||||
ALTER TABLE public.chunks ADD COLUMN IF NOT EXISTS content_version INTEGER DEFAULT 0;
|
||||
|
||||
-- ============================================================
|
||||
-- 3. face_detections table - Migrate V3.x → V4.0
|
||||
-- ============================================================
|
||||
|
||||
-- Add V4.0 columns
|
||||
ALTER TABLE public.face_detections ADD COLUMN IF NOT EXISTS file_uuid VARCHAR(255);
|
||||
ALTER TABLE public.face_detections ADD COLUMN IF NOT EXISTS bbox JSONB;
|
||||
|
||||
-- Migrate data from old columns to new columns
|
||||
UPDATE public.face_detections
|
||||
SET file_uuid = video_uuid
|
||||
WHERE file_uuid IS NULL AND video_uuid IS NOT NULL;
|
||||
|
||||
UPDATE public.face_detections
|
||||
SET bbox = jsonb_build_object('x', x, 'y', y, 'width', width, 'height', height)
|
||||
WHERE bbox IS NULL AND x IS NOT NULL;
|
||||
|
||||
-- Make file_uuid NOT NULL after migration
|
||||
ALTER TABLE public.face_detections ALTER COLUMN file_uuid SET NOT NULL;
|
||||
|
||||
-- Add indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_face_detections_file_uuid ON public.face_detections(file_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_face_detections_identity_id ON public.face_detections(identity_id) WHERE identity_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_face_detections_candidates ON public.face_detections(confidence DESC) WHERE identity_id IS NULL;
|
||||
|
||||
-- ============================================================
|
||||
-- 4. pre_chunks table
|
||||
-- ============================================================
|
||||
ALTER TABLE public.pre_chunks ADD COLUMN IF NOT EXISTS timestamp DOUBLE PRECISION;
|
||||
|
||||
-- ============================================================
|
||||
-- 5. file_identities table (V4.0 N:N relationship)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS public.file_identities (
|
||||
id SERIAL PRIMARY KEY,
|
||||
file_uuid VARCHAR(255) NOT NULL,
|
||||
identity_id INTEGER NOT NULL REFERENCES public.identities(id),
|
||||
face_count INTEGER DEFAULT 0,
|
||||
speaker_count INTEGER DEFAULT 0,
|
||||
first_appearance BIGINT,
|
||||
last_appearance BIGINT,
|
||||
confidence DOUBLE PRECISION DEFAULT 1.0,
|
||||
metadata JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(file_uuid, identity_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_file_identities_file ON public.file_identities(file_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_file_identities_identity ON public.file_identities(identity_id);
|
||||
|
||||
-- ============================================================
|
||||
-- 6. jobs table updates
|
||||
-- ============================================================
|
||||
ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS error_message TEXT;
|
||||
ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS processor_list JSONB;
|
||||
|
||||
-- ============================================================
|
||||
-- Verification
|
||||
-- ============================================================
|
||||
SELECT 'Migration Complete' AS status;
|
||||
|
||||
-- Check column counts
|
||||
SELECT table_name, COUNT(*) AS column_count
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name IN ('videos', 'chunks', 'face_detections', 'pre_chunks', 'file_identities', 'jobs')
|
||||
GROUP BY table_name
|
||||
ORDER BY table_name;
|
||||
|
||||
-- Check data migration
|
||||
SELECT
|
||||
COUNT(*) AS total_faces,
|
||||
COUNT(CASE WHEN file_uuid IS NOT NULL THEN 1 END) AS with_file_uuid,
|
||||
COUNT(CASE WHEN bbox IS NOT NULL THEN 1 END) AS with_bbox
|
||||
FROM public.face_detections;
|
||||
|
||||
COMMIT;
|
||||
Reference in New Issue
Block a user