Initial commit: Momentry Core v0.1

- Rust-based digital asset management system
- Video analysis: ASR, OCR, YOLO, Face, Pose
- RAG capabilities with Qdrant vector database
- Multi-database support: PostgreSQL, Redis, MongoDB
- Monitoring system with launchd plists
- n8n workflow automation integration
This commit is contained in:
accusys
2026-03-16 15:07:33 +08:00
parent ca24794853
commit 75edf0aa71
101 changed files with 19858 additions and 0 deletions

375
monitor/storage/backup_monitor.sh Executable file
View File

@@ -0,0 +1,375 @@
#!/bin/bash
# Momentry 備份監控與溫冷轉移
# 路徑: /Users/accusys/momentry_core_0.1/monitor/storage/backup_monitor.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_DIR="/Users/accusys/momentry/log/monitor"
mkdir -p "$LOG_DIR"
LOG_FILE="$LOG_DIR/backup_check.log"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
# 備份根目錄
BACKUP_BASE="/Users/accusys/momentry/backup"
# 服務列表
SERVICES=("postgresql" "redis" "mariadb" "n8n" "qdrant" "gitea" "ollama" "caddy" "mongodb" "sftpgo" "php")
# 溫冷分層配置
TIER_HOT=7 # 7天內 - 快速存儲
TIER_WARM=30 # 7-30天 - 標準存儲
TIER_COLD=90 # 30-90天 - 低成本存儲
TIER_ARCHIVE=365 # >90天 - 歸檔
# 記錄備份元數據
record_backup() {
local service=$1
local backup_file=$2
local backup_size=$3
local backup_type=$4
psql -U accusys -h localhost -d momentry << EOF 2>/dev/null
INSERT INTO backup_registry (service_name, backup_file, backup_size_bytes, backup_type, status, created_at)
VALUES ('$service', '$backup_file', $backup_size, '$backup_type', 'completed', NOW())
ON CONFLICT DO NOTHING;
EOF
}
# 記錄備份存儲統計
record_backup_stats() {
local tier=$1
local file_count=$2
local total_size=$3
psql -U accusys -h localhost -d momentry << EOF 2>/dev/null
INSERT INTO backup_storage_stats (tier, file_count, total_size_bytes, record_time)
VALUES ('$tier', $file_count, $total_size, NOW());
EOF
}
# 初始化備份目錄結構
init_backup_dirs() {
log "初始化備份目錄結構..."
mkdir -p "$BACKUP_BASE"/{daily,weekly,monthly,archive}
for service in "${SERVICES[@]}"; do
mkdir -p "$BACKUP_BASE/daily/$service"
mkdir -p "$BACKUP_BASE/weekly/$service"
mkdir -p "$BACKUP_BASE/monthly/$service"
done
log "備份目錄結構已初始化"
}
# 檢查備份狀態
check_backup_status() {
log "=== 檢查備份狀態 ==="
# 命名規範: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext}
# 例如: postgresql_db_20260315_030000.sql.gz
local total_backup_size=0
echo ""
echo "========================================"
echo "備份監控狀態"
echo "時間: $(date)"
echo "========================================"
echo ""
echo "命名規範: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext}"
echo ""
for service in "${SERVICES[@]}"; do
service_backup_dir="$BACKUP_BASE/daily/$service"
if [ -d "$service_backup_dir" ]; then
file_count=$(find "$service_backup_dir" -type f 2>/dev/null | wc -l)
size=$(du -sb "$service_backup_dir" 2>/dev/null | cut -f1)
latest_file=$(find "$service_backup_dir" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" -o -name "*.rdb" \) 2>/dev/null | head -1)
# 處理 size 為空或 0 的情況
if [ -z "$size" ] || [ "$size" = "0" ]; then
size=$(find "$service_backup_dir" -type f -exec ls -l {} \; 2>/dev/null | awk '{sum+=$5} END {print sum}')
fi
size_str="0B"
if [ -n "$size" ] && [ "$size" -gt 0 ]; then
if [ "$size" -gt 1073741824 ]; then
size_str="$((size / 1073741824))GB"
elif [ "$size" -gt 1048576 ]; then
size_str="$((size / 1048576))MB"
elif [ "$size" -gt 1024 ]; then
size_str="$((size / 1024))KB"
else
size_str="${size}B"
fi
fi
# 檢查最近備份時間 (使用文件名中的時間戳)
days_since_backup=0
today=$(date +%Y%m%d)
if [ -n "$latest_file" ]; then
# 從文件名提取日期
file_date=$(echo "$latest_file" | sed 's/.*\([0-9]\{8\}\).*/\1/')
if [ -n "$file_date" ] && [ "$file_date" = "$today" ]; then
days_since_backup=0
else
days_since_backup=1
fi
fi
# 狀態指示
if [ "$days_since_backup" -eq 0 ]; then
status="✅ 今日已備份"
elif [ "$days_since_backup" -le 1 ]; then
status="⚠️ 昨日已備份"
elif [ "$days_since_backup" -le 7 ]; then
status="⚠️ ${days_since_backup}天前"
else
status="❌ 超過${days_since_backup}天未備份!"
fi
echo " $service: $file_count 個文件, $size_str | $status"
[ -n "$size" ] && total_backup_size=$((total_backup_size + size))
# 記錄到資料庫
[ -n "$size" ] && record_backup "$service" "$service_backup_dir" "$size" "daily"
else
echo " $service: ❌ 備份目錄不存在"
fi
done
echo ""
echo "----------------------------------------"
echo "存儲分層:"
echo "----------------------------------------"
for tier in daily weekly monthly archive; do
tier_path="$BACKUP_BASE/$tier"
if [ -d "$tier_path" ]; then
file_count=$(find "$tier_path" -type f 2>/dev/null | wc -l)
size=$(du -sb "$tier_path" 2>/dev/null | cut -f1)
tier_size_str="0B"
if [ -n "$size" ] && [ "$size" -gt 0 ] 2>/dev/null; then
if [ "$size" -gt 1073741824 ]; then
tier_size_str="$((size / 1073741824))GB"
elif [ "$size" -gt 1048576 ]; then
tier_size_str="$((size / 1048576))MB"
else
tier_size_str="$((size / 1024))KB"
fi
fi
echo " $tier: $file_count 個文件, $tier_size_str"
[ -n "$size" ] && record_backup_stats "$tier" "$file_count" "$size"
fi
done
total_size_str="0B"
if [ "$total_backup_size" -gt 1073741824 ]; then
total_size_str="$((total_backup_size / 1073741824))GB"
elif [ "$total_backup_size" -gt 1048576 ]; then
total_size_str="$((total_backup_size / 1048576))MB"
elif [ "$total_backup_size" -gt 1024 ]; then
total_size_str="$((total_backup_size / 1024))KB"
else
total_size_str="${total_backup_size}B"
fi
echo ""
echo "----------------------------------------"
echo "總計: ${total_backup_size} bytes ($total_size_str)"
echo "========================================"
# 記錄總計
record_backup_stats "total" 0 "$total_backup_size"
}
# 溫冷轉移 - 將舊備份移動到低成本存儲
tier_backups() {
log "執行溫冷轉移..."
local moved_count=0
# 7天前: daily -> weekly
# 命名格式: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext}
find "$BACKUP_BASE/daily" -type f -mtime +7 | while read -r file; do
service=$(basename "$(dirname "$file")")
# 解析時間戳
filename=$(basename "$file")
timestamp=$(echo "$filename" | grep -oP '\d{8}_\d{6}' || echo "")
if [ -n "$timestamp" ]; then
year=${timestamp:0:4}
week=$(date -j -f "%Y%m%d_%H%M%S" "${timestamp}_0000" +%Y-W%V 2>/dev/null || echo "$year-W$(date +%V)")
else
week=$(date +%Y-W%V)
fi
dest_dir="$BACKUP_BASE/weekly/$service/$week"
mkdir -p "$dest_dir"
mv "$file" "$dest_dir/" 2>/dev/null && log "移動: $file -> $dest_dir" && moved_count=$((moved_count + 1))
done
# 30天前: weekly -> monthly
find "$BACKUP_BASE/weekly" -type f -mtime +30 | while read -r file; do
service=$(basename "$(dirname "$(dirname "$file")")")
month=$(date +%Y-%m)
dest_dir="$BACKUP_BASE/monthly/$service/$month"
mkdir -p "$dest_dir"
mv "$file" "$dest_dir/" 2>/dev/null && log "移動: $file -> $dest_dir" && moved_count=$((moved_count + 1))
done
# 90天前: monthly -> archive (長期歸檔)
find "$BACKUP_BASE/monthly" -type f -mtime +90 | while read -r file; do
service=$(basename "$(dirname "$(dirname "$file")")")
year=$(date +%Y)
dest_dir="$BACKUP_BASE/archive/$service/$year"
mkdir -p "$dest_dir"
mv "$file" "$dest_dir/" 2>/dev/null && log "歸檔: $file -> $dest_dir" && moved_count=$((moved_count + 1))
done
log "溫冷轉移完成: 移動了 $moved_count 個文件"
}
# 清理過期備份
cleanup_old() {
log "清理過期備份..."
# 歸檔超過 365 天
find "$BACKUP_BASE/archive" -type f -mtime +365 -delete 2>/dev/null
# 每月備份保留 12 個月
find "$BACKUP_BASE/monthly" -type f -mtime +365 -delete 2>/dev/null
# 每週備份保留 12 週
find "$BACKUP_BASE/weekly" -type f -mtime +84 -delete 2>/dev/null
# 每日備份保留 30 天
find "$BACKUP_BASE/daily" -type f -mtime +30 -delete 2>/dev/null
log "清理完成"
}
# 驗證備份完整性
verify_backup() {
local backup_file=$1
if [[ "$backup_file" == *.tar.gz ]]; then
tar -tzf "$backup_file" > /dev/null 2>&1
return $?
elif [[ "$backup_file" == *.sql ]]; then
head -1 "$backup_file" | grep -q "SQL" && return 0
return 1
elif [[ "$backup_file" == *.rdb ]]; then
file "$backup_file" | grep -q "data" && return 0
return 1
fi
return 0
}
# 生成備份報告
generate_report() {
local report_file="/Users/accusys/momentry/log/backup_report_$(date +%Y%m%d).txt"
{
echo "========================================"
echo "Momentry 備份報告"
echo "生成時間: $(date)"
echo "========================================"
echo ""
echo "## 備份狀態"
check_backup_status
echo ""
echo "## 存儲使用趨勢 (最近30天)"
psql -U accusys -h localhost -d momentry -t -A -c "
SELECT tier,
COUNT(*) as files,
AVG(total_size_bytes)::bigint as avg_size,
MAX(total_size_bytes)::bigint as max_size
FROM backup_storage_stats
WHERE record_time > NOW() - INTERVAL '30 days'
GROUP BY tier
ORDER BY tier;
" 2>/dev/null || echo " (無數據)"
echo ""
echo "## 建議"
# 檢查是否有服務超過7天未備份
for service in "${SERVICES[@]}"; do
latest=$(find "$BACKUP_BASE/daily/$service" -type f 2>/dev/null | head -1)
if [ -n "$latest" ]; then
days_old=$(($(date +%s) - $(stat -f "%m" "$latest" 2>/dev/null || echo "0")) / 86400)
if [ "$days_old" -gt 7 ]; then
echo " - ⚠️ $service 超過 $days_old 天未備份,建議立即執行備份"
fi
fi
done
} > "$report_file"
log "報告已生成: $report_file"
echo "$report_file"
}
# 主程序
command=${1:-status}
case $command in
status)
check_backup_status
;;
init)
init_backup_dirs
;;
tier)
tier_backups
;;
cleanup)
cleanup_old
;;
verify)
verify_backup "${2:-}"
;;
report)
generate_report
;;
all)
log "執行完整備份維護..."
check_backup_status
tier_backups
cleanup_old
generate_report
log "備份維護完成"
;;
*)
echo "用法: $0 {status|init|tier|cleanup|verify|report|all}"
echo ""
echo " status - 檢查備份狀態"
echo " init - 初始化備份目錄"
echo " tier - 執行溫冷轉移"
echo " cleanup - 清理過期備份"
echo " verify - 驗證備份完整性"
echo " report - 生成備份報告"
echo " all - 執行所有維護任務"
exit 1
;;
esac

View File

@@ -0,0 +1,193 @@
#!/bin/bash
# Momentry Storage 管理 (Layer 7)
# 路徑: /Users/accusys/momentry_core_0.1/monitor/storage/storage_manager.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_DIR="/Users/accusys/momentry/log/monitor"
mkdir -p "$LOG_DIR"
LOG_FILE="$LOG_DIR/storage_check.log"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
# 存儲路徑配置
STORAGE_BASE="/Users/accusys/momentry"
DATA_DIR="$STORAGE_BASE/data"
TEMP_DIR="$STORAGE_BASE/tmp"
BACKUP_DIR="$STORAGE_BASE/backup"
# 用戶集群
CLUSTERS=("family" "work" "wordpress" "shared")
# 記錄使用統計
record_usage() {
local cluster=$1
local tier=$2
local file_count=$3
local total_size=$4
psql -U accusys -h localhost -d momentry << EOF 2>/dev/null
INSERT INTO storage_usage_stats (user_cluster, storage_tier, file_count, total_size_bytes, record_time)
VALUES ('$cluster', '$tier', $file_count, $total_size, NOW());
EOF
}
# 記錄檔案註冊
register_file() {
local file_path=$1
local user_cluster=$2
local file_size=$3
file_name=$(basename "$file_path")
file_hash=$(echo "$file_path" | md5sum | cut -d' ' -f1)
psql -U accusys -h localhost -d momentry << EOF 2>/dev/null
INSERT INTO file_registry (file_name, file_path, file_path_hash, file_size, user_cluster, storage_tier, status, created_at)
VALUES ('$file_name', '$file_path', '$file_hash', $file_size, '$user_cluster', 'hot', 'active', NOW())
ON CONFLICT (file_path_hash) DO UPDATE SET
last_accessed_at = NOW(),
access_count = file_registry.access_count + 1;
EOF
}
# 初始化目錄結構
init_directories() {
echo "初始化目錄結構..."
# 主目錄
mkdir -p "$DATA_DIR"
mkdir -p "$TEMP_DIR"
mkdir -p "$BACKUP_DIR"/{daily,weekly,monthly,archive}
# 用戶集群目錄
for cluster in "${CLUSTERS[@]}"; do
mkdir -p "$DATA_DIR/$cluster"
done
# 臨時子目錄
mkdir -p "$TEMP_DIR"/{upload,processing,cache,session}
echo "目錄結構已初始化"
}
# 顯示存儲狀態
show_status() {
echo "========================================"
echo "Layer 7: Storage Status"
echo "Time: $(date)"
echo "========================================"
echo ""
echo "存儲路徑:"
echo " 數據: $DATA_DIR"
echo " 臨時: $TEMP_DIR"
echo " 備份: $BACKUP_DIR"
echo ""
echo "用戶集群:"
echo "----------------------------------------"
for cluster in "${CLUSTERS[@]}"; do
cluster_path="$DATA_DIR/$cluster"
if [ -d "$cluster_path" ]; then
file_count=$(find "$cluster_path" -type f 2>/dev/null | wc -l)
total_size=$(du -sb "$cluster_path" 2>/dev/null | cut -f1)
size_str="0B"
if [ -n "$total_size" ] && [ "$total_size" -gt 0 ] 2>/dev/null; then
if [ "$total_size" -gt 1073741824 ]; then
size_str="$((total_size / 1073741824))GB"
elif [ "$total_size" -gt 1048576 ]; then
size_str="$((total_size / 1048576))MB"
elif [ "$total_size" -gt 1024 ]; then
size_str="$((total_size / 1024))KB"
else
size_str="${total_size}B"
fi
fi
echo " $cluster: $file_count files, $size_str"
[ -n "$total_size" ] && record_usage "$cluster" "hot" "$file_count" "$total_size"
else
echo " $cluster: (未創建)"
fi
done
echo ""
echo "臨時文件:"
echo "----------------------------------------"
for subdir in upload processing cache session; do
subdir_path="$TEMP_DIR/$subdir"
if [ -d "$subdir_path" ]; then
file_count=$(find "$subdir_path" -type f 2>/dev/null | wc -l)
size=$(du -sb "$subdir_path" 2>/dev/null | cut -f1)
echo " $subdir: $file_count files"
fi
done
echo ""
echo "備份目錄:"
echo "----------------------------------------"
for subdir in daily weekly monthly archive; do
subdir_path="$BACKUP_DIR/$subdir"
if [ -d "$subdir_path" ]; then
file_count=$(find "$subdir_path" -type f 2>/dev/null | wc -l)
size=$(du -sb "$subdir_path" 2>/dev/null | cut -f1)
echo " $subdir: $file_count files"
fi
done
echo ""
echo "========================================"
# 顯示資料庫統計
echo ""
echo "資料庫統計 (file_registry):"
psql -U accusys -h localhost -d momentry -t -A -c "
SELECT user_cluster, storage_tier, COUNT(*) as files,
SUM(file_size)::bigint as total_size
FROM file_registry
WHERE status = 'active'
GROUP BY user_cluster, storage_tier;
" 2>/dev/null || echo " (表未初始化)"
}
# 清理臨時文件
clean_temp() {
echo "清理臨時文件..."
# 清理超過 7 天的上傳
find "$TEMP_DIR/upload" -type f -mtime +7 -delete 2>/dev/null
# 清理超過 30 天的緩存
find "$TEMP_DIR/cache" -type f -mtime +30 -delete 2>/dev/null
# 清理超過 7 天的處理中
find "$TEMP_DIR/processing" -type f -mtime +7 -delete 2>/dev/null
echo "清理完成"
}
# 主程序
command=${1:-status}
case $command in
status)
show_status
;;
init)
init_directories
;;
clean)
clean_temp
;;
*)
echo "用法: $0 {status|init|clean}"
exit 1
;;
esac