diff --git a/docs/BACKUP_VERSIONING.md b/docs/BACKUP_VERSIONING.md new file mode 100644 index 0000000..a486482 --- /dev/null +++ b/docs/BACKUP_VERSIONING.md @@ -0,0 +1,450 @@ +# Momentry 備份版本管理規範 + +| 項目 | 內容 | +|------|------| +| 建立者 | Warren / OpenCode | +| 建立時間 | 2026-03-25 | +| 文件版本 | V1.0 | + +--- + +## 版本歷史 + +| 版本 | 日期 | 目的 | 操作人 | +|------|------|------|--------| +| V1.0 | 2026-03-25 | 建立備份版本管理規範 | OpenCode | + +--- + +## 1. 概述 + +本文檔定義 Momentry 系統的備份版本管理規範，確保新舊架構之間的回滾相容性。 + +### 1.1 版本定義 + +| 版本 | 日期 | 說明 | +|------|------|------| +| v1 | 2026-03-18 | 初始備份架構（不包含新架構組件）| +| v2 | 2026-03-25 | 新架構備份（包含 monitor_jobs, processor_results, Output 目錄）| + +### 1.2 備份版本格式 + +| 版本 | 檔案命名格式 | +|------|-------------| +| v1 | `{service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext}` | +| v2 | `{service}_{type}_v2_{YYYYMMDD}_{HHMMSS}.{ext}` | + +### 1.3 各版本涵蓋範圍 + +| 組件 | v1 | v2 | +|------|-----|-----| +| PostgreSQL (videos, chunks) | ✅ | ✅ | +| PostgreSQL (monitor_jobs) | ❌ | ✅ | +| PostgreSQL (processor_results) | ❌ | ✅ | +| Redis | ✅ | ✅ | +| MongoDB Cache | ⚠️ | ⚠️ | +| Output (probe.json) | ❌ | ✅ | + +> ⚠️ MongoDB 備份目前存在路徑問題，正在修復中 + +--- + +## 2. 備份版本識別 + +### 2.1 檔名識別 + +```bash +# 識別版本 +detect_version() { + local backup_file=$1 + if echo "$backup_file" | grep -q "_v2_"; then + echo "v2" + else + echo "v1" + fi +} + +# 使用範例 +detect_version "postgresql_db_momentry_v2_20260325_030000.sql.gz" +# 輸出: v2 + +detect_version "postgresql_db_momentry_20260324_030000.sql.gz" +# 輸出: v1 +``` + +### 2.2 內容識別 + +```bash +# 檢查是否為 v2 備份 +is_v2_backup() { + local backup_file=$1 + gzip -dc "$backup_file" 2>/dev/null | grep -q "monitor_jobs" && echo "yes" || echo "no" +} + +# 檢查是否包含 processor_results +has_processor_results() { + local backup_file=$1 + gzip -dc "$backup_file" 2>/dev/null | grep -q "processor_results" && echo "yes" || echo "no" +} +``` + +### 2.3 檔案大小比較 + +| 版本 | PostgreSQL 備份大小 | 說明 | +|------|---------------------|------| +| v1 | ~18-19 MB | 基本資料表 | +| v2 | >19 MB | 包含新表格和索引 | + +--- + +## 3. 回滾策略 + +### 3.1 回滾流程圖 + +``` +┌─────────────────────────────────────────┐ +│ 選擇還原目標 │ +└─────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────┐ +│ 選擇備份版本 │ +│ ┌───────────┐ ┌───────────┐ │ +│ │ v1 備份 │ │ v2 備份 │ │ +│ └───────────┘ └───────────┘ │ +└─────────────────────────────────────────┘ + │ + ┌─────────┴─────────┐ + ▼ ▼ + ┌──────────┐ ┌──────────┐ + │ v1 回滾 │ │ v2 回滾 │ + └──────────┘ └──────────┘ + │ │ + ▼ ▼ + ┌──────────┐ ┌──────────┐ + │ 基本資料庫 │ │ 完整還原 │ + └──────────┘ └──────────┘ +``` + +### 3.2 回滾矩陣 + +| 還原目標 | v1 備份 | v2 備份 | +|----------|---------|---------| +| 基本資料庫 | ✅ | ✅ | +| + monitor_jobs | ❌ | ✅ | +| + processor_results | ❌ | ✅ | +| + Output 檔案 | ❌ | ✅ | +| + MongoDB Cache | ⚠️ | ⚠️ | + +### 3.3 回滾相容性說明 + +#### v1 → v2（支援） + +- v1 備份可以還原到 v2 架構 +- 新架構組件會從空白狀態開始 +- 不會造成資料損壞 + +#### v2 → v1（⚠️ 警告） + +``` +⚠️ v2 回滾到 v1 可能導致資料丟失 + +影響範圍： +- monitor_jobs 資料會消失 +- processor_results 資料會消失 +- Output 檔案參照可能失效 + +建議： +1. 在還原前建立 v2 快照 +2. 或使用隔離還原（staging restore） +``` + +--- + +## 4. 還原腳本保護機制 + +### 4.1 還原前檢查 + +```bash +# 還原前檢查版本相容性 +pre_restore_check() { + local backup_file=$1 + local version=$(detect_version "$backup_file") + local current_db_version=$(check_current_db_version) + + echo "備份版本: $version" + echo "目前版本: $current_db_version" + + # v2 → v1: 警告但允許（使用者需確認） + if [ "$version" = "v1" ] && [ "$current_db_version" = "v2" ]; then + echo "⚠️ 警告：即將回滾到 v1" + echo "影響：monitor_jobs 和 processor_results 資料將被清除" + read -p "確認繼續？(y/N): " confirm + [ "$confirm" != "y" ] && exit 1 + fi + + # v1 → v2: 直接允許 + if [ "$version" = "v1" ] && [ "$current_db_version" = "v2" ]; then + echo "ℹ️ 提示：新架構組件將重新初始化" + fi + + # v2 → v2: 直接允許 + # v1 → v1: 直接允許 +} +``` + +### 4.2 隔離還原（Staging Restore） + +```bash +# 只還原到暫存資料庫，不影響生產 +restore_to_staging() { + local backup_file=$1 + local version=$(detect_version "$backup_file") + + echo "執行隔離還原..." + echo "版本: $version" + + # 建立暫存資料庫 + PGPASSWORD="$PG_PASSWORD" psql -U "$PG_USER" -d postgres << EOF + DROP DATABASE IF EXISTS momentry_staging; + CREATE DATABASE momentry_staging; + EOF + + # 還原到暫存資料庫 + PGPASSWORD="$PG_PASSWORD" pg_restore -U "$PG_USER" -d "momentry_staging" \ + --no-owner --no-acl "$backup_file" + + echo "✅ 還原完成：momentry_staging" + echo "驗證命令：psql -U accusys -d momentry_staging -c '\\dt'" +} +``` + +### 4.3 版本驗證命令 + +```bash +# 識別所有備份版本 +ls /Users/accusys/momentry/backup/daily/postgresql/*.sql.gz | \ + xargs -I {} sh -c 'echo "{}: $(detect_version {})"' + +# 驗證 v2 備份內容 +verify_v2_backup() { + local backup_file=$1 + + echo "驗證備份: $backup_file" + + # 檢查 monitor_jobs + if gzip -dc "$backup_file" | grep -q "monitor_jobs"; then + echo "✅ 包含 monitor_jobs" + else + echo "❌ 缺少 monitor_jobs" + return 1 + fi + + # 檢查 processor_results + if gzip -dc "$backup_file" | grep -q "processor_results"; then + echo "✅ 包含 processor_results" + else + echo "❌ 缺少 processor_results" + return 1 + fi + + echo "✅ v2 備份驗證通過" +} +``` + +--- + +## 5. 版本遷移 + +### 5.1 v1 → v2 遷移步驟 + +| 步驟 | 說明 | 驗證 | +|------|------|------| +| 1 | 確認所有 v1 備份已完成 | `ls *.sql.gz \| grep -v v2` | +| 2 | 修改 `backup_all.sh` 加入 v2 標記 | 確認 TIMESTAMP 包含 `v2_` | +| 3 | 修正 MongoDB 路徑 | 確認指向正確目錄 | +| 4 | 新增 Output 目錄備份 | 確認 probe.json 被備份 | +| 5 | 執行測試備份 | 驗證命名格式正確 | +| 6 | 驗證 v2 備份完整性 | `verify_v2_backup` | +| 7 | 正式啟用 v2 備份 | 確認 crontab 使用新版 | + +### 5.2 遷移驗證清單 + +```bash +#!/bin/bash +# verify_v2_migration.sh + +echo "=== v2 遷移驗證 ===" + +# 1. 檢查備份腳本 +echo "1. 檢查備份腳本..." +if grep -q "v2_" /Users/accusys/momentry/scripts/backup_all.sh; then + echo " ✅ 版本標記已啟用" +else + echo " ❌ 版本標記未啟用" +fi + +# 2. 檢查 MongoDB 路徑 +echo "2. 檢查 MongoDB 路徑..." +if grep -q "/opt/homebrew/var/mongodb" /Users/accusys/momentry/scripts/backup_all.sh; then + echo " ✅ MongoDB 路徑已修正" +else + echo " ❌ MongoDB 路徑未修正" +fi + +# 3. 檢查 Output 目錄備份 +echo "3. 檢查 Output 目錄備份..." +if grep -q "momentry_output" /Users/accusys/momentry/scripts/backup_all.sh; then + echo " ✅ Output 目錄備份已啟用" +else + echo " ❌ Output 目錄備份未啟用" +fi + +# 4. 檢查最新備份 +echo "4. 檢查最新備份..." +latest_backup=$(ls -t /Users/accusys/momentry/backup/daily/postgresql/*.sql.gz 2>/dev/null | head -1) +if [ -n "$latest_backup" ]; then + version=$(detect_version "$latest_backup") + echo " 最新備份: $(basename $latest_backup)" + echo " 版本: $version" + if [ "$version" = "v2" ]; then + verify_v2_backup "$latest_backup" + fi +fi + +echo "=== 驗證完成 ===" +``` + +--- + +## 6. 疑難排解 + +### 6.1 常見問題 + +| 問題 | 原因 | 解決方案 | +|------|------|----------| +| 無法識別版本 | 檔名被修改 | 使用內容分析 `gzip -dc \| grep "monitor_jobs"` | +| v2 備份還原失敗 | 磁碟空間不足 | 清理空間後重試 | +| v1 還原覆蓋 v2 | 操作失誤 | 使用隔離還原保護生產資料 | +| MongoDB 備份為空 | 路徑錯誤 | 修正為 `/opt/homebrew/var/mongodb` | + +### 6.2 緊急回滾流程 + +```bash +#!/bin/bash +# emergency_restore.sh + +set -e + +BACKUP_FILE=$1 +VERSION=$2 + +echo "=== 緊急回滾 ===" +echo "備份檔案: $BACKUP_FILE" +echo "目標版本: $VERSION" + +# 1. 建立當前狀態快照 +echo "1. 建立當前狀態快照..." +NOW=$(date +%Y%m%d_%H%M%S) +pg_dump -U accusys -d momentry | gzip > "/tmp/momentry_emergency_$NOW.sql.gz" +echo " 快照: /tmp/momentry_emergency_$NOW.sql.gz" + +# 2. 執行還原 +echo "2. 執行還原..." +gunzip -c "$BACKUP_FILE" | psql -U accusys -d momentry + +# 3. 驗證 +echo "3. 驗證還原..." +psql -U accusys -d momentry -c "SELECT COUNT(*) FROM monitor_jobs;" + +echo "=== 回滾完成 ===" +``` + +--- + +## 7. 備份清單（v2） + +### 7.1 每日備份（v2 格式） + +| 服務 | 備份項目 | 檔案命名 | 說明 | +|------|----------|----------|------| +| PostgreSQL | momentry | `postgresql_db_momentry_v2_{date}_{time}.sql.gz` | 完整資料庫 | +| PostgreSQL | video_register | `postgresql_db_video_register_v2_{date}_{time}.sql.gz` | 影片註冊資料 | +| Redis | RDB | `redis_rdb_v2_{date}_{time}.rdb` | Redis 快照 | +| MongoDB | 資料 | `mongodb_data_v2_{date}_{time}.tar.gz` | MongoDB 資料 | +| n8n | 資料+DB | `n8n_{date}_{time}.tar.gz`, `n8n_db_{date}_{time}.sql.gz` | n8n 完整 | +| SFTPGo | 配置+DB | `sftpgo_{date}_{time}.tar.gz`, `sftpgo_db_{date}_{time}.sql.gz` | SFTPGo | +| Gitea | 資料 | `gitea_{date}_{time}.tar.gz` | Gitea | +| Output | 檔案 | `momentry_output_v2_{date}_{time}.tar.gz` | probe.json 等 | + +### 7.2 備份保留策略 + +| 類型 | 保留期限 | 位置 | +|------|----------|------| +| 每日備份 | 7 天 | `backup/daily/` | +| 每週備份 | 4 週 | `backup/weekly/` | +| 每月備份 | 12 個月 | `backup/monthly/` | +| 歸檔 | 1 年+ | `backup/archive/` | + +--- + +## 8. 相關文件 + +| 文件 | 說明 | +|------|------| +| [SERVICES.md](./SERVICES.md) | 服務說明 | +| [MOMENTRY_CORE_MONITORING.md](./MOMENTRY_CORE_MONITORING.md) | 監控規範 | +| `/Users/accusys/momentry/scripts/backup_all.sh` | 備份腳本 | +| `/Users/accusys/momentry/scripts/restore_all.sh` | 還原腳本 | +| `/Users/accusys/momentry_core_0.1/monitor/storage/backup_monitor.sh` | 備份監控 | + +--- + +## 附錄 A：v2 備份完整性檢查清單 + +```bash +#!/bin/bash +# check_v2_integrity.sh + +BACKUP_DIR="/Users/accusys/momentry/backup/daily" + +echo "=== v2 備份完整性檢查 ===" + +# 檢查 PostgreSQL +echo "1. PostgreSQL..." +pg_backup=$(ls -t "$BACKUP_DIR/postgresql"/postgresql_db_momentry_v2_*.sql.gz 2>/dev/null | head -1) +if [ -n "$pg_backup" ]; then + echo " 備份: $(basename $pg_backup)" + verify_v2_backup "$pg_backup" +else + echo " ❌ 未找到 v2 備份" +fi + +# 檢查 Output +echo "2. Output 目錄..." +output_backup=$(ls -t "$BACKUP_DIR/momentry"/momentry_output_v2_*.tar.gz 2>/dev/null | head -1) +if [ -n "$output_backup" ]; then + echo " 備份: $(basename $output_backup)" + echo " ✅ Output 備份已存在" +else + echo " ❌ 未找到 Output 備份" +fi + +# 檢查 MongoDB +echo "3. MongoDB..." +mongo_backup=$(ls -t "$BACKUP_DIR/mongodb"/mongodb_data_v2_*.tar.gz 2>/dev/null | head -1) +if [ -n "$mongo_backup" ]; then + size=$(stat -f%z "$mongo_backup" 2>/dev/null || stat -c%s "$mongo_backup" 2>/dev/null) + echo " 備份: $(basename $mongo_backup)" + echo " 大小: $size bytes" + if [ "$size" -gt 1000 ]; then + echo " ✅ MongoDB 備份有效" + else + echo " ⚠️ MongoDB 備份可能為空" + fi +else + echo " ❌ 未找到 MongoDB 備份" +fi + +echo "=== 檢查完成 ===" +``` diff --git a/monitor/storage/backup_monitor.sh b/monitor/storage/backup_monitor.sh index 32e51ec..511a5e9 100755 --- a/monitor/storage/backup_monitor.sh +++ b/monitor/storage/backup_monitor.sh @@ -1,38 +1,40 @@ #!/bin/bash +# 確保路徑正確（Crontab 環境可能缺少 PATH） +export PATH="/usr/local/bin:/opt/homebrew/bin:/opt/homebrew/opt/postgresql@18/bin:/usr/bin:/bin:/sbin:$PATH" + # Momentry 備份監控與溫冷轉移 # 路徑: /Users/accusys/momentry_core_0.1/monitor/storage/backup_monitor.sh -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LOG_DIR="/Users/accusys/momentry/log/monitor" mkdir -p "$LOG_DIR" LOG_FILE="$LOG_DIR/backup_check.log" log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" } # 備份根目錄 BACKUP_BASE="/Users/accusys/momentry/backup" -# 服務列表 -SERVICES=("postgresql" "redis" "mariadb" "n8n" "qdrant" "gitea" "ollama" "caddy" "mongodb" "sftpgo" "php") +# 服務列表 (v2 新增 momentry_output) +SERVICES=("postgresql" "redis" "mariadb" "wordpress" "n8n" "qdrant" "gitea" "ollama" "caddy" "mongodb" "sftpgo" "php" "momentry") -# 溫冷分層配置 -TIER_HOT=7 # 7天內 - 快速存儲 -TIER_WARM=30 # 7-30天 - 標準存儲 -TIER_COLD=90 # 30-90天 - 低成本存儲 -TIER_ARCHIVE=365 # >90天 - 歸檔 +# 溫冷分層配置 (預留給未來自動化策略使用) +_TIER_HOT=7 # 7天內 - 快速存儲 +_TIER_WARM=30 # 7-30天 - 標準存儲 +_TIER_COLD=90 # 30-90天 - 低成本存儲 +_TIER_ARCHIVE=365 # >90天 - 歸檔 # 記錄備份元數據 record_backup() { - local service=$1 - local backup_file=$2 - local backup_size=$3 - local backup_type=$4 - - psql -U accusys -h localhost -d momentry << EOF 2>/dev/null + local service=$1 + local backup_file=$2 + local backup_size=$3 + local backup_type=$4 + + psql -U accusys -h localhost -d momentry </dev/null INSERT INTO backup_registry (service_name, backup_file, backup_size_bytes, backup_type, status, created_at) VALUES ('$service', '$backup_file', $backup_size, '$backup_type', 'completed', NOW()) ON CONFLICT DO NOTHING; @@ -41,11 +43,11 @@ EOF # 記錄備份存儲統計 record_backup_stats() { - local tier=$1 - local file_count=$2 - local total_size=$3 - - psql -U accusys -h localhost -d momentry << EOF 2>/dev/null + local tier=$1 + local file_count=$2 + local total_size=$3 + + psql -U accusys -h localhost -d momentry </dev/null INSERT INTO backup_storage_stats (tier, file_count, total_size_bytes, record_time) VALUES ('$tier', $file_count, $total_size, NOW()); EOF @@ -53,253 +55,322 @@ EOF # 初始化備份目錄結構 init_backup_dirs() { - log "初始化備份目錄結構..." - - mkdir -p "$BACKUP_BASE"/{daily,weekly,monthly,archive} - - for service in "${SERVICES[@]}"; do - mkdir -p "$BACKUP_BASE/daily/$service" - mkdir -p "$BACKUP_BASE/weekly/$service" - mkdir -p "$BACKUP_BASE/monthly/$service" - done - - log "備份目錄結構已初始化" + log "初始化備份目錄結構..." + + mkdir -p "$BACKUP_BASE"/{daily,weekly,monthly,archive} + + for service in "${SERVICES[@]}"; do + mkdir -p "$BACKUP_BASE/daily/$service" + mkdir -p "$BACKUP_BASE/weekly/$service" + mkdir -p "$BACKUP_BASE/monthly/$service" + done + + log "備份目錄結構已初始化" } # 檢查備份狀態 check_backup_status() { - log "=== 檢查備份狀態 ===" - - # 命名規範: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext} - # 例如: postgresql_db_20260315_030000.sql.gz - - local total_backup_size=0 - - echo "" - echo "========================================" - echo "備份監控狀態" - echo "時間: $(date)" - echo "========================================" - echo "" - echo "命名規範: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext}" - echo "" - - for service in "${SERVICES[@]}"; do - service_backup_dir="$BACKUP_BASE/daily/$service" - - if [ -d "$service_backup_dir" ]; then - file_count=$(find "$service_backup_dir" -type f 2>/dev/null | wc -l) - size=$(du -sb "$service_backup_dir" 2>/dev/null | cut -f1) - latest_file=$(find "$service_backup_dir" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" -o -name "*.rdb" \) 2>/dev/null | head -1) - - # 處理 size 為空或 0 的情況 - if [ -z "$size" ] || [ "$size" = "0" ]; then - size=$(find "$service_backup_dir" -type f -exec ls -l {} \; 2>/dev/null | awk '{sum+=$5} END {print sum}') - fi - - size_str="0B" - if [ -n "$size" ] && [ "$size" -gt 0 ]; then - if [ "$size" -gt 1073741824 ]; then - size_str="$((size / 1073741824))GB" - elif [ "$size" -gt 1048576 ]; then - size_str="$((size / 1048576))MB" - elif [ "$size" -gt 1024 ]; then - size_str="$((size / 1024))KB" - else - size_str="${size}B" - fi - fi - - # 檢查最近備份時間 (使用文件名中的時間戳) - days_since_backup=0 - today=$(date +%Y%m%d) - - if [ -n "$latest_file" ]; then - # 從文件名提取日期 - file_date=$(echo "$latest_file" | sed 's/.*\([0-9]\{8\}\).*/\1/') - if [ -n "$file_date" ] && [ "$file_date" = "$today" ]; then - days_since_backup=0 - else - days_since_backup=1 - fi - fi - - # 狀態指示 - if [ "$days_since_backup" -eq 0 ]; then - status="✅ 今日已備份" - elif [ "$days_since_backup" -le 1 ]; then - status="⚠️ 昨日已備份" - elif [ "$days_since_backup" -le 7 ]; then - status="⚠️ ${days_since_backup}天前" - else - status="❌ 超過${days_since_backup}天未備份!" - fi - - echo " $service: $file_count 個文件, $size_str | $status" - - [ -n "$size" ] && total_backup_size=$((total_backup_size + size)) - - # 記錄到資料庫 - [ -n "$size" ] && record_backup "$service" "$service_backup_dir" "$size" "daily" - else - echo " $service: ❌ 備份目錄不存在" - fi - done - - echo "" - echo "----------------------------------------" - echo "存儲分層:" - echo "----------------------------------------" - - for tier in daily weekly monthly archive; do - tier_path="$BACKUP_BASE/$tier" - if [ -d "$tier_path" ]; then - file_count=$(find "$tier_path" -type f 2>/dev/null | wc -l) - size=$(du -sb "$tier_path" 2>/dev/null | cut -f1) - - tier_size_str="0B" - if [ -n "$size" ] && [ "$size" -gt 0 ] 2>/dev/null; then - if [ "$size" -gt 1073741824 ]; then - tier_size_str="$((size / 1073741824))GB" - elif [ "$size" -gt 1048576 ]; then - tier_size_str="$((size / 1048576))MB" - else - tier_size_str="$((size / 1024))KB" - fi - fi - - echo " $tier: $file_count 個文件, $tier_size_str" - [ -n "$size" ] && record_backup_stats "$tier" "$file_count" "$size" - fi - done - - total_size_str="0B" - if [ "$total_backup_size" -gt 1073741824 ]; then - total_size_str="$((total_backup_size / 1073741824))GB" - elif [ "$total_backup_size" -gt 1048576 ]; then - total_size_str="$((total_backup_size / 1048576))MB" - elif [ "$total_backup_size" -gt 1024 ]; then - total_size_str="$((total_backup_size / 1024))KB" - else - total_size_str="${total_backup_size}B" - fi - - echo "" - echo "----------------------------------------" - echo "總計: ${total_backup_size} bytes ($total_size_str)" - echo "========================================" - - # 記錄總計 - record_backup_stats "total" 0 "$total_backup_size" + log "=== 檢查備份狀態 ===" + + # 命名規範: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext} + # 例如: postgresql_db_20260315_030000.sql.gz + + local total_backup_size=0 + + echo "" + echo "========================================" + echo "備份監控狀態" + echo "時間: $(date)" + echo "========================================" + echo "" + echo "命名規範: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext}" + echo "" + + for service in "${SERVICES[@]}"; do + service_backup_dir="$BACKUP_BASE/daily/$service" + + if [ -d "$service_backup_dir" ]; then + file_count=$(find "$service_backup_dir" -type f 2>/dev/null | wc -l) + size=$(du -sb "$service_backup_dir" 2>/dev/null | cut -f1) + latest_file=$(find "$service_backup_dir" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" -o -name "*.rdb" \) 2>/dev/null | head -1) + + # 處理 size 為空或 0 的情況 + if [ -z "$size" ] || [ "$size" = "0" ]; then + size=$(find "$service_backup_dir" -type f -exec ls -l {} \; 2>/dev/null | awk '{sum+=$5} END {print sum}') + fi + + size_str="0B" + if [ -n "$size" ] && [ "$size" -gt 0 ]; then + if [ "$size" -gt 1073741824 ]; then + size_str="$((size / 1073741824))GB" + elif [ "$size" -gt 1048576 ]; then + size_str="$((size / 1048576))MB" + elif [ "$size" -gt 1024 ]; then + size_str="$((size / 1024))KB" + else + size_str="${size}B" + fi + fi + + # 檢查最近備份時間 (使用文件名中的時間戳) + days_since_backup=0 + today=$(date +%Y%m%d) + + if [ -n "$latest_file" ]; then + # 從文件名提取日期 (支援 v1 和 v2 格式) + # v1: postgresql_db_momentry_20260325_030000.sql.gz + # v2: postgresql_db_momentry_v2_20260325_030000.sql.gz + file_date=$(echo "$latest_file" | sed -E 's/.*[_v]?2?_([0-9]{8})_[0-9]{6}.*/\1/') + if [ -n "$file_date" ] && [ "$file_date" = "$today" ]; then + days_since_backup=0 + else + days_since_backup=1 + fi + fi + + # 狀態指示 + if [ "$days_since_backup" -eq 0 ]; then + status="✅ 今日已備份" + elif [ "$days_since_backup" -le 1 ]; then + status="⚠️ 昨日已備份" + elif [ "$days_since_backup" -le 7 ]; then + status="⚠️ ${days_since_backup}天前" + else + status="❌ 超過${days_since_backup}天未備份!" + fi + + echo " $service: $file_count 個文件, $size_str | $status" + + [ -n "$size" ] && total_backup_size=$((total_backup_size + size)) + + # 記錄到資料庫 + [ -n "$size" ] && record_backup "$service" "$service_backup_dir" "$size" "daily" + else + echo " $service: ❌ 備份目錄不存在" + fi + done + + echo "" + echo "----------------------------------------" + echo "存儲分層:" + echo "----------------------------------------" + + for tier in daily weekly monthly archive; do + tier_path="$BACKUP_BASE/$tier" + if [ -d "$tier_path" ]; then + file_count=$(find "$tier_path" -type f 2>/dev/null | wc -l) + size=$(du -sb "$tier_path" 2>/dev/null | cut -f1) + + tier_size_str="0B" + if [ -n "$size" ] && [ "$size" -gt 0 ] 2>/dev/null; then + if [ "$size" -gt 1073741824 ]; then + tier_size_str="$((size / 1073741824))GB" + elif [ "$size" -gt 1048576 ]; then + tier_size_str="$((size / 1048576))MB" + else + tier_size_str="$((size / 1024))KB" + fi + fi + + echo " $tier: $file_count 個文件, $tier_size_str" + [ -n "$size" ] && record_backup_stats "$tier" "$file_count" "$size" + fi + done + + total_size_str="0B" + if [ "$total_backup_size" -gt 1073741824 ]; then + total_size_str="$((total_backup_size / 1073741824))GB" + elif [ "$total_backup_size" -gt 1048576 ]; then + total_size_str="$((total_backup_size / 1048576))MB" + elif [ "$total_backup_size" -gt 1024 ]; then + total_size_str="$((total_backup_size / 1024))KB" + else + total_size_str="${total_backup_size}B" + fi + + echo "" + echo "----------------------------------------" + echo "總計: ${total_backup_size} bytes ($total_size_str)" + echo "========================================" + + # 記錄總計 + record_backup_stats "total" 0 "$total_backup_size" + + # 檢查 PATH 錯誤 + check_path_errors +} + +# 檢查 PATH 相關錯誤 +check_path_errors() { + local backup_log="/Users/accusys/momentry/log/backup.log" + + if [ -f "$backup_log" ]; then + # 檢查最近的 command not found 錯誤 + path_errors=$(grep -c "command not found" "$backup_log" 2>/dev/null || echo "0") + + if [ "$path_errors" -gt 0 ]; then + echo "" + echo "========================================" + echo "⚠️ 警告: 檢測到 PATH 相關錯誤!" + echo "========================================" + echo " 錯誤次數: $path_errors" + echo " 請檢查: $backup_log" + echo "" + + # 顯示最近3條錯誤 + echo " 最近錯誤:" + grep -n "command not found" "$backup_log" 2>/dev/null | tail -3 | while read -r line; do + echo " - $line" + done + echo "========================================" + + # 標記無效的備份文件 + mark_invalid_backups + fi + fi +} + +# 標記無效的備份文件 +mark_invalid_backups() { + local backup_log="/Users/accusys/momentry/log/backup.log" + + # 找出失敗的備份日期 (從 command not found 錯誤中提取) + grep "command not found" "$backup_log" 2>/dev/null | grep -oE "[0-9]{4}-[0-9]{2}-[0-9]{2}" | sort -u | while read -r date; do + local date_formatted + date_formatted=$(echo "$date" | tr -d '-') + local marker_file="$BACKUP_BASE/daily/postgresql/INVALID_BACKUP_${date_formatted}_030000.txt" + + if [ ! -f "$marker_file" ]; then + echo "⚠️ 發現無效備份: $date" | tee -a "$LOG_FILE" + cat >"$marker_file" < weekly - # 命名格式: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext} - find "$BACKUP_BASE/daily" -type f -mtime +7 | while read -r file; do - service=$(basename "$(dirname "$file")") - - # 解析時間戳 - filename=$(basename "$file") - timestamp=$(echo "$filename" | grep -oP '\d{8}_\d{6}' || echo "") - - if [ -n "$timestamp" ]; then - year=${timestamp:0:4} - week=$(date -j -f "%Y%m%d_%H%M%S" "${timestamp}_0000" +%Y-W%V 2>/dev/null || echo "$year-W$(date +%V)") - else - week=$(date +%Y-W%V) - fi - - dest_dir="$BACKUP_BASE/weekly/$service/$week" - mkdir -p "$dest_dir" - - mv "$file" "$dest_dir/" 2>/dev/null && log "移動: $file -> $dest_dir" && moved_count=$((moved_count + 1)) - done - - # 30天前: weekly -> monthly - find "$BACKUP_BASE/weekly" -type f -mtime +30 | while read -r file; do - service=$(basename "$(dirname "$(dirname "$file")")") - month=$(date +%Y-%m) - - dest_dir="$BACKUP_BASE/monthly/$service/$month" - mkdir -p "$dest_dir" - - mv "$file" "$dest_dir/" 2>/dev/null && log "移動: $file -> $dest_dir" && moved_count=$((moved_count + 1)) - done - - # 90天前: monthly -> archive (長期歸檔) - find "$BACKUP_BASE/monthly" -type f -mtime +90 | while read -r file; do - service=$(basename "$(dirname "$(dirname "$file")")") - year=$(date +%Y) - - dest_dir="$BACKUP_BASE/archive/$service/$year" - mkdir -p "$dest_dir" - - mv "$file" "$dest_dir/" 2>/dev/null && log "歸檔: $file -> $dest_dir" && moved_count=$((moved_count + 1)) - done - - log "溫冷轉移完成: 移動了 $moved_count 個文件" + log "執行溫冷轉移..." + + local moved_count=0 + + # 7天前: daily -> weekly + # 命名格式: {service}_{type}_{YYYYMMDD}_{HHMMSS}.{ext} + find "$BACKUP_BASE/daily" -type f -mtime +7 | while read -r file; do + service=$(basename "$(dirname "$file")") + + # 解析時間戳 + filename=$(basename "$file") + timestamp=$(echo "$filename" | grep -oP '\d{8}_\d{6}' || echo "") + + if [ -n "$timestamp" ]; then + year=${timestamp:0:4} + week=$(date -j -f "%Y%m%d_%H%M%S" "${timestamp}_0000" +%Y-W%V 2>/dev/null || echo "$year-W$(date +%V)") + else + week=$(date +%Y-W%V) + fi + + dest_dir="$BACKUP_BASE/weekly/$service/$week" + mkdir -p "$dest_dir" + + mv "$file" "$dest_dir/" 2>/dev/null && log "移動: $file -> $dest_dir" && moved_count=$((moved_count + 1)) + done + + # 30天前: weekly -> monthly + find "$BACKUP_BASE/weekly" -type f -mtime +30 | while read -r file; do + service=$(basename "$(dirname "$(dirname "$file")")") + month=$(date +%Y-%m) + + dest_dir="$BACKUP_BASE/monthly/$service/$month" + mkdir -p "$dest_dir" + + mv "$file" "$dest_dir/" 2>/dev/null && log "移動: $file -> $dest_dir" && moved_count=$((moved_count + 1)) + done + + # 90天前: monthly -> archive (長期歸檔) + find "$BACKUP_BASE/monthly" -type f -mtime +90 | while read -r file; do + service=$(basename "$(dirname "$(dirname "$file")")") + year=$(date +%Y) + + dest_dir="$BACKUP_BASE/archive/$service/$year" + mkdir -p "$dest_dir" + + mv "$file" "$dest_dir/" 2>/dev/null && log "歸檔: $file -> $dest_dir" && moved_count=$((moved_count + 1)) + done + + log "溫冷轉移完成: 移動了 $moved_count 個文件" } # 清理過期備份 cleanup_old() { - log "清理過期備份..." - - # 歸檔超過 365 天 - find "$BACKUP_BASE/archive" -type f -mtime +365 -delete 2>/dev/null - - # 每月備份保留 12 個月 - find "$BACKUP_BASE/monthly" -type f -mtime +365 -delete 2>/dev/null - - # 每週備份保留 12 週 - find "$BACKUP_BASE/weekly" -type f -mtime +84 -delete 2>/dev/null - - # 每日備份保留 30 天 - find "$BACKUP_BASE/daily" -type f -mtime +30 -delete 2>/dev/null - - log "清理完成" + log "清理過期備份..." + + # 歸檔超過 365 天 + find "$BACKUP_BASE/archive" -type f -mtime +365 -delete 2>/dev/null + + # 每月備份保留 12 個月 + find "$BACKUP_BASE/monthly" -type f -mtime +365 -delete 2>/dev/null + + # 每週備份保留 12 週 + find "$BACKUP_BASE/weekly" -type f -mtime +84 -delete 2>/dev/null + + # 每日備份保留 30 天 + find "$BACKUP_BASE/daily" -type f -mtime +30 -delete 2>/dev/null + + log "清理完成" } # 驗證備份完整性 verify_backup() { - local backup_file=$1 - - if [[ "$backup_file" == *.tar.gz ]]; then - tar -tzf "$backup_file" > /dev/null 2>&1 - return $? - elif [[ "$backup_file" == *.sql ]]; then - head -1 "$backup_file" | grep -q "SQL" && return 0 - return 1 - elif [[ "$backup_file" == *.rdb ]]; then - file "$backup_file" | grep -q "data" && return 0 - return 1 - fi - - return 0 + local backup_file=$1 + + if [[ "$backup_file" == *.tar.gz ]]; then + tar -tzf "$backup_file" >/dev/null 2>&1 + return $? + elif [[ "$backup_file" == *.sql ]]; then + head -1 "$backup_file" | grep -q "SQL" && return 0 + return 1 + elif [[ "$backup_file" == *.rdb ]]; then + file "$backup_file" | grep -q "data" && return 0 + return 1 + fi + + return 0 } # 生成備份報告 generate_report() { - local report_file="/Users/accusys/momentry/log/backup_report_$(date +%Y%m%d).txt" - - { - echo "========================================" - echo "Momentry 備份報告" - echo "生成時間: $(date)" - echo "========================================" - echo "" - - echo "## 備份狀態" - check_backup_status - - echo "" - echo "## 存儲使用趨勢 (最近30天)" - psql -U accusys -h localhost -d momentry -t -A -c " + local report_file + report_file="/Users/accusys/momentry/log/backup_report_$(date +%Y%m%d).txt" + + { + echo "========================================" + echo "Momentry 備份報告" + echo "生成時間: $(date)" + echo "========================================" + echo "" + + echo "## 備份狀態" + check_backup_status + + echo "" + echo "## 存儲使用趨勢 (最近30天)" + psql -U accusys -h localhost -d momentry -t -A -c " SELECT tier, COUNT(*) as files, AVG(total_size_bytes)::bigint as avg_size, @@ -309,67 +380,67 @@ generate_report() { GROUP BY tier ORDER BY tier; " 2>/dev/null || echo " (無數據)" - - echo "" - echo "## 建議" - - # 檢查是否有服務超過7天未備份 - for service in "${SERVICES[@]}"; do - latest=$(find "$BACKUP_BASE/daily/$service" -type f 2>/dev/null | head -1) - if [ -n "$latest" ]; then - days_old=$(($(date +%s) - $(stat -f "%m" "$latest" 2>/dev/null || echo "0")) / 86400) - if [ "$days_old" -gt 7 ]; then - echo " - ⚠️ $service 超過 $days_old 天未備份，建議立即執行備份" - fi - fi - done - - } > "$report_file" - - log "報告已生成: $report_file" - echo "$report_file" + + echo "" + echo "## 建議" + + # 檢查是否有服務超過7天未備份 + for service in "${SERVICES[@]}"; do + latest=$(find "$BACKUP_BASE/daily/$service" -type f 2>/dev/null | head -1) + if [ -n "$latest" ]; then + days_old=$((($(date +%s) - $(stat -f "%m" "$latest" 2>/dev/null || echo "0")) / 86400)) + if [ "$days_old" -gt 7 ]; then + echo " - ⚠️ $service 超過 $days_old 天未備份，建議立即執行備份" + fi + fi + done + + } >"$report_file" + + log "報告已生成: $report_file" + echo "$report_file" } # 主程序 command=${1:-status} case $command in - status) - check_backup_status - ;; - init) - init_backup_dirs - ;; - tier) - tier_backups - ;; - cleanup) - cleanup_old - ;; - verify) - verify_backup "${2:-}" - ;; - report) - generate_report - ;; - all) - log "執行完整備份維護..." - check_backup_status - tier_backups - cleanup_old - generate_report - log "備份維護完成" - ;; - *) - echo "用法: $0 {status|init|tier|cleanup|verify|report|all}" - echo "" - echo " status - 檢查備份狀態" - echo " init - 初始化備份目錄" - echo " tier - 執行溫冷轉移" - echo " cleanup - 清理過期備份" - echo " verify - 驗證備份完整性" - echo " report - 生成備份報告" - echo " all - 執行所有維護任務" - exit 1 - ;; +status) + check_backup_status + ;; +init) + init_backup_dirs + ;; +tier) + tier_backups + ;; +cleanup) + cleanup_old + ;; +verify) + verify_backup "${2:-}" + ;; +report) + generate_report + ;; +all) + log "執行完整備份維護..." + check_backup_status + tier_backups + cleanup_old + generate_report + log "備份維護完成" + ;; +*) + echo "用法: $0 {status|init|tier|cleanup|verify|report|all}" + echo "" + echo " status - 檢查備份狀態" + echo " init - 初始化備份目錄" + echo " tier - 執行溫冷轉移" + echo " cleanup - 清理過期備份" + echo " verify - 驗證備份完整性" + echo " report - 生成備份報告" + echo " all - 執行所有維護任務" + exit 1 + ;; esac