feat: update core API, database layer, and worker modules
- Remove unused imports (n8n_search, universal_search, Client, Arc, etc.) - Update API endpoints for identity, face recognition, search - Fix postgres_db.rs search_videos parent_uuid column - Add snapshot API and identity agent API - Clean up backup files (.bak, .bak2)
This commit is contained in:
@@ -1,7 +1,9 @@
|
||||
pub mod file_manager;
|
||||
pub mod output_dir;
|
||||
pub mod snapshot_manager;
|
||||
pub mod uuid;
|
||||
|
||||
pub use file_manager::FileManager;
|
||||
pub use output_dir::OutputDir;
|
||||
pub use snapshot_manager::SnapshotManager;
|
||||
pub use uuid::compute_uuid;
|
||||
|
||||
268
src/core/storage/snapshot_manager.rs
Normal file
268
src/core/storage/snapshot_manager.rs
Normal file
@@ -0,0 +1,268 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::core::config;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SnapshotTier {
|
||||
Hot,
|
||||
Warm,
|
||||
Cold,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SnapshotTier {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
SnapshotTier::Hot => write!(f, "hot"),
|
||||
SnapshotTier::Warm => write!(f, "warm"),
|
||||
SnapshotTier::Cold => write!(f, "cold"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct SnapshotStatus {
|
||||
pub file_uuid: String,
|
||||
pub tier: SnapshotTier,
|
||||
pub hits: u64,
|
||||
pub types: Vec<String>,
|
||||
pub generated_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SnapshotManager {
|
||||
base_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl SnapshotManager {
|
||||
pub fn new(user_dir: &str) -> Self {
|
||||
let snapshot_dir_name = config::snapshot::SNAPSHOT_DIR_NAME.as_str();
|
||||
let base_dir = Path::new(user_dir).join(snapshot_dir_name);
|
||||
Self { base_dir }
|
||||
}
|
||||
|
||||
pub fn base_dir(&self) -> &Path {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
pub fn file_snapshot_dir(&self, file_uuid: &str) -> PathBuf {
|
||||
self.base_dir.join(file_uuid)
|
||||
}
|
||||
|
||||
pub fn file_type_dir(&self, file_uuid: &str, snapshot_type: &str) -> PathBuf {
|
||||
self.base_dir.join(file_uuid).join(snapshot_type)
|
||||
}
|
||||
|
||||
pub fn identity_snapshot_dir(&self, identity_uuid: &str) -> PathBuf {
|
||||
self.base_dir.join("identities").join(identity_uuid)
|
||||
}
|
||||
|
||||
pub fn identity_face_dir(&self, identity_uuid: &str) -> PathBuf {
|
||||
self.base_dir
|
||||
.join("identities")
|
||||
.join(identity_uuid)
|
||||
.join("faces")
|
||||
}
|
||||
|
||||
pub fn ensure_file_dirs(&self, file_uuid: &str) -> std::io::Result<()> {
|
||||
let dir = self.file_snapshot_dir(file_uuid);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
for snap_type in ["faces", "logos", "products", "ocr"] {
|
||||
std::fs::create_dir_all(dir.join(snap_type))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn ensure_identity_dirs(&self, identity_uuid: &str) -> std::io::Result<()> {
|
||||
let dir = self.identity_snapshot_dir(identity_uuid);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
std::fs::create_dir_all(dir.join("faces"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compute_tier(hits: u64) -> SnapshotTier {
|
||||
let threshold = *config::snapshot::HOT_THRESHOLD;
|
||||
if hits >= threshold {
|
||||
SnapshotTier::Hot
|
||||
} else if hits > 0 {
|
||||
SnapshotTier::Warm
|
||||
} else {
|
||||
SnapshotTier::Cold
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tier_ttl(&self, tier: SnapshotTier) -> u64 {
|
||||
match tier {
|
||||
SnapshotTier::Hot => *config::snapshot::HOT_TTL_SECS,
|
||||
SnapshotTier::Warm => *config::snapshot::WARM_TTL_SECS,
|
||||
SnapshotTier::Cold => 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot_exists(&self, file_uuid: &str, snapshot_type: &str) -> bool {
|
||||
self.file_type_dir(file_uuid, snapshot_type).exists()
|
||||
}
|
||||
|
||||
pub fn list_snapshot_types(&self, file_uuid: &str) -> Vec<String> {
|
||||
let dir = self.file_snapshot_dir(file_uuid);
|
||||
if !dir.exists() {
|
||||
return Vec::new();
|
||||
}
|
||||
std::fs::read_dir(&dir)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.path().is_dir())
|
||||
.filter_map(|e| e.file_name().to_str().map(String::from))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn remove_file_snapshots(&self, file_uuid: &str) -> std::io::Result<()> {
|
||||
let dir = self.file_snapshot_dir(file_uuid);
|
||||
if dir.exists() {
|
||||
std::fs::remove_dir_all(&dir)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_identity_snapshots(&self, identity_uuid: &str) -> std::io::Result<()> {
|
||||
let dir = self.identity_snapshot_dir(identity_uuid);
|
||||
if dir.exists() {
|
||||
std::fs::remove_dir_all(&dir)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn create_test_manager() -> (SnapshotManager, tempfile::TempDir) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let manager = SnapshotManager::new(temp_dir.path().to_str().unwrap());
|
||||
(manager, temp_dir)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_tier_hot() {
|
||||
assert_eq!(SnapshotManager::compute_tier(5), SnapshotTier::Hot);
|
||||
assert_eq!(SnapshotManager::compute_tier(10), SnapshotTier::Hot);
|
||||
assert_eq!(SnapshotManager::compute_tier(100), SnapshotTier::Hot);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_tier_warm() {
|
||||
assert_eq!(SnapshotManager::compute_tier(1), SnapshotTier::Warm);
|
||||
assert_eq!(SnapshotManager::compute_tier(4), SnapshotTier::Warm);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_tier_cold() {
|
||||
assert_eq!(SnapshotManager::compute_tier(0), SnapshotTier::Cold);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tier_display() {
|
||||
assert_eq!(SnapshotTier::Hot.to_string(), "hot");
|
||||
assert_eq!(SnapshotTier::Warm.to_string(), "warm");
|
||||
assert_eq!(SnapshotTier::Cold.to_string(), "cold");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ensure_file_dirs_creates_structure() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let file_uuid = "test_file_123";
|
||||
|
||||
manager.ensure_file_dirs(file_uuid).unwrap();
|
||||
|
||||
assert!(manager.file_snapshot_dir(file_uuid).exists());
|
||||
assert!(manager.file_type_dir(file_uuid, "faces").exists());
|
||||
assert!(manager.file_type_dir(file_uuid, "logos").exists());
|
||||
assert!(manager.file_type_dir(file_uuid, "products").exists());
|
||||
assert!(manager.file_type_dir(file_uuid, "ocr").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ensure_identity_dirs_creates_structure() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let identity_uuid = "test_identity_456";
|
||||
|
||||
manager.ensure_identity_dirs(identity_uuid).unwrap();
|
||||
|
||||
assert!(manager.identity_snapshot_dir(identity_uuid).exists());
|
||||
assert!(manager.identity_face_dir(identity_uuid).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_snapshot_types_empty() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let types = manager.list_snapshot_types("nonexistent");
|
||||
assert!(types.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_snapshot_types_after_creation() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let file_uuid = "test_file_789";
|
||||
|
||||
manager.ensure_file_dirs(file_uuid).unwrap();
|
||||
let types = manager.list_snapshot_types(file_uuid);
|
||||
|
||||
assert_eq!(types.len(), 4);
|
||||
assert!(types.contains(&"faces".to_string()));
|
||||
assert!(types.contains(&"logos".to_string()));
|
||||
assert!(types.contains(&"products".to_string()));
|
||||
assert!(types.contains(&"ocr".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_file_snapshots() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let file_uuid = "test_file_remove";
|
||||
|
||||
manager.ensure_file_dirs(file_uuid).unwrap();
|
||||
assert!(manager.file_snapshot_dir(file_uuid).exists());
|
||||
|
||||
manager.remove_file_snapshots(file_uuid).unwrap();
|
||||
assert!(!manager.file_snapshot_dir(file_uuid).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_identity_snapshots() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let identity_uuid = "test_identity_remove";
|
||||
|
||||
manager.ensure_identity_dirs(identity_uuid).unwrap();
|
||||
assert!(manager.identity_snapshot_dir(identity_uuid).exists());
|
||||
|
||||
manager.remove_identity_snapshots(identity_uuid).unwrap();
|
||||
assert!(!manager.identity_snapshot_dir(identity_uuid).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snapshot_exists() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
let file_uuid = "test_exists";
|
||||
|
||||
assert!(!manager.snapshot_exists(file_uuid, "faces"));
|
||||
|
||||
manager.ensure_file_dirs(file_uuid).unwrap();
|
||||
assert!(manager.snapshot_exists(file_uuid, "faces"));
|
||||
assert!(!manager.snapshot_exists(file_uuid, "nonexistent"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tier_ttl() {
|
||||
let (manager, _temp) = create_test_manager();
|
||||
|
||||
let hot_ttl = manager.tier_ttl(SnapshotTier::Hot);
|
||||
assert_eq!(hot_ttl, *config::snapshot::HOT_TTL_SECS);
|
||||
|
||||
let warm_ttl = manager.tier_ttl(SnapshotTier::Warm);
|
||||
assert_eq!(warm_ttl, *config::snapshot::WARM_TTL_SECS);
|
||||
|
||||
let cold_ttl = manager.tier_ttl(SnapshotTier::Cold);
|
||||
assert_eq!(cold_ttl, 0);
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,12 @@ use sha2::{Digest, Sha256};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Compute UUID from file path using SHA256
|
||||
/// UUID = SHA256(user_path + filename)[0:16]
|
||||
/// UUID = SHA256(user_path + filename)[0:32]
|
||||
pub fn compute_uuid(user_path: &str, filename: &str) -> String {
|
||||
let key = format!("{}/{}", user_path.trim_end_matches('/'), filename);
|
||||
let hash = Sha256::digest(key.as_bytes());
|
||||
let hash_str = hex::encode(hash);
|
||||
hash_str[0..16].to_string()
|
||||
hash_str[0..32].to_string()
|
||||
}
|
||||
|
||||
/// Compute UUID from full file path
|
||||
@@ -29,19 +29,16 @@ pub fn compute_uuid_from_path(full_path: &str) -> String {
|
||||
/// Input: ./demo/video.mp4 or ./demo/path/to/video.mp4
|
||||
/// Returns: (username, filepath) e.g., ("demo", "video.mp4") or ("demo", "path/to/video.mp4")
|
||||
pub fn extract_user_from_relative_path(relative_path: &str) -> (String, String) {
|
||||
// Remove leading ./
|
||||
let path = relative_path.strip_prefix("./").unwrap_or(relative_path);
|
||||
|
||||
let path_buf = PathBuf::from(path);
|
||||
|
||||
// First component is username
|
||||
let mut components = path_buf.components();
|
||||
let username = components
|
||||
.next()
|
||||
.map(|c| c.as_os_str().to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
// Remaining path (filepath)
|
||||
let filepath: String = components
|
||||
.map(|c| c.as_os_str().to_string_lossy().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
@@ -57,6 +54,62 @@ pub fn compute_uuid_from_relative_path(relative_path: &str) -> String {
|
||||
compute_uuid(&username, &filepath)
|
||||
}
|
||||
|
||||
/// Get MAC address of primary network interface
|
||||
/// Returns MAC address in format: a1:b2:c3:d4:e5:f6
|
||||
pub fn get_mac_address() -> String {
|
||||
use mac_address::get_mac_address;
|
||||
|
||||
match get_mac_address() {
|
||||
Ok(Some(mac)) => {
|
||||
let bytes = mac.bytes();
|
||||
format!(
|
||||
"{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
|
||||
bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5]
|
||||
)
|
||||
}
|
||||
Ok(None) => "00:00:00:00:00:00".to_string(),
|
||||
Err(_) => "00:00:00:00:00:00".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute Birth UUID (Stable Identity with Location)
|
||||
/// UUID = SHA256(mac_address|birthday|path|filename)[0:32]
|
||||
///
|
||||
/// This UUID format ensures:
|
||||
/// - Location Encoding: Path is part of the identity (like location code in ID card).
|
||||
/// - Stability: Uses the original Birthday, not the current timestamp.
|
||||
/// - Uniqueness: Different MAC, Birthday, Path, or Filename produces different UUIDs.
|
||||
pub fn compute_birth_uuid(
|
||||
mac_address: &str,
|
||||
birthday: &str, // Fixed timestamp of original registration
|
||||
path: &str, // Canonical file path (Location)
|
||||
filename: &str,
|
||||
) -> String {
|
||||
let key = format!(
|
||||
"{}|{}|{}|{}",
|
||||
mac_address,
|
||||
birthday,
|
||||
path.trim_end_matches('/'),
|
||||
filename
|
||||
);
|
||||
let hash = Sha256::digest(key.as_bytes());
|
||||
hex::encode(hash)[0..32].to_string()
|
||||
}
|
||||
|
||||
/// Check if UUID is Birth UUID format (32 characters)
|
||||
pub fn is_birth_uuid(uuid: &str) -> bool {
|
||||
uuid.len() == 32 && !uuid.contains('_')
|
||||
}
|
||||
|
||||
/// Extract username from sftpgo user home path
|
||||
/// Input: ./demo/video.mp4 or /Users/.../demo/video.mp4
|
||||
/// Returns: username (e.g., "demo")
|
||||
pub fn extract_username_from_path(path: &str) -> String {
|
||||
let relative = path.strip_prefix("./").unwrap_or(path);
|
||||
let parts: Vec<&str> = relative.split('/').collect();
|
||||
parts.first().copied().unwrap_or("demo").to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -64,14 +117,14 @@ mod tests {
|
||||
#[test]
|
||||
fn test_uuid_computation() {
|
||||
let uuid = compute_uuid("/Users/test/Videos", "video.mp4");
|
||||
assert_eq!(uuid.len(), 16);
|
||||
assert_eq!(uuid.len(), 32);
|
||||
println!("UUID: {}", uuid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_uuid_from_path() {
|
||||
let uuid = compute_uuid_from_path("/Users/test/Videos/video.mp4");
|
||||
assert_eq!(uuid.len(), 16);
|
||||
assert_eq!(uuid.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -90,11 +143,102 @@ mod tests {
|
||||
let uuid1 = compute_uuid_from_relative_path("./demo/video.mp4");
|
||||
let uuid2 = compute_uuid_from_relative_path("./demo/video.mp4");
|
||||
assert_eq!(uuid1, uuid2);
|
||||
assert_eq!(uuid1.len(), 16);
|
||||
assert_eq!(uuid1.len(), 32);
|
||||
|
||||
// Different users with same filename should have different UUIDs
|
||||
let uuid_demo = compute_uuid_from_relative_path("./demo/video.mp4");
|
||||
let uuid_warren = compute_uuid_from_relative_path("./warren/video.mp4");
|
||||
assert_ne!(uuid_demo, uuid_warren);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_mac_address() {
|
||||
let mac = get_mac_address();
|
||||
assert_eq!(mac.len(), 17); // a1:b2:c3:d4:e5:f6
|
||||
assert!(mac.contains(':'));
|
||||
println!("MAC Address: {}", mac);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_birth_uuid_generation() {
|
||||
let uuid = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T22:00:00+08:00",
|
||||
"/Users/test/Videos",
|
||||
"video.mp4",
|
||||
);
|
||||
assert_eq!(uuid.len(), 32);
|
||||
println!("Birth UUID: {}", uuid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_birth_uuid_different_mac() {
|
||||
let uuid1 = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/test/Videos",
|
||||
"video.mp4",
|
||||
);
|
||||
let uuid2 = compute_birth_uuid(
|
||||
"d4:e5:f6:a1:b2:c3",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/test/Videos",
|
||||
"video.mp4",
|
||||
);
|
||||
assert_ne!(uuid1, uuid2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_birth_uuid_different_path() {
|
||||
// Moving file to different path creates new identity
|
||||
let uuid1 = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/test/Videos",
|
||||
"video.mp4",
|
||||
);
|
||||
let uuid2 = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/test/Archive",
|
||||
"video.mp4",
|
||||
);
|
||||
assert_ne!(uuid1, uuid2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_birth_uuid_same_elements() {
|
||||
let uuid1 = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/test/Videos",
|
||||
"video.mp4",
|
||||
);
|
||||
let uuid2 = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/test/Videos",
|
||||
"video.mp4",
|
||||
);
|
||||
assert_eq!(uuid1, uuid2); // Same elements = same UUID
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_birth_uuid() {
|
||||
let birth_uuid = compute_birth_uuid(
|
||||
"a1:b2:c3:d4:e5:f6",
|
||||
"2026-04-27T10:00:00",
|
||||
"/Users/demo",
|
||||
"video.mp4",
|
||||
);
|
||||
assert!(is_birth_uuid(&birth_uuid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_username_from_path() {
|
||||
let username = extract_username_from_path("./demo/video.mp4");
|
||||
assert_eq!(username, "demo");
|
||||
|
||||
let username = extract_username_from_path("./warren/path/to/video.mp4");
|
||||
assert_eq!(username, "warren");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user