feat: update core API, database layer, and worker modules

- Remove unused imports (n8n_search, universal_search, Client, Arc, etc.)
- Update API endpoints for identity, face recognition, search
- Fix postgres_db.rs search_videos parent_uuid column
- Add snapshot API and identity agent API
- Clean up backup files (.bak, .bak2)
This commit is contained in:
Warren
2026-04-30 15:07:02 +08:00
parent 8f2208dd63
commit 2b23d1cfbd
148 changed files with 8553 additions and 48637 deletions

View File

@@ -1,7 +1,9 @@
pub mod file_manager;
pub mod output_dir;
pub mod snapshot_manager;
pub mod uuid;
pub use file_manager::FileManager;
pub use output_dir::OutputDir;
pub use snapshot_manager::SnapshotManager;
pub use uuid::compute_uuid;

View File

@@ -0,0 +1,268 @@
use std::path::{Path, PathBuf};
use crate::core::config;
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SnapshotTier {
Hot,
Warm,
Cold,
}
impl std::fmt::Display for SnapshotTier {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SnapshotTier::Hot => write!(f, "hot"),
SnapshotTier::Warm => write!(f, "warm"),
SnapshotTier::Cold => write!(f, "cold"),
}
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SnapshotStatus {
pub file_uuid: String,
pub tier: SnapshotTier,
pub hits: u64,
pub types: Vec<String>,
pub generated_at: Option<String>,
}
#[derive(Debug, Clone)]
pub struct SnapshotManager {
base_dir: PathBuf,
}
impl SnapshotManager {
pub fn new(user_dir: &str) -> Self {
let snapshot_dir_name = config::snapshot::SNAPSHOT_DIR_NAME.as_str();
let base_dir = Path::new(user_dir).join(snapshot_dir_name);
Self { base_dir }
}
pub fn base_dir(&self) -> &Path {
&self.base_dir
}
pub fn file_snapshot_dir(&self, file_uuid: &str) -> PathBuf {
self.base_dir.join(file_uuid)
}
pub fn file_type_dir(&self, file_uuid: &str, snapshot_type: &str) -> PathBuf {
self.base_dir.join(file_uuid).join(snapshot_type)
}
pub fn identity_snapshot_dir(&self, identity_uuid: &str) -> PathBuf {
self.base_dir.join("identities").join(identity_uuid)
}
pub fn identity_face_dir(&self, identity_uuid: &str) -> PathBuf {
self.base_dir
.join("identities")
.join(identity_uuid)
.join("faces")
}
pub fn ensure_file_dirs(&self, file_uuid: &str) -> std::io::Result<()> {
let dir = self.file_snapshot_dir(file_uuid);
std::fs::create_dir_all(&dir)?;
for snap_type in ["faces", "logos", "products", "ocr"] {
std::fs::create_dir_all(dir.join(snap_type))?;
}
Ok(())
}
pub fn ensure_identity_dirs(&self, identity_uuid: &str) -> std::io::Result<()> {
let dir = self.identity_snapshot_dir(identity_uuid);
std::fs::create_dir_all(&dir)?;
std::fs::create_dir_all(dir.join("faces"))?;
Ok(())
}
pub fn compute_tier(hits: u64) -> SnapshotTier {
let threshold = *config::snapshot::HOT_THRESHOLD;
if hits >= threshold {
SnapshotTier::Hot
} else if hits > 0 {
SnapshotTier::Warm
} else {
SnapshotTier::Cold
}
}
pub fn tier_ttl(&self, tier: SnapshotTier) -> u64 {
match tier {
SnapshotTier::Hot => *config::snapshot::HOT_TTL_SECS,
SnapshotTier::Warm => *config::snapshot::WARM_TTL_SECS,
SnapshotTier::Cold => 0,
}
}
pub fn snapshot_exists(&self, file_uuid: &str, snapshot_type: &str) -> bool {
self.file_type_dir(file_uuid, snapshot_type).exists()
}
pub fn list_snapshot_types(&self, file_uuid: &str) -> Vec<String> {
let dir = self.file_snapshot_dir(file_uuid);
if !dir.exists() {
return Vec::new();
}
std::fs::read_dir(&dir)
.into_iter()
.flatten()
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.filter_map(|e| e.file_name().to_str().map(String::from))
.collect()
}
pub fn remove_file_snapshots(&self, file_uuid: &str) -> std::io::Result<()> {
let dir = self.file_snapshot_dir(file_uuid);
if dir.exists() {
std::fs::remove_dir_all(&dir)?;
}
Ok(())
}
pub fn remove_identity_snapshots(&self, identity_uuid: &str) -> std::io::Result<()> {
let dir = self.identity_snapshot_dir(identity_uuid);
if dir.exists() {
std::fs::remove_dir_all(&dir)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn create_test_manager() -> (SnapshotManager, tempfile::TempDir) {
let temp_dir = tempfile::tempdir().unwrap();
let manager = SnapshotManager::new(temp_dir.path().to_str().unwrap());
(manager, temp_dir)
}
#[test]
fn test_compute_tier_hot() {
assert_eq!(SnapshotManager::compute_tier(5), SnapshotTier::Hot);
assert_eq!(SnapshotManager::compute_tier(10), SnapshotTier::Hot);
assert_eq!(SnapshotManager::compute_tier(100), SnapshotTier::Hot);
}
#[test]
fn test_compute_tier_warm() {
assert_eq!(SnapshotManager::compute_tier(1), SnapshotTier::Warm);
assert_eq!(SnapshotManager::compute_tier(4), SnapshotTier::Warm);
}
#[test]
fn test_compute_tier_cold() {
assert_eq!(SnapshotManager::compute_tier(0), SnapshotTier::Cold);
}
#[test]
fn test_tier_display() {
assert_eq!(SnapshotTier::Hot.to_string(), "hot");
assert_eq!(SnapshotTier::Warm.to_string(), "warm");
assert_eq!(SnapshotTier::Cold.to_string(), "cold");
}
#[test]
fn test_ensure_file_dirs_creates_structure() {
let (manager, _temp) = create_test_manager();
let file_uuid = "test_file_123";
manager.ensure_file_dirs(file_uuid).unwrap();
assert!(manager.file_snapshot_dir(file_uuid).exists());
assert!(manager.file_type_dir(file_uuid, "faces").exists());
assert!(manager.file_type_dir(file_uuid, "logos").exists());
assert!(manager.file_type_dir(file_uuid, "products").exists());
assert!(manager.file_type_dir(file_uuid, "ocr").exists());
}
#[test]
fn test_ensure_identity_dirs_creates_structure() {
let (manager, _temp) = create_test_manager();
let identity_uuid = "test_identity_456";
manager.ensure_identity_dirs(identity_uuid).unwrap();
assert!(manager.identity_snapshot_dir(identity_uuid).exists());
assert!(manager.identity_face_dir(identity_uuid).exists());
}
#[test]
fn test_list_snapshot_types_empty() {
let (manager, _temp) = create_test_manager();
let types = manager.list_snapshot_types("nonexistent");
assert!(types.is_empty());
}
#[test]
fn test_list_snapshot_types_after_creation() {
let (manager, _temp) = create_test_manager();
let file_uuid = "test_file_789";
manager.ensure_file_dirs(file_uuid).unwrap();
let types = manager.list_snapshot_types(file_uuid);
assert_eq!(types.len(), 4);
assert!(types.contains(&"faces".to_string()));
assert!(types.contains(&"logos".to_string()));
assert!(types.contains(&"products".to_string()));
assert!(types.contains(&"ocr".to_string()));
}
#[test]
fn test_remove_file_snapshots() {
let (manager, _temp) = create_test_manager();
let file_uuid = "test_file_remove";
manager.ensure_file_dirs(file_uuid).unwrap();
assert!(manager.file_snapshot_dir(file_uuid).exists());
manager.remove_file_snapshots(file_uuid).unwrap();
assert!(!manager.file_snapshot_dir(file_uuid).exists());
}
#[test]
fn test_remove_identity_snapshots() {
let (manager, _temp) = create_test_manager();
let identity_uuid = "test_identity_remove";
manager.ensure_identity_dirs(identity_uuid).unwrap();
assert!(manager.identity_snapshot_dir(identity_uuid).exists());
manager.remove_identity_snapshots(identity_uuid).unwrap();
assert!(!manager.identity_snapshot_dir(identity_uuid).exists());
}
#[test]
fn test_snapshot_exists() {
let (manager, _temp) = create_test_manager();
let file_uuid = "test_exists";
assert!(!manager.snapshot_exists(file_uuid, "faces"));
manager.ensure_file_dirs(file_uuid).unwrap();
assert!(manager.snapshot_exists(file_uuid, "faces"));
assert!(!manager.snapshot_exists(file_uuid, "nonexistent"));
}
#[test]
fn test_tier_ttl() {
let (manager, _temp) = create_test_manager();
let hot_ttl = manager.tier_ttl(SnapshotTier::Hot);
assert_eq!(hot_ttl, *config::snapshot::HOT_TTL_SECS);
let warm_ttl = manager.tier_ttl(SnapshotTier::Warm);
assert_eq!(warm_ttl, *config::snapshot::WARM_TTL_SECS);
let cold_ttl = manager.tier_ttl(SnapshotTier::Cold);
assert_eq!(cold_ttl, 0);
}
}

View File

@@ -2,12 +2,12 @@ use sha2::{Digest, Sha256};
use std::path::PathBuf;
/// Compute UUID from file path using SHA256
/// UUID = SHA256(user_path + filename)[0:16]
/// UUID = SHA256(user_path + filename)[0:32]
pub fn compute_uuid(user_path: &str, filename: &str) -> String {
let key = format!("{}/{}", user_path.trim_end_matches('/'), filename);
let hash = Sha256::digest(key.as_bytes());
let hash_str = hex::encode(hash);
hash_str[0..16].to_string()
hash_str[0..32].to_string()
}
/// Compute UUID from full file path
@@ -29,19 +29,16 @@ pub fn compute_uuid_from_path(full_path: &str) -> String {
/// Input: ./demo/video.mp4 or ./demo/path/to/video.mp4
/// Returns: (username, filepath) e.g., ("demo", "video.mp4") or ("demo", "path/to/video.mp4")
pub fn extract_user_from_relative_path(relative_path: &str) -> (String, String) {
// Remove leading ./
let path = relative_path.strip_prefix("./").unwrap_or(relative_path);
let path_buf = PathBuf::from(path);
// First component is username
let mut components = path_buf.components();
let username = components
.next()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.unwrap_or_default();
// Remaining path (filepath)
let filepath: String = components
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>()
@@ -57,6 +54,62 @@ pub fn compute_uuid_from_relative_path(relative_path: &str) -> String {
compute_uuid(&username, &filepath)
}
/// Get MAC address of primary network interface
/// Returns MAC address in format: a1:b2:c3:d4:e5:f6
pub fn get_mac_address() -> String {
use mac_address::get_mac_address;
match get_mac_address() {
Ok(Some(mac)) => {
let bytes = mac.bytes();
format!(
"{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5]
)
}
Ok(None) => "00:00:00:00:00:00".to_string(),
Err(_) => "00:00:00:00:00:00".to_string(),
}
}
/// Compute Birth UUID (Stable Identity with Location)
/// UUID = SHA256(mac_address|birthday|path|filename)[0:32]
///
/// This UUID format ensures:
/// - Location Encoding: Path is part of the identity (like location code in ID card).
/// - Stability: Uses the original Birthday, not the current timestamp.
/// - Uniqueness: Different MAC, Birthday, Path, or Filename produces different UUIDs.
pub fn compute_birth_uuid(
mac_address: &str,
birthday: &str, // Fixed timestamp of original registration
path: &str, // Canonical file path (Location)
filename: &str,
) -> String {
let key = format!(
"{}|{}|{}|{}",
mac_address,
birthday,
path.trim_end_matches('/'),
filename
);
let hash = Sha256::digest(key.as_bytes());
hex::encode(hash)[0..32].to_string()
}
/// Check if UUID is Birth UUID format (32 characters)
pub fn is_birth_uuid(uuid: &str) -> bool {
uuid.len() == 32 && !uuid.contains('_')
}
/// Extract username from sftpgo user home path
/// Input: ./demo/video.mp4 or /Users/.../demo/video.mp4
/// Returns: username (e.g., "demo")
pub fn extract_username_from_path(path: &str) -> String {
let relative = path.strip_prefix("./").unwrap_or(path);
let parts: Vec<&str> = relative.split('/').collect();
parts.first().copied().unwrap_or("demo").to_string()
}
#[cfg(test)]
mod tests {
use super::*;
@@ -64,14 +117,14 @@ mod tests {
#[test]
fn test_uuid_computation() {
let uuid = compute_uuid("/Users/test/Videos", "video.mp4");
assert_eq!(uuid.len(), 16);
assert_eq!(uuid.len(), 32);
println!("UUID: {}", uuid);
}
#[test]
fn test_uuid_from_path() {
let uuid = compute_uuid_from_path("/Users/test/Videos/video.mp4");
assert_eq!(uuid.len(), 16);
assert_eq!(uuid.len(), 32);
}
#[test]
@@ -90,11 +143,102 @@ mod tests {
let uuid1 = compute_uuid_from_relative_path("./demo/video.mp4");
let uuid2 = compute_uuid_from_relative_path("./demo/video.mp4");
assert_eq!(uuid1, uuid2);
assert_eq!(uuid1.len(), 16);
assert_eq!(uuid1.len(), 32);
// Different users with same filename should have different UUIDs
let uuid_demo = compute_uuid_from_relative_path("./demo/video.mp4");
let uuid_warren = compute_uuid_from_relative_path("./warren/video.mp4");
assert_ne!(uuid_demo, uuid_warren);
}
#[test]
fn test_get_mac_address() {
let mac = get_mac_address();
assert_eq!(mac.len(), 17); // a1:b2:c3:d4:e5:f6
assert!(mac.contains(':'));
println!("MAC Address: {}", mac);
}
#[test]
fn test_birth_uuid_generation() {
let uuid = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T22:00:00+08:00",
"/Users/test/Videos",
"video.mp4",
);
assert_eq!(uuid.len(), 32);
println!("Birth UUID: {}", uuid);
}
#[test]
fn test_birth_uuid_different_mac() {
let uuid1 = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T10:00:00",
"/Users/test/Videos",
"video.mp4",
);
let uuid2 = compute_birth_uuid(
"d4:e5:f6:a1:b2:c3",
"2026-04-27T10:00:00",
"/Users/test/Videos",
"video.mp4",
);
assert_ne!(uuid1, uuid2);
}
#[test]
fn test_birth_uuid_different_path() {
// Moving file to different path creates new identity
let uuid1 = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T10:00:00",
"/Users/test/Videos",
"video.mp4",
);
let uuid2 = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T10:00:00",
"/Users/test/Archive",
"video.mp4",
);
assert_ne!(uuid1, uuid2);
}
#[test]
fn test_birth_uuid_same_elements() {
let uuid1 = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T10:00:00",
"/Users/test/Videos",
"video.mp4",
);
let uuid2 = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T10:00:00",
"/Users/test/Videos",
"video.mp4",
);
assert_eq!(uuid1, uuid2); // Same elements = same UUID
}
#[test]
fn test_is_birth_uuid() {
let birth_uuid = compute_birth_uuid(
"a1:b2:c3:d4:e5:f6",
"2026-04-27T10:00:00",
"/Users/demo",
"video.mp4",
);
assert!(is_birth_uuid(&birth_uuid));
}
#[test]
fn test_extract_username_from_path() {
let username = extract_username_from_path("./demo/video.mp4");
assert_eq!(username, "demo");
let username = extract_username_from_path("./warren/path/to/video.mp4");
assert_eq!(username, "warren");
}
}