feat: Initial v0.9 release with API Key authentication
## v0.9.20260325_144654 ### Features - API Key Authentication System - Job Worker System - V2 Backup Versioning ### Bug Fixes - get_processor_results_by_job column mapping Co-authored-by: OpenCode
This commit is contained in:
accusys
@@ -1,5 +1,7 @@
|
||||
pub mod file_manager;
|
||||
pub mod output_dir;
|
||||
pub mod uuid;
|
||||
|
||||
pub use file_manager::FileManager;
|
||||
pub use output_dir::OutputDir;
|
||||
pub use uuid::compute_uuid;
|
||||
|
||||
226
src/core/storage/output_dir.rs
Normal file
226
src/core/storage/output_dir.rs
Normal file
@@ -0,0 +1,226 @@
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::{DateTime, Datelike, Local, Timelike};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct OutputDir {
|
||||
base_path: PathBuf,
|
||||
backup_enabled: bool,
|
||||
backup_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl OutputDir {
|
||||
pub fn new() -> Self {
|
||||
let base_path = std::env::var("MOMENTRY_OUTPUT_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from("./output"));
|
||||
|
||||
let backup_enabled = std::env::var("MOMENTRY_BACKUP_ENABLED")
|
||||
.map(|v| v == "true")
|
||||
.unwrap_or(false);
|
||||
|
||||
let backup_dir = std::env::var("MOMENTRY_BACKUP_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from("/Users/accusys/momentry/backup/momentry"));
|
||||
|
||||
Self {
|
||||
base_path,
|
||||
backup_enabled,
|
||||
backup_dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_base_path(&self) -> &Path {
|
||||
&self.base_path
|
||||
}
|
||||
|
||||
pub fn get_backup_dir(&self) -> &Path {
|
||||
&self.backup_dir
|
||||
}
|
||||
|
||||
pub fn ensure_dir(&self) -> Result<()> {
|
||||
std::fs::create_dir_all(&self.base_path).context(format!(
|
||||
"Failed to create output directory: {:?}",
|
||||
self.base_path
|
||||
))?;
|
||||
|
||||
if self.backup_enabled {
|
||||
std::fs::create_dir_all(&self.backup_dir).context(format!(
|
||||
"Failed to create backup directory: {:?}",
|
||||
self.backup_dir
|
||||
))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_output_path(&self, uuid: &str, extension: &str) -> PathBuf {
|
||||
self.base_path.join(format!("{}.{}", uuid, extension))
|
||||
}
|
||||
|
||||
fn get_timestamp() -> String {
|
||||
let now = Local::now();
|
||||
format!(
|
||||
"{:04}{:02}{:02}_{:02}{:02}{:02}",
|
||||
now.year(),
|
||||
now.month(),
|
||||
now.day(),
|
||||
now.hour(),
|
||||
now.minute(),
|
||||
now.second()
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_backup_path(&self, uuid: &str, extension: &str) -> Option<PathBuf> {
|
||||
if !self.backup_enabled {
|
||||
return None;
|
||||
}
|
||||
|
||||
let timestamp = Self::get_timestamp();
|
||||
let filename = format!("momentry_data_{}_{}.{}", timestamp, uuid, extension);
|
||||
|
||||
Some(self.backup_dir.join(filename))
|
||||
}
|
||||
|
||||
pub fn backup_file(&self, uuid: &str, extension: &str) -> Result<Option<PathBuf>> {
|
||||
if !self.backup_enabled {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let source = self.get_output_path(uuid, extension);
|
||||
if !source.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let backup_path = match self.get_backup_path(uuid, extension) {
|
||||
Some(path) => path,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
std::fs::copy(&source, &backup_path)
|
||||
.context(format!("Failed to backup file to: {:?}", backup_path))?;
|
||||
|
||||
let sha256_path = backup_path.with_extension(format!("{}.sha256", extension));
|
||||
|
||||
let source_content = std::fs::read(&source)?;
|
||||
let hash = format!("{:x}", md5::compute(&source_content));
|
||||
std::fs::write(
|
||||
&sha256_path,
|
||||
format!(
|
||||
"{} {}\n",
|
||||
hash,
|
||||
backup_path.file_name().unwrap().to_string_lossy()
|
||||
),
|
||||
)?;
|
||||
|
||||
Ok(Some(backup_path))
|
||||
}
|
||||
|
||||
pub fn cleanup_old_backups(&self, days: u32) -> Result<u32> {
|
||||
if !self.backup_enabled {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
if !self.backup_dir.exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let cutoff = Local::now() - chrono::Duration::days(days as i64);
|
||||
let mut deleted_count = 0;
|
||||
|
||||
for entry in std::fs::read_dir(&self.backup_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(name) = path.file_name() {
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.starts_with("momentry_data_") && name_str.len() == 43 {
|
||||
let date_part = &name_str[14..22];
|
||||
if let Ok(date) =
|
||||
DateTime::parse_from_str(&format!("{} 000000", date_part), "%Y%m%d %H%M%S")
|
||||
{
|
||||
if date.with_timezone(&Local) < cutoff {
|
||||
std::fs::remove_file(&path)?;
|
||||
deleted_count += 1;
|
||||
|
||||
let sha256_path = path.with_extension("sha256");
|
||||
if sha256_path.exists() {
|
||||
let _ = std::fs::remove_file(sha256_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(deleted_count)
|
||||
}
|
||||
|
||||
pub fn list_backups(&self) -> Result<Vec<BackupInfo>> {
|
||||
if !self.backup_dir.exists() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let mut backups = Vec::new();
|
||||
|
||||
for entry in std::fs::read_dir(&self.backup_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(name) = path.file_name() {
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.starts_with("momentry_data_") && name_str.ends_with(".sha256") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if name_str.starts_with("momentry_data_") {
|
||||
let date_part = &name_str[14..22];
|
||||
backups.push(BackupInfo {
|
||||
filename: name_str.to_string(),
|
||||
date: date_part.to_string(),
|
||||
path: path.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
backups.sort_by(|a, b| b.date.cmp(&a.date));
|
||||
Ok(backups)
|
||||
}
|
||||
|
||||
pub fn verify_backup(&self, backup_path: &Path) -> Result<bool> {
|
||||
let sha256_path = backup_path.with_extension("sha256");
|
||||
|
||||
if !sha256_path.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let sha256_content = std::fs::read_to_string(&sha256_path)?;
|
||||
let expected_hash = sha256_content.split_whitespace().next().unwrap_or("");
|
||||
|
||||
let source_content = std::fs::read(backup_path)?;
|
||||
let actual_hash = format!("{:x}", md5::compute(&source_content));
|
||||
|
||||
Ok(expected_hash == actual_hash)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for OutputDir {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackupInfo {
|
||||
pub filename: String,
|
||||
pub date: String,
|
||||
pub path: PathBuf,
|
||||
}
|
||||
@@ -25,39 +25,36 @@ pub fn compute_uuid_from_path(full_path: &str) -> String {
|
||||
compute_uuid(&parent, &filename)
|
||||
}
|
||||
|
||||
/// Extract relative path from full path given data root
|
||||
/// Returns (relative_path, filename)
|
||||
pub fn extract_relative_path(full_path: &str, data_root: &str) -> (String, String) {
|
||||
let full_path = PathBuf::from(full_path);
|
||||
let data_root = PathBuf::from(data_root);
|
||||
/// Extract username and filepath from relative path
|
||||
/// Input: ./demo/video.mp4 or ./demo/path/to/video.mp4
|
||||
/// Returns: (username, filepath) e.g., ("demo", "video.mp4") or ("demo", "path/to/video.mp4")
|
||||
pub fn extract_user_from_relative_path(relative_path: &str) -> (String, String) {
|
||||
// Remove leading ./
|
||||
let path = relative_path.strip_prefix("./").unwrap_or(relative_path);
|
||||
|
||||
// Canonicalize both paths
|
||||
let full_canonical = full_path.canonicalize().unwrap_or(full_path.clone());
|
||||
let root_canonical = data_root.canonicalize().unwrap_or(data_root.clone());
|
||||
let path_buf = PathBuf::from(path);
|
||||
|
||||
// Try to strip the data root prefix
|
||||
let relative = full_canonical
|
||||
.strip_prefix(&root_canonical)
|
||||
.unwrap_or(&full_canonical);
|
||||
|
||||
// Separate into parent directory and filename
|
||||
let filename = relative
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
// First component is username
|
||||
let mut components = path_buf.components();
|
||||
let username = components
|
||||
.next()
|
||||
.map(|c| c.as_os_str().to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let parent = relative
|
||||
.parent()
|
||||
.map(|p| p.to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
// Remaining path (filepath)
|
||||
let filepath: String = components
|
||||
.map(|c| c.as_os_str().to_string_lossy().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
|
||||
(parent, filename)
|
||||
(username, filepath)
|
||||
}
|
||||
|
||||
/// Compute UUID from full path using data root for relative path extraction
|
||||
pub fn compute_uuid_from_path_with_root(full_path: &str, data_root: &str) -> String {
|
||||
let (parent, filename) = extract_relative_path(full_path, data_root);
|
||||
compute_uuid(&parent, &filename)
|
||||
/// Compute UUID from relative path (like ./demo/video.mp4)
|
||||
/// The username is extracted from the first path component
|
||||
pub fn compute_uuid_from_relative_path(relative_path: &str) -> String {
|
||||
let (username, filepath) = extract_user_from_relative_path(relative_path);
|
||||
compute_uuid(&username, &filepath)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -78,24 +75,26 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_relative_path_extraction() {
|
||||
let (parent, filename) =
|
||||
extract_relative_path("/data/sftpgo/data/demo/video.mp4", "/data/sftpgo/data");
|
||||
assert_eq!(parent, "demo");
|
||||
assert_eq!(filename, "video.mp4");
|
||||
fn test_extract_user_from_relative_path() {
|
||||
let (username, filepath) = extract_user_from_relative_path("./demo/video.mp4");
|
||||
assert_eq!(username, "demo");
|
||||
assert_eq!(filepath, "video.mp4");
|
||||
|
||||
let (username, filepath) = extract_user_from_relative_path("./demo/path/to/video.mp4");
|
||||
assert_eq!(username, "demo");
|
||||
assert_eq!(filepath, "path/to/video.mp4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_uuid_with_data_root() {
|
||||
let uuid1 = compute_uuid_from_path_with_root(
|
||||
"/data/sftpgo/data/demo/video.mp4",
|
||||
"/data/sftpgo/data",
|
||||
);
|
||||
let uuid2 = compute_uuid_from_path_with_root(
|
||||
"/data/sftpgo/data/demo/video.mp4",
|
||||
"/data/sftpgo/data",
|
||||
);
|
||||
fn test_uuid_from_relative_path() {
|
||||
let uuid1 = compute_uuid_from_relative_path("./demo/video.mp4");
|
||||
let uuid2 = compute_uuid_from_relative_path("./demo/video.mp4");
|
||||
assert_eq!(uuid1, uuid2);
|
||||
assert_eq!(uuid1.len(), 16);
|
||||
|
||||
// Different users with same filename should have different UUIDs
|
||||
let uuid_demo = compute_uuid_from_relative_path("./demo/video.mp4");
|
||||
let uuid_warren = compute_uuid_from_relative_path("./warren/video.mp4");
|
||||
assert_ne!(uuid_demo, uuid_warren);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user