feat: Initial v0.9 release with API Key authentication

## v0.9.20260325_144654

### Features
- API Key Authentication System
- Job Worker System
- V2 Backup Versioning

### Bug Fixes
- get_processor_results_by_job column mapping

Co-authored-by: OpenCode
This commit is contained in:
accusys
2026-03-25 14:52:51 +08:00
parent 47e86b696f
commit 383201cacd
193 changed files with 40268 additions and 422 deletions

85
src/core/cache/keys.rs vendored Normal file
View File

@@ -0,0 +1,85 @@
pub const CATEGORY_VIDEOS: &str = "videos";
pub const CATEGORY_SEARCH: &str = "search";
pub const CATEGORY_HYBRID_SEARCH: &str = "hybrid_search";
pub const CATEGORY_N8N_SEARCH: &str = "n8n_search";
pub const CATEGORY_VIDEO_META: &str = "video_meta";
pub const CATEGORY_HEALTH: &str = "health";
pub const KEY_PREFIX_VIDEOS_LIST: &str = "videos:list:";
pub const KEY_PREFIX_VIDEO: &str = "video:";
pub const KEY_PREFIX_SEARCH: &str = "search:";
pub const KEY_PREFIX_SEARCH_HYBRID: &str = "search:hybrid:";
pub const KEY_PREFIX_SEARCH_N8N: &str = "search:n8n:";
pub const KEY_HEALTH: &str = "health:basic";
pub fn videos_list(page: usize, limit: usize) -> String {
format!("{}page={}:limit={}", KEY_PREFIX_VIDEOS_LIST, page, limit)
}
pub fn video_meta(uuid: &str) -> String {
format!("{}{}", KEY_PREFIX_VIDEO, uuid)
}
pub fn search(query_hash: &str) -> String {
format!("{}{}", KEY_PREFIX_SEARCH, query_hash)
}
pub fn hybrid_search(query_hash: &str) -> String {
format!("{}{}", KEY_PREFIX_SEARCH_HYBRID, query_hash)
}
pub fn n8n_search(query_hash: &str) -> String {
format!("{}{}", KEY_PREFIX_SEARCH_N8N, query_hash)
}
pub fn health() -> String {
KEY_HEALTH.to_string()
}
pub fn videos_list_prefix() -> String {
format!("^{}", KEY_PREFIX_VIDEOS_LIST)
}
pub fn video_prefix(uuid: &str) -> String {
format!("^{}{}", KEY_PREFIX_VIDEO, uuid)
}
pub fn search_prefix() -> String {
format!("^{}", KEY_PREFIX_SEARCH)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_videos_list() {
assert_eq!(videos_list(1, 20), "videos:list:page=1:limit=20");
assert_eq!(videos_list(2, 50), "videos:list:page=2:limit=50");
}
#[test]
fn test_video_meta() {
assert_eq!(video_meta("abc123"), "video:abc123");
}
#[test]
fn test_search() {
assert_eq!(search("hash123"), "search:hash123");
}
#[test]
fn test_hybrid_search() {
assert_eq!(hybrid_search("hash123"), "search:hybrid:hash123");
}
#[test]
fn test_n8n_search() {
assert_eq!(n8n_search("hash123"), "search:n8n:hash123");
}
#[test]
fn test_health() {
assert_eq!(health(), "health:basic");
}
}

10
src/core/cache/mod.rs vendored Normal file
View File

@@ -0,0 +1,10 @@
pub mod keys;
pub mod mongo_cache;
pub mod redis_cache;
#[cfg(test)]
mod tests;
pub use keys::*;
pub use mongo_cache::MongoCache;
pub use redis_cache::RedisCache;

311
src/core/cache/mongo_cache.rs vendored Normal file
View File

@@ -0,0 +1,311 @@
use anyhow::{Context, Result};
use bson::{doc, oid::ObjectId, DateTime as BsonDateTime, Document};
use chrono::{DateTime, Duration, Utc};
use mongodb::{Client, Collection, Database, IndexModel};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::RwLock;
use super::keys;
use crate::core::config::cache as cache_config;
const DB_NAME: &str = "momento";
const COLLECTION_NAME: &str = "cache";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheEntry {
#[serde(rename = "_id", skip_serializing_if = "Option::is_none")]
pub id: Option<ObjectId>,
pub key: String,
pub value: serde_json::Value,
pub category: String,
pub created_at: DateTime<Utc>,
pub expires_at: DateTime<Utc>,
#[serde(default)]
pub hit_count: i64,
#[serde(default)]
pub last_access: DateTime<Utc>,
}
#[derive(Debug, Clone)]
pub struct CacheSettings {
pub enabled: bool,
pub ttl_videos: u64,
pub ttl_search: u64,
pub ttl_hybrid_search: u64,
pub ttl_video_meta: u64,
}
impl Default for CacheSettings {
fn default() -> Self {
Self {
enabled: *cache_config::MONGODB_CACHE_ENABLED,
ttl_videos: *cache_config::MONGODB_CACHE_TTL_VIDEOS,
ttl_search: *cache_config::MONGODB_CACHE_TTL_SEARCH,
ttl_hybrid_search: *cache_config::MONGODB_CACHE_TTL_HYBRID_SEARCH,
ttl_video_meta: *cache_config::MONGODB_CACHE_TTL_VIDEO_META,
}
}
}
#[derive(Clone)]
pub struct MongoCache {
#[allow(dead_code)]
client: Client,
db: Database,
collection: Collection<Document>,
settings: CacheSettings,
initialized: Arc<RwLock<bool>>,
}
impl MongoCache {
pub async fn init() -> Result<Self> {
let uri = crate::core::config::MONGODB_URL.as_str();
let client = Client::with_uri_str(uri)
.await
.context("Failed to connect to MongoDB")?;
let db = client.database(DB_NAME);
let collection: Collection<Document> = db.collection(COLLECTION_NAME);
let settings = CacheSettings::default();
let cache = Self {
client,
db,
collection,
settings,
initialized: Arc::new(RwLock::new(false)),
};
cache.ensure_indexes().await?;
Ok(cache)
}
async fn ensure_indexes(&self) -> Result<()> {
let mut guard = self.initialized.write().await;
if *guard {
return Ok(());
}
let ttl_index = IndexModel::builder()
.keys(doc! { "expires_at": 1 })
.options(
mongodb::options::IndexOptions::builder()
.expire_after(std::time::Duration::from_secs(0))
.build(),
)
.build();
let key_index = IndexModel::builder()
.keys(doc! { "key": 1 })
.options(
mongodb::options::IndexOptions::builder()
.unique(true)
.build(),
)
.build();
let category_index = IndexModel::builder().keys(doc! { "category": 1 }).build();
self.collection
.create_indexes([ttl_index, key_index, category_index], None)
.await
.context("Failed to create cache indexes")?;
*guard = true;
tracing::info!("MongoDB cache indexes ensured");
Ok(())
}
pub fn is_enabled(&self) -> bool {
self.settings.enabled
}
pub fn ttl_videos(&self) -> u64 {
self.settings.ttl_videos
}
pub fn ttl_search(&self) -> u64 {
self.settings.ttl_search
}
pub fn ttl_hybrid_search(&self) -> u64 {
self.settings.ttl_hybrid_search
}
pub fn ttl_video_meta(&self) -> u64 {
self.settings.ttl_video_meta
}
pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>> {
if !self.is_enabled() {
return Ok(None);
}
let filter = doc! { "key": key };
let result = self.collection.find_one(filter, None).await?;
if let Some(doc) = result {
if let Some(value_bson) = doc.get("value") {
let json_value: serde_json::Value = bson::from_bson(value_bson.clone())?;
let value: T = serde_json::from_value(json_value)?;
if let Ok(id) = doc.get_object_id("_id") {
let update = doc! {
"$inc": { "hit_count": 1i64 },
"$set": { "last_access": BsonDateTime::from_chrono(Utc::now()) }
};
if let Err(e) = self
.collection
.update_one(doc! { "_id": id }, update, None)
.await
{
tracing::warn!("Failed to update cache hit count: {}", e);
}
}
return Ok(Some(value));
}
}
Ok(None)
}
pub async fn set<T: Serialize>(
&self,
key: &str,
value: &T,
ttl_secs: u64,
category: &str,
) -> Result<()> {
if !self.is_enabled() {
return Ok(());
}
let now = Utc::now();
let expires_at = now + Duration::seconds(ttl_secs as i64);
let json_value = serde_json::to_value(value)?;
let bson_value = bson::to_bson(&json_value)?;
let filter = doc! { "key": key };
let update = doc! {
"$set": {
"value": bson_value,
"category": category,
"expires_at": BsonDateTime::from_chrono(expires_at),
"last_access": BsonDateTime::from_chrono(now),
},
"$setOnInsert": {
"key": key,
"created_at": BsonDateTime::from_chrono(now),
"hit_count": 0i64,
}
};
let options = mongodb::options::UpdateOptions::builder()
.upsert(true)
.build();
self.collection
.update_one(filter, update, options)
.await
.context("Failed to set cache entry")?;
Ok(())
}
pub async fn delete(&self, key: &str) -> Result<bool> {
if !self.is_enabled() {
return Ok(false);
}
let filter = doc! { "key": key };
let result = self.collection.delete_one(filter, None).await?;
Ok(result.deleted_count > 0)
}
pub async fn invalidate_category(&self, category: &str) -> Result<u64> {
if !self.is_enabled() {
return Ok(0);
}
let filter = doc! { "category": category };
let result = self.collection.delete_many(filter, None).await?;
let count = result.deleted_count;
tracing::debug!("Invalidated {} entries in category: {}", count, category);
Ok(count)
}
pub async fn invalidate_prefix(&self, prefix: &str) -> Result<u64> {
if !self.is_enabled() {
return Ok(0);
}
let regex_pattern = format!("^{}", prefix);
let filter = doc! { "key": { "$regex": &regex_pattern } };
let result = self.collection.delete_many(filter, None).await?;
let count = result.deleted_count;
tracing::debug!("Invalidated {} entries with prefix: {}", count, prefix);
Ok(count)
}
pub async fn get_or_fetch<F, Fut, T>(
&self,
key: &str,
ttl_secs: u64,
category: &str,
fetcher: F,
) -> Result<T>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
T: DeserializeOwned + Serialize,
{
if let Some(cached) = self.get::<T>(key).await? {
tracing::debug!("Cache hit for key: {}", key);
return Ok(cached);
}
tracing::debug!("Cache miss for key: {}", key);
let value = fetcher().await?;
if let Err(e) = self.set(key, &value, ttl_secs, category).await {
tracing::warn!("Failed to cache value: {}", e);
}
Ok(value)
}
pub async fn invalidate_videos_list(&self) -> Result<u64> {
self.invalidate_category(keys::CATEGORY_VIDEOS).await
}
pub async fn invalidate_video(&self, uuid: &str) -> Result<u64> {
let key = keys::video_meta(uuid);
let count = self.delete(&key).await? as u64;
let list_count = self.invalidate_videos_list().await?;
Ok(count + list_count)
}
pub async fn invalidate_all_search(&self) -> Result<u64> {
let count1 = self.invalidate_category(keys::CATEGORY_SEARCH).await?;
let count2 = self
.invalidate_category(keys::CATEGORY_HYBRID_SEARCH)
.await?;
let count3 = self.invalidate_category(keys::CATEGORY_N8N_SEARCH).await?;
Ok(count1 + count2 + count3)
}
pub async fn health_check(&self) -> Result<bool> {
self.db.run_command(doc! { "ping": 1 }, None).await?;
Ok(true)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_cache_settings_default() {
let settings = CacheSettings::default();
assert!(settings.enabled);
assert_eq!(settings.ttl_videos, 300);
assert_eq!(settings.ttl_search, 300);
}
}

120
src/core/cache/tests.rs vendored Normal file
View File

@@ -0,0 +1,120 @@
use crate::core::cache::keys;
use crate::core::cache::mongo_cache::CacheSettings;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_cache_settings_default() {
let settings = CacheSettings::default();
assert!(settings.enabled);
assert_eq!(settings.ttl_videos, 300);
assert_eq!(settings.ttl_search, 300);
assert_eq!(settings.ttl_hybrid_search, 600);
assert_eq!(settings.ttl_video_meta, 3600);
}
#[test]
fn test_cache_key_videos_list() {
let key = keys::videos_list(1, 20);
assert_eq!(key, "videos:list:page=1:limit=20");
let key2 = keys::videos_list(2, 50);
assert_eq!(key2, "videos:list:page=2:limit=50");
}
#[test]
fn test_cache_key_video_meta() {
let key = keys::video_meta("abc123");
assert_eq!(key, "video:abc123");
let uuid = "5dea6618a606e7c7";
let key = keys::video_meta(uuid);
assert_eq!(key, "video:5dea6618a606e7c7");
}
#[test]
fn test_cache_key_search() {
let key = keys::search("hash123");
assert_eq!(key, "search:hash123");
}
#[test]
fn test_cache_key_hybrid_search() {
let key = keys::hybrid_search("hash123");
assert_eq!(key, "search:hybrid:hash123");
}
#[test]
fn test_cache_key_n8n_search() {
let key = keys::n8n_search("hash123");
assert_eq!(key, "search:n8n:hash123");
}
#[test]
fn test_cache_key_health() {
let key = keys::health();
assert_eq!(key, "health:basic");
}
#[test]
fn test_cache_categories() {
assert_eq!(keys::CATEGORY_VIDEOS, "videos");
assert_eq!(keys::CATEGORY_SEARCH, "search");
assert_eq!(keys::CATEGORY_HYBRID_SEARCH, "hybrid_search");
assert_eq!(keys::CATEGORY_VIDEO_META, "video_meta");
assert_eq!(keys::CATEGORY_N8N_SEARCH, "n8n_search");
assert_eq!(keys::CATEGORY_HEALTH, "health");
}
#[test]
fn test_cache_key_prefixes() {
assert_eq!(keys::KEY_PREFIX_VIDEOS_LIST, "videos:list:");
assert_eq!(keys::KEY_PREFIX_VIDEO, "video:");
assert_eq!(keys::KEY_PREFIX_SEARCH, "search:");
assert_eq!(keys::KEY_PREFIX_SEARCH_HYBRID, "search:hybrid:");
assert_eq!(keys::KEY_PREFIX_SEARCH_N8N, "search:n8n:");
assert_eq!(keys::KEY_HEALTH, "health:basic");
}
#[test]
fn test_cache_ttl_values() {
let settings = CacheSettings::default();
assert!(settings.ttl_videos >= 60 && settings.ttl_videos <= 600);
assert!(settings.ttl_search >= 60 && settings.ttl_search <= 600);
assert!(settings.ttl_hybrid_search >= 60 && settings.ttl_hybrid_search <= 3600);
assert!(settings.ttl_video_meta >= 300 && settings.ttl_video_meta <= 7200);
}
#[test]
fn test_cache_key_videos_list_prefix_format() {
let key = keys::videos_list(1, 10);
assert!(key.starts_with("videos:list:"));
}
#[test]
fn test_cache_key_video_meta_prefix_format() {
let key = keys::video_meta("uuid123");
assert!(key.starts_with("video:"));
}
#[test]
fn test_cache_key_search_prefix_format() {
let key = keys::search("test");
assert!(key.starts_with("search:"));
}
#[test]
fn test_cache_key_hybrid_search_prefix_format() {
let key = keys::hybrid_search("test");
assert!(key.starts_with("search:hybrid:"));
}
#[test]
fn test_cache_key_n8n_search_prefix_format() {
let key = keys::n8n_search("test");
assert!(key.starts_with("search:n8n:"));
}
}