feat: Initial v0.9 release with API Key authentication
## v0.9.20260325_144654 ### Features - API Key Authentication System - Job Worker System - V2 Backup Versioning ### Bug Fixes - get_processor_results_by_job column mapping Co-authored-by: OpenCode
This commit is contained in:
85
src/core/cache/keys.rs
vendored
Normal file
85
src/core/cache/keys.rs
vendored
Normal file
@@ -0,0 +1,85 @@
|
||||
pub const CATEGORY_VIDEOS: &str = "videos";
|
||||
pub const CATEGORY_SEARCH: &str = "search";
|
||||
pub const CATEGORY_HYBRID_SEARCH: &str = "hybrid_search";
|
||||
pub const CATEGORY_N8N_SEARCH: &str = "n8n_search";
|
||||
pub const CATEGORY_VIDEO_META: &str = "video_meta";
|
||||
pub const CATEGORY_HEALTH: &str = "health";
|
||||
|
||||
pub const KEY_PREFIX_VIDEOS_LIST: &str = "videos:list:";
|
||||
pub const KEY_PREFIX_VIDEO: &str = "video:";
|
||||
pub const KEY_PREFIX_SEARCH: &str = "search:";
|
||||
pub const KEY_PREFIX_SEARCH_HYBRID: &str = "search:hybrid:";
|
||||
pub const KEY_PREFIX_SEARCH_N8N: &str = "search:n8n:";
|
||||
pub const KEY_HEALTH: &str = "health:basic";
|
||||
|
||||
pub fn videos_list(page: usize, limit: usize) -> String {
|
||||
format!("{}page={}:limit={}", KEY_PREFIX_VIDEOS_LIST, page, limit)
|
||||
}
|
||||
|
||||
pub fn video_meta(uuid: &str) -> String {
|
||||
format!("{}{}", KEY_PREFIX_VIDEO, uuid)
|
||||
}
|
||||
|
||||
pub fn search(query_hash: &str) -> String {
|
||||
format!("{}{}", KEY_PREFIX_SEARCH, query_hash)
|
||||
}
|
||||
|
||||
pub fn hybrid_search(query_hash: &str) -> String {
|
||||
format!("{}{}", KEY_PREFIX_SEARCH_HYBRID, query_hash)
|
||||
}
|
||||
|
||||
pub fn n8n_search(query_hash: &str) -> String {
|
||||
format!("{}{}", KEY_PREFIX_SEARCH_N8N, query_hash)
|
||||
}
|
||||
|
||||
pub fn health() -> String {
|
||||
KEY_HEALTH.to_string()
|
||||
}
|
||||
|
||||
pub fn videos_list_prefix() -> String {
|
||||
format!("^{}", KEY_PREFIX_VIDEOS_LIST)
|
||||
}
|
||||
|
||||
pub fn video_prefix(uuid: &str) -> String {
|
||||
format!("^{}{}", KEY_PREFIX_VIDEO, uuid)
|
||||
}
|
||||
|
||||
pub fn search_prefix() -> String {
|
||||
format!("^{}", KEY_PREFIX_SEARCH)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_videos_list() {
|
||||
assert_eq!(videos_list(1, 20), "videos:list:page=1:limit=20");
|
||||
assert_eq!(videos_list(2, 50), "videos:list:page=2:limit=50");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_video_meta() {
|
||||
assert_eq!(video_meta("abc123"), "video:abc123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_search() {
|
||||
assert_eq!(search("hash123"), "search:hash123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hybrid_search() {
|
||||
assert_eq!(hybrid_search("hash123"), "search:hybrid:hash123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_n8n_search() {
|
||||
assert_eq!(n8n_search("hash123"), "search:n8n:hash123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_health() {
|
||||
assert_eq!(health(), "health:basic");
|
||||
}
|
||||
}
|
||||
10
src/core/cache/mod.rs
vendored
Normal file
10
src/core/cache/mod.rs
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
pub mod keys;
|
||||
pub mod mongo_cache;
|
||||
pub mod redis_cache;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use keys::*;
|
||||
pub use mongo_cache::MongoCache;
|
||||
pub use redis_cache::RedisCache;
|
||||
311
src/core/cache/mongo_cache.rs
vendored
Normal file
311
src/core/cache/mongo_cache.rs
vendored
Normal file
@@ -0,0 +1,311 @@
|
||||
use anyhow::{Context, Result};
|
||||
use bson::{doc, oid::ObjectId, DateTime as BsonDateTime, Document};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use mongodb::{Client, Collection, Database, IndexModel};
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::keys;
|
||||
use crate::core::config::cache as cache_config;
|
||||
|
||||
const DB_NAME: &str = "momento";
|
||||
const COLLECTION_NAME: &str = "cache";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheEntry {
|
||||
#[serde(rename = "_id", skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<ObjectId>,
|
||||
pub key: String,
|
||||
pub value: serde_json::Value,
|
||||
pub category: String,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
#[serde(default)]
|
||||
pub hit_count: i64,
|
||||
#[serde(default)]
|
||||
pub last_access: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CacheSettings {
|
||||
pub enabled: bool,
|
||||
pub ttl_videos: u64,
|
||||
pub ttl_search: u64,
|
||||
pub ttl_hybrid_search: u64,
|
||||
pub ttl_video_meta: u64,
|
||||
}
|
||||
|
||||
impl Default for CacheSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: *cache_config::MONGODB_CACHE_ENABLED,
|
||||
ttl_videos: *cache_config::MONGODB_CACHE_TTL_VIDEOS,
|
||||
ttl_search: *cache_config::MONGODB_CACHE_TTL_SEARCH,
|
||||
ttl_hybrid_search: *cache_config::MONGODB_CACHE_TTL_HYBRID_SEARCH,
|
||||
ttl_video_meta: *cache_config::MONGODB_CACHE_TTL_VIDEO_META,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MongoCache {
|
||||
#[allow(dead_code)]
|
||||
client: Client,
|
||||
db: Database,
|
||||
collection: Collection<Document>,
|
||||
settings: CacheSettings,
|
||||
initialized: Arc<RwLock<bool>>,
|
||||
}
|
||||
|
||||
impl MongoCache {
|
||||
pub async fn init() -> Result<Self> {
|
||||
let uri = crate::core::config::MONGODB_URL.as_str();
|
||||
let client = Client::with_uri_str(uri)
|
||||
.await
|
||||
.context("Failed to connect to MongoDB")?;
|
||||
let db = client.database(DB_NAME);
|
||||
let collection: Collection<Document> = db.collection(COLLECTION_NAME);
|
||||
let settings = CacheSettings::default();
|
||||
|
||||
let cache = Self {
|
||||
client,
|
||||
db,
|
||||
collection,
|
||||
settings,
|
||||
initialized: Arc::new(RwLock::new(false)),
|
||||
};
|
||||
|
||||
cache.ensure_indexes().await?;
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
async fn ensure_indexes(&self) -> Result<()> {
|
||||
let mut guard = self.initialized.write().await;
|
||||
if *guard {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let ttl_index = IndexModel::builder()
|
||||
.keys(doc! { "expires_at": 1 })
|
||||
.options(
|
||||
mongodb::options::IndexOptions::builder()
|
||||
.expire_after(std::time::Duration::from_secs(0))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
let key_index = IndexModel::builder()
|
||||
.keys(doc! { "key": 1 })
|
||||
.options(
|
||||
mongodb::options::IndexOptions::builder()
|
||||
.unique(true)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
let category_index = IndexModel::builder().keys(doc! { "category": 1 }).build();
|
||||
|
||||
self.collection
|
||||
.create_indexes([ttl_index, key_index, category_index], None)
|
||||
.await
|
||||
.context("Failed to create cache indexes")?;
|
||||
|
||||
*guard = true;
|
||||
tracing::info!("MongoDB cache indexes ensured");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.settings.enabled
|
||||
}
|
||||
|
||||
pub fn ttl_videos(&self) -> u64 {
|
||||
self.settings.ttl_videos
|
||||
}
|
||||
|
||||
pub fn ttl_search(&self) -> u64 {
|
||||
self.settings.ttl_search
|
||||
}
|
||||
|
||||
pub fn ttl_hybrid_search(&self) -> u64 {
|
||||
self.settings.ttl_hybrid_search
|
||||
}
|
||||
|
||||
pub fn ttl_video_meta(&self) -> u64 {
|
||||
self.settings.ttl_video_meta
|
||||
}
|
||||
|
||||
pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>> {
|
||||
if !self.is_enabled() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let filter = doc! { "key": key };
|
||||
let result = self.collection.find_one(filter, None).await?;
|
||||
|
||||
if let Some(doc) = result {
|
||||
if let Some(value_bson) = doc.get("value") {
|
||||
let json_value: serde_json::Value = bson::from_bson(value_bson.clone())?;
|
||||
let value: T = serde_json::from_value(json_value)?;
|
||||
|
||||
if let Ok(id) = doc.get_object_id("_id") {
|
||||
let update = doc! {
|
||||
"$inc": { "hit_count": 1i64 },
|
||||
"$set": { "last_access": BsonDateTime::from_chrono(Utc::now()) }
|
||||
};
|
||||
if let Err(e) = self
|
||||
.collection
|
||||
.update_one(doc! { "_id": id }, update, None)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Failed to update cache hit count: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(Some(value));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub async fn set<T: Serialize>(
|
||||
&self,
|
||||
key: &str,
|
||||
value: &T,
|
||||
ttl_secs: u64,
|
||||
category: &str,
|
||||
) -> Result<()> {
|
||||
if !self.is_enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
let expires_at = now + Duration::seconds(ttl_secs as i64);
|
||||
let json_value = serde_json::to_value(value)?;
|
||||
let bson_value = bson::to_bson(&json_value)?;
|
||||
|
||||
let filter = doc! { "key": key };
|
||||
let update = doc! {
|
||||
"$set": {
|
||||
"value": bson_value,
|
||||
"category": category,
|
||||
"expires_at": BsonDateTime::from_chrono(expires_at),
|
||||
"last_access": BsonDateTime::from_chrono(now),
|
||||
},
|
||||
"$setOnInsert": {
|
||||
"key": key,
|
||||
"created_at": BsonDateTime::from_chrono(now),
|
||||
"hit_count": 0i64,
|
||||
}
|
||||
};
|
||||
|
||||
let options = mongodb::options::UpdateOptions::builder()
|
||||
.upsert(true)
|
||||
.build();
|
||||
self.collection
|
||||
.update_one(filter, update, options)
|
||||
.await
|
||||
.context("Failed to set cache entry")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete(&self, key: &str) -> Result<bool> {
|
||||
if !self.is_enabled() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let filter = doc! { "key": key };
|
||||
let result = self.collection.delete_one(filter, None).await?;
|
||||
Ok(result.deleted_count > 0)
|
||||
}
|
||||
|
||||
pub async fn invalidate_category(&self, category: &str) -> Result<u64> {
|
||||
if !self.is_enabled() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let filter = doc! { "category": category };
|
||||
let result = self.collection.delete_many(filter, None).await?;
|
||||
let count = result.deleted_count;
|
||||
tracing::debug!("Invalidated {} entries in category: {}", count, category);
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub async fn invalidate_prefix(&self, prefix: &str) -> Result<u64> {
|
||||
if !self.is_enabled() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let regex_pattern = format!("^{}", prefix);
|
||||
let filter = doc! { "key": { "$regex": ®ex_pattern } };
|
||||
let result = self.collection.delete_many(filter, None).await?;
|
||||
let count = result.deleted_count;
|
||||
tracing::debug!("Invalidated {} entries with prefix: {}", count, prefix);
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub async fn get_or_fetch<F, Fut, T>(
|
||||
&self,
|
||||
key: &str,
|
||||
ttl_secs: u64,
|
||||
category: &str,
|
||||
fetcher: F,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
T: DeserializeOwned + Serialize,
|
||||
{
|
||||
if let Some(cached) = self.get::<T>(key).await? {
|
||||
tracing::debug!("Cache hit for key: {}", key);
|
||||
return Ok(cached);
|
||||
}
|
||||
|
||||
tracing::debug!("Cache miss for key: {}", key);
|
||||
let value = fetcher().await?;
|
||||
if let Err(e) = self.set(key, &value, ttl_secs, category).await {
|
||||
tracing::warn!("Failed to cache value: {}", e);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub async fn invalidate_videos_list(&self) -> Result<u64> {
|
||||
self.invalidate_category(keys::CATEGORY_VIDEOS).await
|
||||
}
|
||||
|
||||
pub async fn invalidate_video(&self, uuid: &str) -> Result<u64> {
|
||||
let key = keys::video_meta(uuid);
|
||||
let count = self.delete(&key).await? as u64;
|
||||
let list_count = self.invalidate_videos_list().await?;
|
||||
Ok(count + list_count)
|
||||
}
|
||||
|
||||
pub async fn invalidate_all_search(&self) -> Result<u64> {
|
||||
let count1 = self.invalidate_category(keys::CATEGORY_SEARCH).await?;
|
||||
let count2 = self
|
||||
.invalidate_category(keys::CATEGORY_HYBRID_SEARCH)
|
||||
.await?;
|
||||
let count3 = self.invalidate_category(keys::CATEGORY_N8N_SEARCH).await?;
|
||||
Ok(count1 + count2 + count3)
|
||||
}
|
||||
|
||||
pub async fn health_check(&self) -> Result<bool> {
|
||||
self.db.run_command(doc! { "ping": 1 }, None).await?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_cache_settings_default() {
|
||||
let settings = CacheSettings::default();
|
||||
assert!(settings.enabled);
|
||||
assert_eq!(settings.ttl_videos, 300);
|
||||
assert_eq!(settings.ttl_search, 300);
|
||||
}
|
||||
}
|
||||
120
src/core/cache/tests.rs
vendored
Normal file
120
src/core/cache/tests.rs
vendored
Normal file
@@ -0,0 +1,120 @@
|
||||
use crate::core::cache::keys;
|
||||
use crate::core::cache::mongo_cache::CacheSettings;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_cache_settings_default() {
|
||||
let settings = CacheSettings::default();
|
||||
assert!(settings.enabled);
|
||||
assert_eq!(settings.ttl_videos, 300);
|
||||
assert_eq!(settings.ttl_search, 300);
|
||||
assert_eq!(settings.ttl_hybrid_search, 600);
|
||||
assert_eq!(settings.ttl_video_meta, 3600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_videos_list() {
|
||||
let key = keys::videos_list(1, 20);
|
||||
assert_eq!(key, "videos:list:page=1:limit=20");
|
||||
|
||||
let key2 = keys::videos_list(2, 50);
|
||||
assert_eq!(key2, "videos:list:page=2:limit=50");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_video_meta() {
|
||||
let key = keys::video_meta("abc123");
|
||||
assert_eq!(key, "video:abc123");
|
||||
|
||||
let uuid = "5dea6618a606e7c7";
|
||||
let key = keys::video_meta(uuid);
|
||||
assert_eq!(key, "video:5dea6618a606e7c7");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_search() {
|
||||
let key = keys::search("hash123");
|
||||
assert_eq!(key, "search:hash123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_hybrid_search() {
|
||||
let key = keys::hybrid_search("hash123");
|
||||
assert_eq!(key, "search:hybrid:hash123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_n8n_search() {
|
||||
let key = keys::n8n_search("hash123");
|
||||
assert_eq!(key, "search:n8n:hash123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_health() {
|
||||
let key = keys::health();
|
||||
assert_eq!(key, "health:basic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_categories() {
|
||||
assert_eq!(keys::CATEGORY_VIDEOS, "videos");
|
||||
assert_eq!(keys::CATEGORY_SEARCH, "search");
|
||||
assert_eq!(keys::CATEGORY_HYBRID_SEARCH, "hybrid_search");
|
||||
assert_eq!(keys::CATEGORY_VIDEO_META, "video_meta");
|
||||
assert_eq!(keys::CATEGORY_N8N_SEARCH, "n8n_search");
|
||||
assert_eq!(keys::CATEGORY_HEALTH, "health");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_prefixes() {
|
||||
assert_eq!(keys::KEY_PREFIX_VIDEOS_LIST, "videos:list:");
|
||||
assert_eq!(keys::KEY_PREFIX_VIDEO, "video:");
|
||||
assert_eq!(keys::KEY_PREFIX_SEARCH, "search:");
|
||||
assert_eq!(keys::KEY_PREFIX_SEARCH_HYBRID, "search:hybrid:");
|
||||
assert_eq!(keys::KEY_PREFIX_SEARCH_N8N, "search:n8n:");
|
||||
assert_eq!(keys::KEY_HEALTH, "health:basic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_ttl_values() {
|
||||
let settings = CacheSettings::default();
|
||||
|
||||
assert!(settings.ttl_videos >= 60 && settings.ttl_videos <= 600);
|
||||
assert!(settings.ttl_search >= 60 && settings.ttl_search <= 600);
|
||||
assert!(settings.ttl_hybrid_search >= 60 && settings.ttl_hybrid_search <= 3600);
|
||||
assert!(settings.ttl_video_meta >= 300 && settings.ttl_video_meta <= 7200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_videos_list_prefix_format() {
|
||||
let key = keys::videos_list(1, 10);
|
||||
assert!(key.starts_with("videos:list:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_video_meta_prefix_format() {
|
||||
let key = keys::video_meta("uuid123");
|
||||
assert!(key.starts_with("video:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_search_prefix_format() {
|
||||
let key = keys::search("test");
|
||||
assert!(key.starts_with("search:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_hybrid_search_prefix_format() {
|
||||
let key = keys::hybrid_search("test");
|
||||
assert!(key.starts_with("search:hybrid:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_n8n_search_prefix_format() {
|
||||
let key = keys::n8n_search("test");
|
||||
assert!(key.starts_with("search:n8n:"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user