feat: add momentry_playground binary for development
- Add separate momentry_playground binary with distinct configuration - Production (momentry): Port 3002, Redis prefix 'momentry:' - Development (momentry_playground): Port 3003, Redis prefix 'momentry_dev:' - Add SERVER_PORT and REDIS_KEY_PREFIX config via environment variables - Replace all hardcoded Redis key prefixes with configurable values - Create .env.development for playground environment settings - Update .env with production defaults - Add dotenv dependency for environment file loading Configuration isolation allows running both binaries simultaneously without port conflicts or Redis key collisions.
This commit is contained in:
@@ -6,6 +6,8 @@ use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::core::config::REDIS_KEY_PREFIX;
|
||||
|
||||
pub struct RedisClient {
|
||||
client: Client,
|
||||
state: Arc<RwLock<RedisState>>,
|
||||
@@ -18,13 +20,8 @@ pub struct RedisState {
|
||||
|
||||
impl RedisClient {
|
||||
pub fn new() -> Result<Self> {
|
||||
let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| {
|
||||
let password =
|
||||
std::env::var("REDIS_PASSWORD").unwrap_or_else(|_| "accusys".to_string());
|
||||
format!("redis://:{}@localhost:6379", password)
|
||||
});
|
||||
|
||||
let client = Client::open(redis_url.as_str()).context("Failed to connect to Redis")?;
|
||||
let client = Client::open(crate::core::config::REDIS_URL.as_str())
|
||||
.context("Failed to connect to Redis")?;
|
||||
|
||||
Ok(Self {
|
||||
client,
|
||||
@@ -49,7 +46,8 @@ impl RedisClient {
|
||||
|
||||
pub async fn get_job_status(&self, uuid: &str) -> Result<Option<JobStatus>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let key = format!("momentry:job:{}", uuid);
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}job:{}", prefix, uuid);
|
||||
|
||||
let status: Option<String> = conn.hget(&key, "status").await?;
|
||||
if status.is_none() {
|
||||
@@ -83,7 +81,8 @@ impl RedisClient {
|
||||
status: &ProcessorStatus,
|
||||
) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let key = format!("momentry:job:{}:processor:{}", uuid, processor);
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}job:{}:processor:{}", prefix, uuid, processor);
|
||||
|
||||
let _: Option<String> = conn
|
||||
.hset_multiple(
|
||||
@@ -111,7 +110,8 @@ impl RedisClient {
|
||||
processor: &str,
|
||||
) -> Result<Option<ProcessorStatus>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let key = format!("momentry:job:{}:processor:{}", uuid, processor);
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}job:{}:processor:{}", prefix, uuid, processor);
|
||||
|
||||
let status: Option<String> = conn.hget(&key, "status").await?;
|
||||
if status.is_none() {
|
||||
@@ -138,7 +138,8 @@ impl RedisClient {
|
||||
|
||||
pub async fn publish_progress(&self, uuid: &str, message: &ProgressMessage) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let channel = format!("momentry:progress:{}", uuid);
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let channel = format!("{}progress:{}", prefix, uuid);
|
||||
|
||||
let json = serde_json::to_string(message)?;
|
||||
let _: usize = conn.publish(&channel, json).await?;
|
||||
@@ -148,7 +149,8 @@ impl RedisClient {
|
||||
|
||||
pub async fn subscribe_progress(&self, uuid: &str) -> Result<redis::aio::PubSub> {
|
||||
let mut pubsub = self.client.get_async_pubsub().await?;
|
||||
let channel = format!("momentry:progress:{}", uuid);
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let channel = format!("{}progress:{}", prefix, uuid);
|
||||
|
||||
pubsub.subscribe(channel).await?;
|
||||
|
||||
@@ -174,45 +176,285 @@ impl RedisClient {
|
||||
|
||||
pub async fn add_to_active_jobs(&self, uuid: &str) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let _: usize = conn.sadd("momentry:jobs:active", uuid).await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let _: usize = conn.sadd(format!("{}jobs:active", prefix), uuid).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn move_to_completed_jobs(&self, uuid: &str) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let _: bool = conn
|
||||
.smove("momentry:jobs:active", "momentry:jobs:completed", uuid)
|
||||
.smove(
|
||||
format!("{}jobs:active", prefix),
|
||||
format!("{}jobs:completed", prefix),
|
||||
uuid,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn move_to_failed_jobs(&self, uuid: &str) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let _: bool = conn
|
||||
.smove("momentry:jobs:active", "momentry:jobs:failed", uuid)
|
||||
.smove(
|
||||
format!("{}jobs:active", prefix),
|
||||
format!("{}jobs:failed", prefix),
|
||||
uuid,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_active_jobs(&self) -> Result<Vec<String>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let jobs: Vec<String> = conn.smembers("momentry:jobs:active").await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let jobs: Vec<String> = conn.smembers(format!("{}jobs:active", prefix)).await?;
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
pub async fn set_health(&self, status: &str) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let _: String = conn
|
||||
.set_ex("momentry:health:momentry_core", status, 60)
|
||||
.set_ex(format!("{}health:momentry_core", prefix), status, 60)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_health(&self) -> Result<Option<String>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let health: Option<String> = conn.get("momentry:health:momentry_core").await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let health: Option<String> = conn.get(format!("{}health:momentry_core", prefix)).await?;
|
||||
Ok(health)
|
||||
}
|
||||
|
||||
pub async fn sync_monitor_job(&self, job: &MonitorJobRedis) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}monitor:job:{}", prefix, job.uuid);
|
||||
|
||||
let _: Option<String> = conn
|
||||
.hset_multiple(
|
||||
&key,
|
||||
&[
|
||||
("uuid", job.uuid.as_str()),
|
||||
("status", job.status.as_str()),
|
||||
("current_processor", job.current_processor.as_str()),
|
||||
("progress_total", job.progress_total.to_string().as_str()),
|
||||
(
|
||||
"progress_current",
|
||||
job.progress_current.to_string().as_str(),
|
||||
),
|
||||
("error_count", job.error_count.to_string().as_str()),
|
||||
("started_at", job.started_at.as_str()),
|
||||
("updated_at", job.updated_at.as_str()),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let _: bool = conn.expire(&key, 86400).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn publish_job_error(&self, uuid: &str, error: &str) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let channel = format!("{}errors:{}", prefix, uuid);
|
||||
|
||||
let error_msg = JobErrorMessage {
|
||||
uuid: uuid.to_string(),
|
||||
error: error.to_string(),
|
||||
timestamp: chrono::Utc::now().timestamp(),
|
||||
};
|
||||
|
||||
let json = serde_json::to_string(&error_msg)?;
|
||||
let _: usize = conn.publish(&channel, json).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn publish_anomaly_alert(&self, alert: &AnomalyAlertMessage) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let channel = format!("{}anomaly:alerts", prefix);
|
||||
|
||||
let json = serde_json::to_string(alert)?;
|
||||
let _: usize = conn.publish(&channel, json).await?;
|
||||
|
||||
let key = format!("{}anomaly:key:{}", prefix, alert.key_id);
|
||||
let alert_data = serde_json::json!({
|
||||
"key_id": alert.key_id,
|
||||
"anomaly_type": alert.anomaly_type,
|
||||
"severity": alert.severity,
|
||||
"timestamp": alert.timestamp,
|
||||
"message": alert.message,
|
||||
});
|
||||
let _: Option<String> = conn
|
||||
.hset(&key, "latest", &serde_json::to_string(&alert_data)?)
|
||||
.await?;
|
||||
let _: bool = conn.expire(&key, 86400).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn subscribe_anomaly_alerts(&self) -> Result<redis::aio::PubSub> {
|
||||
let mut pubsub = self.client.get_async_pubsub().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
pubsub
|
||||
.subscribe(format!("{}anomaly:alerts", prefix))
|
||||
.await?;
|
||||
Ok(pubsub)
|
||||
}
|
||||
|
||||
pub async fn get_latest_anomaly(&self, key_id: &str) -> Result<Option<AnomalyAlertMessage>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}anomaly:key:{}", prefix, key_id);
|
||||
let latest: Option<String> = conn.hget(&key, "latest").await?;
|
||||
|
||||
if let Some(json) = latest {
|
||||
let alert: AnomalyAlertMessage = serde_json::from_str(&json)?;
|
||||
Ok(Some(alert))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn subscribe_job_errors(&self, uuid: &str) -> Result<redis::aio::PubSub> {
|
||||
let mut pubsub = self.client.get_async_pubsub().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let channel = format!("{}errors:{}", prefix, uuid);
|
||||
|
||||
pubsub.subscribe(channel).await?;
|
||||
|
||||
Ok(pubsub)
|
||||
}
|
||||
|
||||
pub async fn update_worker_job_status(
|
||||
&self,
|
||||
uuid: &str,
|
||||
job_id: i64,
|
||||
status: &str,
|
||||
current_processor: Option<&str>,
|
||||
progress: i32,
|
||||
total: i32,
|
||||
) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}worker:job:{}", prefix, uuid);
|
||||
|
||||
let _: Option<String> = conn
|
||||
.hset_multiple(
|
||||
&key,
|
||||
&[
|
||||
("job_id", job_id.to_string().as_str()),
|
||||
("status", status),
|
||||
("current_processor", current_processor.unwrap_or("")),
|
||||
("progress_current", progress.to_string().as_str()),
|
||||
("progress_total", total.to_string().as_str()),
|
||||
("updated_at", &chrono::Utc::now().to_rfc3339()),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let _: bool = conn.expire(&key, 86400).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn update_worker_processor_status(
|
||||
&self,
|
||||
uuid: &str,
|
||||
processor: &str,
|
||||
status: &str,
|
||||
error: Option<&str>,
|
||||
) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}worker:job:{}:processor:{}", prefix, uuid, processor);
|
||||
|
||||
let now = chrono::Utc::now().to_rfc3339();
|
||||
|
||||
let mut fields: Vec<(&str, &str)> = vec![("status", status), ("updated_at", &now)];
|
||||
|
||||
if let Some(err) = error {
|
||||
fields.push(("error", err));
|
||||
}
|
||||
|
||||
let _: Option<String> = conn.hset_multiple(&key, &fields).await?;
|
||||
let _: bool = conn.expire(&key, 86400).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_worker_job_status(&self, uuid: &str) -> Result<Option<WorkerJobStatus>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let key = format!("{}worker:job:{}", prefix, uuid);
|
||||
|
||||
let exists: bool = conn.exists(&key).await?;
|
||||
if !exists {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let status: String = conn.hget(&key, "status").await?;
|
||||
let job_id: i64 = conn.hget(&key, "job_id").await?;
|
||||
let current_processor: String = conn.hget(&key, "current_processor").await?;
|
||||
let progress_current: i32 = conn.hget(&key, "progress_current").await?;
|
||||
let progress_total: i32 = conn.hget(&key, "progress_total").await?;
|
||||
let updated_at: String = conn.hget(&key, "updated_at").await?;
|
||||
|
||||
Ok(Some(WorkerJobStatus {
|
||||
uuid: uuid.to_string(),
|
||||
job_id,
|
||||
status,
|
||||
current_processor,
|
||||
progress_current,
|
||||
progress_total,
|
||||
updated_at,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn delete_worker_job(&self, uuid: &str) -> Result<()> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
|
||||
let key = format!("{}worker:job:{}", prefix, uuid);
|
||||
let _: i32 = conn.del(&key).await?;
|
||||
|
||||
let processor_types = ["asr", "cut", "yolo", "ocr", "face", "pose", "asrx"];
|
||||
for ptype in processor_types {
|
||||
let proc_key = format!("{}worker:job:{}:processor:{}", prefix, uuid, ptype);
|
||||
let _: i32 = conn.del(&proc_key).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_all_worker_jobs(&self) -> Result<Vec<WorkerJobInfo>> {
|
||||
let mut conn = self.get_conn_internal().await?;
|
||||
let prefix = REDIS_KEY_PREFIX.as_str();
|
||||
let keys: Vec<String> = conn.keys(format!("{}worker:job:*", prefix)).await?;
|
||||
|
||||
let mut jobs = Vec::new();
|
||||
for key in keys {
|
||||
let uuid = key.replace(&format!("{}worker:job:", prefix), "");
|
||||
if let Some(status) = self.get_worker_job_status(&uuid).await? {
|
||||
jobs.push(WorkerJobInfo {
|
||||
uuid,
|
||||
job_id: status.job_id,
|
||||
status: status.status,
|
||||
progress_current: status.progress_current,
|
||||
progress_total: status.progress_total,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(jobs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RedisClient {
|
||||
@@ -260,3 +502,55 @@ pub struct ProgressData {
|
||||
pub current: Option<i32>,
|
||||
pub total: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MonitorJobRedis {
|
||||
pub uuid: String,
|
||||
pub status: String,
|
||||
pub current_processor: String,
|
||||
pub progress_total: i32,
|
||||
pub progress_current: i32,
|
||||
pub error_count: i32,
|
||||
pub started_at: String,
|
||||
pub updated_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobErrorMessage {
|
||||
pub uuid: String,
|
||||
pub error: String,
|
||||
pub timestamp: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AnomalyAlertMessage {
|
||||
pub key_id: String,
|
||||
pub anomaly_type: String,
|
||||
pub severity: String,
|
||||
pub ip_address: Option<String>,
|
||||
pub request_count: Option<i32>,
|
||||
pub error_rate: Option<f64>,
|
||||
pub unique_ips: Option<i32>,
|
||||
pub timestamp: i64,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkerJobStatus {
|
||||
pub uuid: String,
|
||||
pub job_id: i64,
|
||||
pub status: String,
|
||||
pub current_processor: String,
|
||||
pub progress_current: i32,
|
||||
pub progress_total: i32,
|
||||
pub updated_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkerJobInfo {
|
||||
pub uuid: String,
|
||||
pub job_id: i64,
|
||||
pub status: String,
|
||||
pub progress_current: i32,
|
||||
pub progress_total: i32,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user