Test Gitea Runner functionality

This commit is contained in:
Warren
2026-05-30 14:08:55 +08:00
parent 596d8d5e27
commit b362e9b3f1
44 changed files with 1 additions and 0 deletions

View File

@@ -0,0 +1,82 @@
pub fn voice_for_lang(lang: &str) -> String {
match lang {
"zh_TW" => "Meijia",
"zh_CN" => "Ting-Ting",
"en_US" => "Samantha",
"en_GB" => "Daniel",
"ja_JP" => "Kyoko",
"ko_KR" => "Yuna",
"fr_FR" => "Amelie",
"de_DE" => "Anna",
_ => "Meijia",
}
.to_string()
}
pub fn phrase_for_lang(lang: &str) -> String {
match lang {
"zh_TW" | "zh_CN" => "語音測試一二三",
"en_US" | "en_GB" => "Test one two three",
"ja_JP" => "これはテストです",
"ko_KR" => "테스트입니다",
"fr_FR" => "Ceci est un test",
"de_DE" => "Das ist ein Test",
_ => "Test",
}
.to_string()
}
pub fn audio_devices() -> (Vec<String>, Vec<String>, String, String) {
let run = |t: &str| -> Vec<String> {
if let Ok(r) = std::process::Command::new("SwitchAudioSource")
.args(["-a", "-t", t, "-f", "json"])
.output()
{
String::from_utf8_lossy(&r.stdout)
.lines()
.filter_map(|l| {
serde_json::from_str::<serde_json::Value>(l)
.ok()
.and_then(|v| v["name"].as_str().map(|s| s.to_string()))
})
.collect()
} else {
vec![]
}
};
let current = |t: &str| -> String {
std::process::Command::new("SwitchAudioSource")
.args(["-c", "-t", t])
.output()
.map(|r| String::from_utf8_lossy(&r.stdout).trim().to_string())
.unwrap_or_default()
};
let out = run("output");
let inp = run("input");
let co = current("output");
let ci = current("input");
(out, inp, co, ci)
}
pub fn inject_audio_devices(
html: &str,
out: &[String],
inp: &[String],
cur_out: &str,
cur_in: &str,
) -> String {
let mut out_opts = String::from("<option value=\"\">🔊 System</option>");
for d in out {
let sel = if d == cur_out { " selected" } else { "" };
out_opts.push_str(&format!("<option value=\"{d}\"{sel}>{d}</option>"));
}
let mut inp_opts = String::from("<option value=\"\">🎤 System</option>");
for d in inp {
let sel = if d == cur_in { " selected" } else { "" };
inp_opts.push_str(&format!("<option value=\"{d}\"{sel}>{d}</option>"));
}
html.replace("{out_devs}", &out_opts)
.replace("{in_devs}", &inp_opts)
}

320
markbase-core/src/auth.rs Normal file
View File

@@ -0,0 +1,320 @@
use bcrypt::{hash, verify, DEFAULT_COST};
use chrono::{Duration, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use uuid::Uuid;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct User {
pub user_id: String,
pub username: String,
pub password_hash: String,
pub created_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Session {
pub token: String,
pub user_id: String,
pub username: String,
pub created_at: String,
pub expires_at: String,
pub groups: Vec<String>,
pub permissions: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoginRequest {
pub username: String,
pub password: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoginResponse {
pub token: String,
pub expires_at: String,
pub user_id: String,
pub groups: Vec<String>,
pub permissions: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdminLoginRequest {
pub username: String,
pub password: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdminSession {
pub token: String,
pub username: String,
pub created_at: String,
pub expires_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AdminLoginResponse {
pub token: String,
pub expires_at: String,
pub username: String,
}
#[derive(Clone)]
pub struct AuthState {
pub sessions: Arc<Mutex<HashMap<String, Session>>>,
pub users: Arc<Mutex<HashMap<String, User>>>,
pub auth_db: Option<crate::sync::AuthDb>,
pub admin_sessions: Arc<Mutex<HashMap<String, AdminSession>>>,
}
impl AuthState {
pub fn new() -> Self {
let mut users = HashMap::new();
// Create default demo user
let password_hash = hash("demo123", DEFAULT_COST).unwrap();
users.insert(
"demo".to_string(),
User {
user_id: "demo".to_string(),
username: "demo".to_string(),
password_hash,
created_at: Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
},
);
AuthState {
sessions: Arc::new(Mutex::new(HashMap::new())),
users: Arc::new(Mutex::new(users)),
auth_db: None,
admin_sessions: Arc::new(Mutex::new(HashMap::new())),
}
}
pub fn with_sync(auth_db_path: &str) -> Self {
let auth_db = crate::sync::AuthDb::new(auth_db_path).ok();
AuthState {
sessions: Arc::new(Mutex::new(HashMap::new())),
users: Arc::new(Mutex::new(HashMap::new())),
auth_db,
admin_sessions: Arc::new(Mutex::new(HashMap::new())),
}
}
pub fn login(&self, username: &str, password: &str) -> Option<LoginResponse> {
let users = self.users.lock().unwrap();
let user = users.get(username)?;
if verify(password, &user.password_hash).unwrap_or(false) {
let token = Uuid::new_v4().to_string();
let now = Utc::now();
let expires_at = now + Duration::hours(24);
let session = Session {
token: token.clone(),
user_id: user.user_id.clone(),
username: user.username.clone(),
created_at: now.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
groups: vec![],
permissions: "{}".to_string(),
};
let mut sessions = self.sessions.lock().unwrap();
sessions.insert(token.clone(), session);
Some(LoginResponse {
token,
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
user_id: user.user_id.clone(),
groups: vec![],
permissions: "{}".to_string(),
})
} else {
None
}
}
pub fn admin_login(&self, username: &str, password: &str) -> Option<AdminLoginResponse> {
if let Some(auth_db) = &self.auth_db {
match auth_db.get_admin(username) {
Ok(Some(admin)) if admin.status == 1 => {
if verify(password, &admin.password_hash).unwrap_or(false) {
let token = Uuid::new_v4().to_string();
let now = Utc::now();
let expires_at = now + Duration::hours(24);
let session = AdminSession {
token: token.clone(),
username: username.to_string(),
created_at: now.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
};
let mut admin_sessions = self.admin_sessions.lock().unwrap();
admin_sessions.insert(token.clone(), session);
log::info!("Admin {} logged in successfully", username);
Some(AdminLoginResponse {
token,
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
username: username.to_string(),
})
} else {
log::warn!("Invalid password for admin {}", username);
None
}
}
Ok(Some(_)) => {
log::warn!("Admin {} is not active", username);
None
}
Ok(None) => {
log::warn!("Admin {} not found", username);
None
}
Err(e) => {
log::error!("Failed to get admin {}: {}", username, e);
None
}
}
} else {
log::warn!("Auth DB not available for admin login");
None
}
}
pub fn verify_admin_token(&self, token: &str) -> Option<AdminSession> {
let admin_sessions = self.admin_sessions.lock().unwrap();
if let Some(session) = admin_sessions.get(token) {
let expires_at = chrono::DateTime::parse_from_rfc3339(&session.expires_at)
.ok()
.map(|dt| dt.with_timezone(&Utc));
if let Some(exp) = expires_at {
if Utc::now() < exp {
return Some(session.clone());
} else {
log::warn!("Admin token {} has expired", token);
}
}
}
None
}
pub fn login_with_sync(&self, username: &str, password: &str) -> Option<LoginResponse> {
if let Some(auth_db) = &self.auth_db {
// Get user from auth.sqlite
let user = match auth_db.get_user(username) {
Ok(Some(user)) => user,
Ok(None) => {
log::warn!("User {} not found in auth database", username);
return None;
}
Err(e) => {
log::error!("Failed to get user {}: {}", username, e);
return None;
}
};
if user.status != 1 {
log::warn!("User {} is disabled", username);
return None;
}
if verify(password, &user.password_hash).unwrap_or(false) {
let groups = auth_db.get_user_groups(username).unwrap_or_default();
let permissions = user.permissions.clone();
let token = Uuid::new_v4().to_string();
let now = Utc::now();
let expires_at = now + Duration::hours(24);
let session = Session {
token: token.clone(),
user_id: username.to_string(),
username: username.to_string(),
created_at: now.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
groups: groups.clone(),
permissions: permissions.clone(),
};
let mut sessions = self.sessions.lock().unwrap();
sessions.insert(token.clone(), session);
log::info!("User {} logged in successfully", username);
Some(LoginResponse {
token,
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
user_id: username.to_string(),
groups,
permissions,
})
} else {
log::warn!("Invalid password for user {}", username);
None
}
} else {
self.login(username, password)
}
}
pub fn verify_token(&self, token: &str) -> Option<Session> {
let sessions = self.sessions.lock().unwrap();
let session = sessions.get(token)?;
// Check expiration
let expires_at = chrono::DateTime::parse_from_rfc3339(&session.expires_at)
.ok()?
.with_timezone(&Utc);
if Utc::now() > expires_at {
return None;
}
Some(session.clone())
}
pub fn logout(&self, token: &str) -> bool {
let mut sessions = self.sessions.lock().unwrap();
sessions.remove(token).is_some()
}
pub fn create_user(&self, username: &str, password: &str) -> Result<String, String> {
let mut users = self.users.lock().unwrap();
if users.contains_key(username) {
return Err("User already exists".to_string());
}
let password_hash = hash(password, DEFAULT_COST)
.map_err(|e| e.to_string())?;
let user_id = Uuid::new_v4().to_string();
let user = User {
user_id: user_id.clone(),
username: username.to_string(),
password_hash,
created_at: Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
};
users.insert(username.to_string(), user);
Ok(user_id)
}
}
// Authorization header parser
pub fn parse_auth_header(header: &str) -> Option<String> {
if header.starts_with("Bearer ") {
Some(header.trim_start_matches("Bearer ").to_string())
} else {
None
}
}

View File

@@ -0,0 +1,61 @@
use axum::response::Json;
use std::sync::Mutex;
use crate::audio;
static CMD_QUEUE: Mutex<Vec<(String, Option<String>)>> = Mutex::new(Vec::new());
pub async fn post_command(
Json(body): Json<serde_json::Value>,
) -> impl axum::response::IntoResponse {
let cmd = body["cmd"].as_str().unwrap_or("").to_string();
let val = body["val"].as_str().map(|s| s.to_string());
let out = body["out"].as_str().map(|s| s.to_string());
if cmd == "test_voice" {
let lang = val.as_deref().unwrap_or("zh_TW");
let voice_name = audio::voice_for_lang(lang);
let phrase = audio::phrase_for_lang(lang);
if let Some(d) = out.as_deref() {
if !d.is_empty() {
std::process::Command::new("SwitchAudioSource")
.args(["-t", "output", "-s", d])
.output()
.ok();
}
}
std::process::Command::new("say")
.args(["-v", &voice_name, &phrase])
.spawn()
.ok();
} else if cmd == "vol_up" {
std::process::Command::new("osascript")
.args([
"-e",
"set volume output volume (output volume of (get volume settings)) + 10",
])
.output()
.ok();
} else if cmd == "vol_down" {
std::process::Command::new("osascript")
.args([
"-e",
"set volume output volume (output volume of (get volume settings)) - 10",
])
.output()
.ok();
} else {
CMD_QUEUE.lock().unwrap().push((cmd, val));
}
Json(serde_json::json!({"ok": true}))
}
pub async fn get_commands() -> Json<serde_json::Value> {
let mut queue = CMD_QUEUE.lock().unwrap();
let cmds: Vec<_> = queue
.drain(..)
.map(|(c, v)| serde_json::json!({"cmd": c, "val": v}))
.collect();
Json(serde_json::json!(cmds))
}

263
markbase-core/src/config.rs Normal file
View File

@@ -0,0 +1,263 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MarkBaseConfig {
pub server: ServerConfig,
pub postgresql: PostgreSQLConfig,
pub authentication: AuthenticationConfig,
pub test: TestConfig,
pub logging: LoggingConfig,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ServerConfig {
pub host: String,
pub port: u16,
pub log_level: String,
pub auth_db_path: String,
pub users_db_dir: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PostgreSQLConfig {
pub host: String,
pub port: u16,
pub user: String,
pub password: String,
pub database: String,
pub connection_pool_size: u8,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuthenticationConfig {
pub bcrypt_cost: u32,
pub token_validity_hours: u8,
pub session_storage: String,
pub max_sessions_per_user: u8,
pub default_user: String,
pub default_password: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TestConfig {
pub users: Vec<String>,
pub password: String,
pub login_test_iterations: u16,
pub verify_test_iterations: u16,
pub api_test_iterations: u16,
pub performance_report: bool,
pub output_format: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoggingConfig {
pub level: String,
pub file_path: String,
pub console_output: bool,
pub structured_logging: bool,
}
impl MarkBaseConfig {
pub fn load(path: &Path) -> Result<Self> {
let content = std::fs::read_to_string(path)?;
let config: MarkBaseConfig = toml::from_str(&content)?;
Ok(config)
}
pub fn save(&self, path: &Path) -> Result<()> {
let content = toml::to_string_pretty(self)?;
std::fs::write(path, content)?;
Ok(())
}
pub fn default_config() -> Self {
Self {
server: ServerConfig {
host: "127.0.0.1".to_string(),
port: 11438,
log_level: "info".to_string(),
auth_db_path: "data/auth.sqlite".to_string(),
users_db_dir: "data/users".to_string(),
},
postgresql: PostgreSQLConfig {
host: "127.0.0.1".to_string(),
port: 5432,
user: "sftpgo".to_string(),
password: "sftpgo_pass_2026".to_string(),
database: "sftpgo".to_string(),
connection_pool_size: 5,
},
authentication: AuthenticationConfig {
bcrypt_cost: 10,
token_validity_hours: 24,
session_storage: "memory".to_string(),
max_sessions_per_user: 5,
default_user: "demo".to_string(),
default_password: "demo123".to_string(),
},
test: TestConfig {
users: vec!["warren".to_string(), "momentry".to_string(), "demo".to_string()],
password: "demo123".to_string(),
login_test_iterations: 10,
verify_test_iterations: 100,
api_test_iterations: 50,
performance_report: true,
output_format: "markdown".to_string(),
},
logging: LoggingConfig {
level: "info".to_string(),
file_path: "logs/markbase.log".to_string(),
console_output: true,
structured_logging: false,
},
}
}
pub fn merge_env(&mut self) {
if let Ok(host) = std::env::var("MB_HOST") {
self.server.host = host;
}
if let Ok(port) = std::env::var("MB_PORT") {
if let Ok(p) = port.parse() {
self.server.port = p;
}
}
if let Ok(log_level) = std::env::var("MB_LOG_LEVEL") {
self.server.log_level = log_level;
}
if let Ok(pg_host) = std::env::var("PG_HOST") {
self.postgresql.host = pg_host;
}
if let Ok(pg_port) = std::env::var("PG_PORT") {
if let Ok(p) = pg_port.parse() {
self.postgresql.port = p;
}
}
if let Ok(pg_user) = std::env::var("PG_USER") {
self.postgresql.user = pg_user;
}
if let Ok(pg_password) = std::env::var("PG_PASSWORD") {
self.postgresql.password = pg_password;
}
if let Ok(pg_database) = std::env::var("PG_DATABASE") {
self.postgresql.database = pg_database;
}
if let Ok(bcrypt_cost) = std::env::var("MB_BCRYPT_COST") {
if let Ok(c) = bcrypt_cost.parse() {
self.authentication.bcrypt_cost = c;
}
}
if let Ok(token_hours) = std::env::var("MB_TOKEN_VALIDITY_HOURS") {
if let Ok(h) = token_hours.parse() {
self.authentication.token_validity_hours = h;
}
}
}
pub fn get(&self, key: &str) -> Option<String> {
match key {
"server.host" => Some(self.server.host.clone()),
"server.port" => Some(self.server.port.to_string()),
"server.log_level" => Some(self.server.log_level.clone()),
"server.auth_db_path" => Some(self.server.auth_db_path.clone()),
"server.users_db_dir" => Some(self.server.users_db_dir.clone()),
"postgresql.host" => Some(self.postgresql.host.clone()),
"postgresql.port" => Some(self.postgresql.port.to_string()),
"postgresql.user" => Some(self.postgresql.user.clone()),
"postgresql.password" => Some(self.postgresql.password.clone()),
"postgresql.database" => Some(self.postgresql.database.clone()),
"postgresql.connection_pool_size" => Some(self.postgresql.connection_pool_size.to_string()),
"authentication.bcrypt_cost" => Some(self.authentication.bcrypt_cost.to_string()),
"authentication.token_validity_hours" => Some(self.authentication.token_validity_hours.to_string()),
"authentication.session_storage" => Some(self.authentication.session_storage.clone()),
"authentication.max_sessions_per_user" => Some(self.authentication.max_sessions_per_user.to_string()),
"authentication.default_user" => Some(self.authentication.default_user.clone()),
"authentication.default_password" => Some(self.authentication.default_password.clone()),
"test.users" => Some(serde_json::to_string(&self.test.users).unwrap_or_default()),
"test.password" => Some(self.test.password.clone()),
"test.login_test_iterations" => Some(self.test.login_test_iterations.to_string()),
"test.verify_test_iterations" => Some(self.test.verify_test_iterations.to_string()),
"test.api_test_iterations" => Some(self.test.api_test_iterations.to_string()),
"test.performance_report" => Some(self.test.performance_report.to_string()),
"test.output_format" => Some(self.test.output_format.clone()),
"logging.level" => Some(self.logging.level.clone()),
"logging.file_path" => Some(self.logging.file_path.clone()),
"logging.console_output" => Some(self.logging.console_output.to_string()),
"logging.structured_logging" => Some(self.logging.structured_logging.to_string()),
_ => None,
}
}
pub fn set(&mut self, key: &str, value: &str) -> Result<()> {
match key {
"server.host" => self.server.host = value.to_string(),
"server.port" => self.server.port = value.parse()?,
"server.log_level" => self.server.log_level = value.to_string(),
"server.auth_db_path" => self.server.auth_db_path = value.to_string(),
"server.users_db_dir" => self.server.users_db_dir = value.to_string(),
"postgresql.host" => self.postgresql.host = value.to_string(),
"postgresql.port" => self.postgresql.port = value.parse()?,
"postgresql.user" => self.postgresql.user = value.to_string(),
"postgresql.password" => self.postgresql.password = value.to_string(),
"postgresql.database" => self.postgresql.database = value.to_string(),
"postgresql.connection_pool_size" => self.postgresql.connection_pool_size = value.parse()?,
"authentication.bcrypt_cost" => self.authentication.bcrypt_cost = value.parse()?,
"authentication.token_validity_hours" => self.authentication.token_validity_hours = value.parse()?,
"authentication.session_storage" => self.authentication.session_storage = value.to_string(),
"authentication.max_sessions_per_user" => self.authentication.max_sessions_per_user = value.parse()?,
"authentication.default_user" => self.authentication.default_user = value.to_string(),
"authentication.default_password" => self.authentication.default_password = value.to_string(),
"test.password" => self.test.password = value.to_string(),
"test.login_test_iterations" => self.test.login_test_iterations = value.parse()?,
"test.verify_test_iterations" => self.test.verify_test_iterations = value.parse()?,
"test.api_test_iterations" => self.test.api_test_iterations = value.parse()?,
"test.performance_report" => self.test.performance_report = value.parse()?,
"test.output_format" => self.test.output_format = value.to_string(),
"logging.level" => self.logging.level = value.to_string(),
"logging.file_path" => self.logging.file_path = value.to_string(),
"logging.console_output" => self.logging.console_output = value.parse()?,
"logging.structured_logging" => self.logging.structured_logging = value.parse()?,
_ => return Err(anyhow::anyhow!("Invalid config key: {}", key)),
}
Ok(())
}
pub fn validate(&self) -> Result<()> {
if self.server.port < 1024 {
return Err(anyhow::anyhow!("Invalid server port: {}. Must be >= 1024", self.server.port));
}
if self.postgresql.port == 0 {
return Err(anyhow::anyhow!("Invalid PostgreSQL port: {}", self.postgresql.port));
}
if self.authentication.bcrypt_cost < 4 || self.authentication.bcrypt_cost > 31 {
return Err(anyhow::anyhow!("Invalid bcrypt_cost: {}. Must be 4-31", self.authentication.bcrypt_cost));
}
if self.authentication.token_validity_hours == 0 {
return Err(anyhow::anyhow!("Invalid token_validity_hours: {}. Must be >= 1",
self.authentication.token_validity_hours));
}
if self.test.users.is_empty() {
return Err(anyhow::anyhow!("test.users must not be empty"));
}
Ok(())
}
}

17
markbase-core/src/lib.rs Normal file
View File

@@ -0,0 +1,17 @@
pub mod audio;
pub mod auth;
pub mod command;
pub mod config;
pub mod filetree;
pub mod fskit;
pub mod fuse;
pub mod nfs;
pub mod pg_client;
pub mod raid;
pub mod render;
pub mod scan;
pub mod server;
pub mod sync;
pub mod webdav;
pub use filetree::node::FileNode;

274
markbase-core/src/main.rs Normal file
View File

@@ -0,0 +1,274 @@
use clap::{Parser, Subcommand};
use std::path::Path;
#[derive(Parser)]
#[command(name = "markbase", about = "Momentry Display Engine")]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Start display server
Display {
#[arg(short, long, default_value = "11438")]
port: u16,
/// Optional initial markdown file
file: Option<String>,
},
/// Render markdown to HTML (stdout)
Render {
file: String,
#[arg(short, long)]
output: Option<String>,
},
/// Configuration management
Config {
#[command(subcommand)]
action: ConfigCommands,
},
/// Scan and import files from directory
Scan {
/// User ID
#[arg(short, long)]
user: String,
/// Directory to scan
#[arg(short, long)]
dir: String,
/// Batch size for database insertion
#[arg(short, long, default_value = "100")]
batch: usize,
/// Skip SHA256 hash calculation (faster import)
#[arg(short, long, default_value = "true")]
skip_hash: bool,
/// Number of threads for hash calculation (if skip_hash=false)
#[arg(short, long, default_value = "4")]
threads: usize,
},
/// Compute SHA256 hashes for imported files
Hash {
/// User ID
#[arg(short, long)]
user: String,
/// Number of threads for parallel hash calculation
#[arg(short, long, default_value = "4")]
threads: usize,
},
}
#[derive(Subcommand)]
enum ConfigCommands {
/// Initialize default configuration file
Init {
#[arg(short, long)]
force: bool,
},
/// Show current configuration
Show {
#[arg(short, long)]
section: Option<String>,
},
/// Edit configuration
Edit {
#[arg(short, long)]
key: String,
#[arg(short, long)]
value: String,
},
/// Validate configuration
Validate,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Display { port, file } => {
markbase::server::run(port, file).await?;
}
Commands::Render { file, output } => {
let md = std::fs::read_to_string(&file)?;
let html = markbase::render::md_to_html(&md);
if let Some(path) = &output {
std::fs::write(path, html)?;
} else {
println!("{html}");
}
}
Commands::Config { action } => {
handle_config_command(action)?;
}
Commands::Scan { user, dir, batch, skip_hash, threads } => {
use markbase::scan::ScanOptions;
let options = ScanOptions {
skip_hash,
threads,
};
markbase::scan::scan_directory(&user, &dir, batch, options)?;
}
Commands::Hash { user, threads } => {
markbase::scan::compute_hashes(&user, threads)?;
}
Commands::WebDAV { action } => {
handle_webdav_command(action)?;
}
}
Ok(())
}
fn handle_config_command(action: ConfigCommands) -> anyhow::Result<()> {
match action {
ConfigCommands::Init { force } => {
let config_path = Path::new("config/markbase.toml");
if config_path.exists() && !force {
println!("Configuration file already exists at config/markbase.toml");
println!("Use --force to overwrite");
return Ok(());
}
let config = markbase::config::MarkBaseConfig::default_config();
config.save(config_path)?;
println!("✓ Configuration file created: config/markbase.toml");
println!("Default values:");
println!(" Server port: {}", config.server.port);
println!(" PostgreSQL host: {}", config.postgresql.host);
println!(" Test users: {}", config.test.users.join(", "));
}
ConfigCommands::Show { section } => {
let config_path = Path::new("config/markbase.toml");
if !config_path.exists() {
println!("Configuration file not found. Run 'markbase config init' first.");
return Ok(());
}
let config = markbase::config::MarkBaseConfig::load(config_path)?;
if let Some(s) = section {
show_section(&config, &s);
} else {
println!("{}", toml::to_string_pretty(&config)?);
}
}
ConfigCommands::Edit { key, value } => {
let config_path = Path::new("config/markbase.toml");
if !config_path.exists() {
println!("Configuration file not found. Run 'markbase config init' first.");
return Ok(());
}
let mut config = markbase::config::MarkBaseConfig::load(config_path)?;
match config.get(&key) {
Some(old_value) => {
config.set(&key, &value)?;
config.validate()?;
config.save(config_path)?;
println!("✓ Updated {}: {}{}", key, old_value, value);
}
None => {
println!("Invalid config key: {}", key);
println!("Valid keys: server.*, postgresql.*, authentication.*, test.*, logging.*");
}
}
}
ConfigCommands::Validate => {
let config_path = Path::new("config/markbase.toml");
if !config_path.exists() {
println!("Configuration file not found. Run 'markbase config init' first.");
return Ok(());
}
let config = markbase::config::MarkBaseConfig::load(config_path)?;
match config.validate() {
Ok(_) => {
println!("✓ Configuration is valid");
}
Err(e) => {
println!("✗ Configuration validation failed: {}", e);
}
}
}
}
Ok(())
}
fn show_section(config: &markbase::config::MarkBaseConfig, section: &str) {
match section {
"server" => println!("{}", toml::to_string_pretty(&config.server).unwrap()),
"postgresql" => println!("{}", toml::to_string_pretty(&config.postgresql).unwrap()),
"authentication" => println!("{}", toml::to_string_pretty(&config.authentication).unwrap()),
"test" => println!("{}", toml::to_string_pretty(&config.test).unwrap()),
"logging" => println!("{}", toml::to_string_pretty(&config.logging).unwrap()),
_ => println!("Invalid section: {}. Valid sections: server, postgresql, authentication, test, logging", section),
}
}
fn handle_webdav_command(action: WebDAVCommands) -> anyhow::Result<()> {
match action {
WebDAVCommands::Start { port, user } => {
use std::path::PathBuf;
use std::sync::Arc;
use markbase::webdav::MarkBaseWebDAV;
use markbase::filetree::FileTree;
use tokio::net::TcpListener;
let db_path = PathBuf::from(FileTree::user_db_path(&user));
if !db_path.exists() {
return Err(anyhow::anyhow!("User database not found: {}", db_path.display()));
}
println!("=== MarkBase WebDAV Server ===");
println!("User: {}", user);
println!("Port: {}", port);
println!("Database: {}", db_path.display());
println!("");
let webdav = MarkBaseWebDAV::new(user.clone(), db_path);
let dav_handler = webdav.create_handler();
let addr = format!("127.0.0.1:{}", port);
println!("Listening on: {}", addr);
println!("Mount with Finder:");
println!(" Connect to Server → http://localhost:{}/webdav", port);
println!("");
println!("Press Ctrl+C to stop...");
tokio::spawn(async move {
use axum::{Router, Extension, routing::any};
let app = Router::new()
.route("/webdav/*path", any(|req: axum::http::Request<axum::body::Body>, Extension(h): Extension<Arc<dav_server::DavHandler>>| async move {
use http_body_util::BodyExt;
let body = req.into_body().collect().await.unwrap().to_bytes();
let req = http::Request::new(body);
h.handle(req).await
}))
.route("/webdav", any(|req: axum::http::Request<axum::body::Body>, Extension(h): Extension<Arc<dav_server::DavHandler>>| async move {
use http_body_util::BodyExt;
let body = req.into_body().collect().await.unwrap().to_bytes();
let req = http::Request::new(body);
h.handle(req).await
}))
.layer(Extension(dav_handler));
let listener = TcpListener::bind(&addr).await.unwrap();
axum::serve(listener, app).await.unwrap();
});
tokio::signal::ctrl_c().await?;
println!("\nShutting down...");
}
}
Ok(())
}

1149
markbase-core/src/page.html Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,303 @@
use anyhow::Result;
use tokio_postgres::{NoTls, Client};
use crate::sync::{PgUser, PgGroup, PgUserGroupMapping, PgAdmin};
pub struct PgClient {
host: String,
port: u16,
user: String,
password: String,
database: String,
}
impl PgClient {
pub fn new() -> Self {
Self {
host: "127.0.0.1".to_string(),
port: 5432,
user: "sftpgo".to_string(),
password: "sftpgo_pass_2026".to_string(),
database: "sftpgo".to_string(),
}
}
pub fn from_env() -> Self {
Self {
host: std::env::var("PG_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
port: std::env::var("PG_PORT")
.unwrap_or_else(|_| "5432".to_string())
.parse()
.unwrap_or(5432),
user: std::env::var("PG_USER").unwrap_or_else(|_| "sftpgo".to_string()),
password: std::env::var("PG_PASSWORD")
.unwrap_or_else(|_| "sftpgo_pass_2026".to_string()),
database: std::env::var("PG_DATABASE")
.unwrap_or_else(|_| "sftpgo".to_string()),
}
}
pub async fn connect(&self) -> Result<Client> {
let config = format!(
"host={} port={} user={} password={} dbname={}",
self.host, self.port, self.user, self.password, self.database
);
let (client, connection) = tokio_postgres::connect(&config, NoTls).await?;
tokio::spawn(async move {
if let Err(e) = connection.await {
log::error!("PostgreSQL connection error: {}", e);
}
});
Ok(client)
}
pub async fn fetch_users(&self) -> Result<Vec<PgUser>> {
let client = self.connect().await?;
let rows = client.query(
"SELECT username, password, email, status, home_dir, permissions,
uid, gid, last_login, created_at, updated_at
FROM users
WHERE status = 1 AND deleted_at = 0",
&[]
).await?;
let users = rows
.into_iter()
.map(|row| PgUser {
username: row.get::<_, String>(0),
password_hash: row.get::<_, String>(1),
email: row.get::<_, Option<String>>(2),
status: row.get::<_, i32>(3),
home_dir: row.get::<_, String>(4),
permissions: row.get::<_, String>(5),
uid: row.get::<_, i64>(6),
gid: row.get::<_, i64>(7),
last_login: row.get::<_, i64>(8),
created_at: row.get::<_, i64>(9),
updated_at: row.get::<_, i64>(10),
})
.collect();
Ok(users)
}
pub async fn fetch_groups(&self) -> Result<Vec<PgGroup>> {
let client = self.connect().await?;
let rows = client.query(
"SELECT name, description, created_at, updated_at FROM groups",
&[]
).await?;
let groups = rows
.into_iter()
.map(|row| PgGroup {
name: row.get::<_, String>(0),
description: row.get::<_, Option<String>>(1),
created_at: row.get::<_, i64>(2),
updated_at: row.get::<_, i64>(3),
})
.collect();
Ok(groups)
}
pub async fn fetch_admins(&self) -> Result<Vec<PgAdmin>> {
let client = self.connect().await?;
let rows = client
.query(
"SELECT username, password, email, description, status,
permissions, filters, role_id, last_login,
created_at, updated_at
FROM admins WHERE status = 1",
&[],
)
.await?;
let admins = rows
.into_iter()
.map(|row| PgAdmin {
username: row.get::<_, String>(0),
password_hash: row.get::<_, String>(1),
email: row.get::<_, Option<String>>(2),
description: row.get::<_, Option<String>>(3),
status: row.get::<_, i32>(4),
permissions: row.get::<_, String>(5),
filters: row.get::<_, Option<String>>(6),
role_id: row.get::<_, Option<i32>>(7),
last_login: row.get::<_, i64>(8),
created_at: row.get::<_, i64>(9),
updated_at: row.get::<_, i64>(10),
})
.collect();
Ok(admins)
}
pub async fn fetch_mappings(&self) -> Result<Vec<PgUserGroupMapping>> {
let client = self.connect().await?;
let rows = client.query(
"SELECT u.username, g.name
FROM users_groups_mapping ug
JOIN users u ON ug.user_id = u.id
JOIN groups g ON ug.group_id = g.id
WHERE u.status = 1",
&[]
).await?;
let mappings = rows
.into_iter()
.map(|row| PgUserGroupMapping {
username: row.get::<_, String>(0),
group_name: row.get::<_, String>(1),
})
.collect();
Ok(mappings)
}
}
pub struct SftpGoSync {
pg_client: PgClient,
auth_db: crate::sync::AuthDb,
}
impl SftpGoSync {
pub fn new(auth_db_path: &str) -> Result<Self> {
Ok(Self {
pg_client: PgClient::new(),
auth_db: crate::sync::AuthDb::new(auth_db_path)?,
})
}
pub async fn full_sync(&self) -> Result<crate::sync::SyncResult> {
let mut result = crate::sync::SyncResult::default();
result.sync_type = "full".to_string();
result.sync_time = chrono::Utc::now().timestamp();
log::info!("Starting full sync from SFTPGo PostgreSQL");
// 1. Sync users
match self.pg_client.fetch_users().await {
Ok(users) => {
log::info!("Fetched {} users from PostgreSQL", users.len());
for user in users {
match self.auth_db.save_user(&user) {
Ok(_) => result.users_synced += 1,
Err(e) => {
result.users_failed += 1;
result.errors.push(format!("User {} sync failed: {}", user.username, e));
log::error!("Failed to sync user {}: {}", user.username, e);
}
}
}
}
Err(e) => {
log::error!("Failed to fetch users from PostgreSQL: {}", e);
result.errors.push(format!("PG users fetch failed: {}", e));
result.users_failed = 1;
}
}
// 2. Sync groups
match self.pg_client.fetch_groups().await {
Ok(groups) => {
log::info!("Fetched {} groups from PostgreSQL", groups.len());
for group in groups {
match self.auth_db.save_group(&group) {
Ok(_) => result.groups_synced += 1,
Err(e) => {
result.groups_failed += 1;
result.errors.push(format!("Group {} sync failed: {}", group.name, e));
log::error!("Failed to sync group {}: {}", group.name, e);
}
}
}
}
Err(e) => {
log::error!("Failed to fetch groups from PostgreSQL: {}", e);
result.errors.push(format!("PG groups fetch failed: {}", e));
result.groups_failed = 1;
}
}
// 3. Sync mappings
match self.pg_client.fetch_mappings().await {
Ok(mappings) => {
log::info!("Fetched {} mappings from PostgreSQL", mappings.len());
for mapping in mappings {
match self.auth_db.save_mapping(&mapping) {
Ok(_) => result.mappings_synced += 1,
Err(e) => {
result.mappings_failed += 1;
result.errors.push(format!(
"Mapping {}->{} sync failed: {}",
mapping.username, mapping.group_name, e
));
log::error!(
"Failed to sync mapping {}->{}: {}",
mapping.username, mapping.group_name, e
);
}
}
}
}
Err(e) => {
log::error!("Failed to fetch mappings from PostgreSQL: {}", e);
result.errors.push(format!("PG mappings fetch failed: {}", e));
result.mappings_failed = 1;
}
}
// 4. Sync admins
match self.pg_client.fetch_admins().await {
Ok(admins) => {
let admins_count = admins.len();
log::info!("Fetched {} admins from PostgreSQL", admins_count);
match self.auth_db.sync_admins(admins) {
Ok(count) => result.admins_synced = count,
Err(e) => {
result.admins_failed = admins_count;
result.errors.push(format!("Admins sync failed: {}", e));
log::error!("Failed to sync admins: {}", e);
}
}
}
Err(e) => {
log::error!("Failed to fetch admins from PostgreSQL: {}", e);
result.errors.push(format!("PG admins fetch failed: {}", e));
result.admins_failed = 1;
}
}
// 5. Determine final status
if result.users_failed > 0 || result.groups_failed > 0 || result.mappings_failed > 0 || result.admins_failed > 0 {
if result.users_synced > 0 || result.groups_synced > 0 || result.mappings_synced > 0 || result.admins_synced > 0 {
result.status = "partial_success".to_string();
} else {
result.status = "cached".to_string();
}
} else {
result.status = "success".to_string();
}
// 6. Save sync log
self.auth_db.save_sync_log(&result)?;
log::info!(
"Sync completed: users={}, groups={}, mappings={}, admins={}, status={}",
result.users_synced,
result.groups_synced,
result.mappings_synced,
result.admins_synced,
result.status
);
Ok(result)
}
}

View File

@@ -0,0 +1,31 @@
use pulldown_cmark::{html, Options, Parser};
pub fn md_to_html(content: &str) -> String {
let mut opts = Options::empty();
opts.insert(Options::ENABLE_TABLES);
opts.insert(Options::ENABLE_FOOTNOTES);
opts.insert(Options::ENABLE_STRIKETHROUGH);
opts.insert(Options::ENABLE_TASKLISTS);
opts.insert(Options::ENABLE_HEADING_ATTRIBUTES);
let parser = Parser::new_ext(content, opts);
let mut body = String::new();
html::push_html(&mut body, parser);
body
}
const HTML: &str = include_str!("page.html");
pub fn page(title: &str, content: &str) -> String {
HTML.replace("{__TITLE__}", title)
.replace("{__CONTENT__}", content)
}
pub fn render_page(title: &str, content: &str) -> String {
let content = content
.replace(
"<code class=\"language-mermaid\">",
"<div class=\"mermaid\">",
)
.replace("</code>", "</div>");
page(title, &content).replace("startOnLoad:false", "startOnLoad:true")
}

595
markbase-core/src/scan.rs Normal file
View File

@@ -0,0 +1,595 @@
use anyhow::{Context, Result};
use rusqlite::Connection;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::Instant;
use crate::filetree::node::{Aliases, FileNode, NodeType};
use crate::filetree::FileTree;
pub struct ScanOptions {
pub skip_hash: bool,
pub threads: usize,
}
impl Default for ScanOptions {
fn default() -> Self {
ScanOptions {
skip_hash: true,
threads: 4,
}
}
}
pub fn scan_directory(user_id: &str, dir: &str, batch_size: usize, options: ScanOptions) -> Result<()> {
let start = Instant::now();
let dir_path = Path::new(dir);
if !dir_path.exists() {
anyhow::bail!("Directory not found: {}", dir);
}
println!("=== File Scan Performance Test ===");
println!("User ID: {}", user_id);
println!("Directory: {}", dir);
println!("Batch size: {}", batch_size);
println!("Skip hash: {}", options.skip_hash);
if !options.skip_hash {
println!("Hash threads: {}", options.threads);
}
println!();
println!("[1/4] Scanning directory structure...");
let scan_start = Instant::now();
let mut folders: Vec<(String, String, Option<String>)> = Vec::new();
let mut files: Vec<(String, String, u64, String)> = Vec::new();
scan_recursive(dir_path, dir_path, &mut folders, &mut files)?;
let scan_duration = scan_start.elapsed();
println!(" Scanned {} folders, {} files in {:.2}s",
folders.len(), files.len(), scan_duration.as_secs_f64());
println!();
println!("[2/5] Generating node IDs...");
let id_start = Instant::now();
let mac = get_mac_address()?;
let mut folder_nodes: Vec<FileNode> = Vec::new();
let mut file_nodes: Vec<FileNode> = Vec::new();
let mut file_info: Vec<(String, String)> = Vec::new();
let mac_str = get_mac_address()?;
let root_node_id = generate_uuid(&dir_path.to_string_lossy(), "Home", &mac_str, chrono::Utc::now().timestamp() as u64);
folder_nodes.push(FileNode {
node_id: root_node_id.clone(),
label: "Home".to_string(),
aliases: Aliases::empty(),
file_uuid: None,
sha256: None,
parent_id: None,
children: Vec::new(),
node_type: NodeType::Folder,
icon: Some("🏠".to_string()),
color: None,
bg_color: None,
file_size: None,
registered_at: None,
created_at: chrono::Utc::now().timestamp().to_string(),
updated_at: chrono::Utc::now().timestamp().to_string(),
sort_order: 0,
});
let folder_id_map: HashMap<String, String> = {
let mut map = HashMap::new();
map.insert(dir_path.to_string_lossy().to_string(), root_node_id.clone());
for (path_str, label, _parent_path) in &folders {
let mtime = fs::metadata(path_str)
.and_then(|m| m.modified())
.unwrap_or(std::time::SystemTime::UNIX_EPOCH);
let mtime_secs = mtime.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let node_id = generate_uuid(path_str, label, &mac_str, mtime_secs);
map.insert(path_str.clone(), node_id);
}
map
};
for (path_str, label, parent_path) in &folders {
let node_id = folder_id_map.get(path_str).cloned().unwrap();
let parent_node_id = if let Some(ref parent_p) = parent_path {
folder_id_map.get(parent_p).cloned()
} else {
Some(root_node_id.clone())
};
folder_nodes.push(FileNode {
node_id,
label: label.clone(),
aliases: Aliases::empty(),
file_uuid: None,
sha256: None,
parent_id: parent_node_id,
children: Vec::new(),
node_type: NodeType::Folder,
icon: Some("📁".to_string()),
color: None,
bg_color: None,
file_size: None,
registered_at: None,
created_at: chrono::Utc::now().timestamp().to_string(),
updated_at: chrono::Utc::now().timestamp().to_string(),
sort_order: 0,
});
}
for (path_str, filename, size, _ext) in &files {
let mtime = fs::metadata(path_str)
.and_then(|m| m.modified())
.unwrap_or(std::time::SystemTime::UNIX_EPOCH);
let mtime_secs = mtime.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let node_id = generate_uuid(path_str, filename, &mac, mtime_secs);
let file_dir = Path::new(path_str).parent().unwrap_or(dir_path);
let parent_node_id = if file_dir == dir_path {
Some(root_node_id.clone())
} else {
folder_id_map.get(file_dir.to_string_lossy().as_ref()).cloned()
};
let node_id_clone = node_id.clone();
file_info.push((node_id_clone.clone(), path_str.clone()));
file_nodes.push(FileNode {
node_id: node_id_clone.clone(),
label: filename.clone(),
aliases: {
let mut aliases = Aliases::empty();
aliases.set("path", path_str);
aliases
},
file_uuid: Some(node_id_clone.clone()),
sha256: None,
parent_id: parent_node_id,
children: Vec::new(),
node_type: NodeType::File,
icon: get_file_icon(filename),
color: None,
bg_color: None,
file_size: Some(*size as i64),
registered_at: Some(chrono::Utc::now().timestamp().to_string()),
created_at: chrono::Utc::now().timestamp().to_string(),
updated_at: chrono::Utc::now().timestamp().to_string(),
sort_order: 0,
});
}
let id_duration = id_start.elapsed();
println!(" Generated {} folder IDs, {} file IDs in {:.2}s",
folder_nodes.len(), file_nodes.len(), id_duration.as_secs_f64());
println!();
println!("[3/5] Opening database...");
let db_start = Instant::now();
let db_path = FileTree::user_db_path(user_id);
if !Path::new(&db_path).exists() {
FileTree::init_user_db(user_id)?;
}
let conn = FileTree::open_user_db(user_id)
.with_context(|| format!("Failed to open database for user {}", user_id))?;
let db_duration = db_start.elapsed();
println!(" Database opened in {:.2}s", db_duration.as_secs_f64());
println!();
println!("[4/5] Inserting nodes (batch size: {})...", batch_size);
let insert_start = Instant::now();
let tx = conn.unchecked_transaction()?;
let folder_count = folder_nodes.len();
let file_count = file_nodes.len();
let total_nodes = folder_count + file_count;
let mut inserted = 0;
for node in folder_nodes {
insert_node(&conn, &node)?;
inserted += 1;
if inserted % batch_size == 0 {
print!("\r Inserted {}/{} nodes...", inserted, total_nodes);
use std::io::Write;
std::io::stdout().flush().ok();
}
}
for node in file_nodes {
insert_node(&conn, &node)?;
if let Some(ref file_uuid) = node.file_uuid {
let path = node.aliases.get("path").cloned().unwrap_or_default();
if !path.is_empty() {
conn.execute(
"INSERT OR IGNORE INTO file_locations (file_uuid, location, label, added_at)
VALUES (?1, ?2, 'origin', ?3)",
rusqlite::params![file_uuid, path, chrono::Utc::now().timestamp().to_string()],
)?;
}
}
inserted += 1;
if inserted % batch_size == 0 {
print!("\r Inserted {}/{} nodes...", inserted, total_nodes);
use std::io::Write;
std::io::stdout().flush().ok();
}
}
tx.commit()?;
let insert_duration = insert_start.elapsed();
println!("\r Inserted {} nodes in {:.2}s ({:.0} nodes/sec)",
total_nodes,
insert_duration.as_secs_f64(),
total_nodes as f64 / insert_duration.as_secs_f64());
println!();
println!("[5/5] Updating folder children_json...");
let children_start = Instant::now();
conn.execute(
"UPDATE file_nodes
SET children_json = (
SELECT json_group_array(node_id)
FROM file_nodes AS child
WHERE child.parent_id = file_nodes.node_id
)
WHERE node_type = 'folder'",
[],
)?;
let children_duration = children_start.elapsed();
println!(" Updated children_json for {} folders in {:.2}s",
folder_count,
children_duration.as_secs_f64());
let total_duration = start.elapsed();
println!();
println!("=== Summary ===");
println!("Total time: {:.2}s", total_duration.as_secs_f64());
println!("Folders: {}", folder_count);
println!("Files: {}", file_count);
println!("Total nodes: {}", total_nodes);
println!("Database: {}", FileTree::user_db_path(user_id));
println!();
println!("Performance breakdown:");
println!(" - Scanning: {:.2}s ({:.0}%)",
scan_duration.as_secs_f64(),
scan_duration.as_secs_f64() / total_duration.as_secs_f64() * 100.0);
println!(" - ID gen: {:.2}s ({:.0}%)",
id_duration.as_secs_f64(),
id_duration.as_secs_f64() / total_duration.as_secs_f64() * 100.0);
println!(" - DB open: {:.2}s ({:.0}%)",
db_duration.as_secs_f64(),
db_duration.as_secs_f64() / total_duration.as_secs_f64() * 100.0);
println!(" - Insertion: {:.2}s ({:.0}%)",
insert_duration.as_secs_f64(),
insert_duration.as_secs_f64() / total_duration.as_secs_f64() * 100.0);
println!(" - Children JSON: {:.2}s ({:.0}%)",
children_duration.as_secs_f64(),
children_duration.as_secs_f64() / total_duration.as_secs_f64() * 100.0);
if !options.skip_hash {
println!();
println!("=== Starting background hash calculation ===");
println!("Files to hash: {}", file_info.len());
println!("Threads: {}", options.threads);
let file_count = file_info.len();
let hash_start = Instant::now();
compute_hashes_parallel(user_id, file_info, options.threads)?;
let hash_duration = hash_start.elapsed();
println!();
println!("Hash calculation completed in {:.2}s ({:.0} files/sec)",
hash_duration.as_secs_f64(),
file_count as f64 / hash_duration.as_secs_f64());
} else {
println!();
println!(" SHA256 hashes skipped. Run 'markbase hash --user {}' to compute hashes.", user_id);
}
Ok(())
}
pub fn compute_hashes(user_id: &str, threads: usize) -> Result<()> {
println!("=== Background Hash Calculation ===");
println!("User ID: {}", user_id);
println!("Threads: {}", threads);
println!();
let conn = FileTree::open_user_db(user_id)?;
let file_info: Vec<(String, String)> = conn
.prepare("SELECT node_id, aliases_json FROM file_nodes WHERE node_type = 'file' AND sha256 IS NULL")?
.query_map([], |row| {
let node_id: String = row.get(0)?;
let aliases_json: String = row.get(1)?;
let aliases: HashMap<String, String> = serde_json::from_str(&aliases_json).unwrap_or_default();
let path = aliases.get("path").cloned().unwrap_or_default();
Ok((node_id, path))
})?
.filter_map(|r| r.ok())
.filter(|(_, path)| !path.is_empty())
.collect();
if file_info.is_empty() {
println!("No files need hashing. All files already have SHA256.");
return Ok(());
}
println!("Files to hash: {}", file_info.len());
let file_count = file_info.len();
let start = Instant::now();
compute_hashes_parallel(user_id, file_info, threads)?;
let duration = start.elapsed();
println!();
println!("Hash calculation completed in {:.2}s ({:.0} files/sec)",
duration.as_secs_f64(),
file_count as f64 / duration.as_secs_f64());
Ok(())
}
fn compute_hashes_parallel(user_id: &str, file_info: Vec<(String, String)>, threads: usize) -> Result<()> {
let db_path = FileTree::user_db_path(user_id);
let user_id = user_id.to_string();
let file_info = Arc::new(file_info);
let results: Arc<Mutex<HashMap<String, String>>> = Arc::new(Mutex::new(HashMap::new()));
let processed: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
let total = file_info.len();
let mut handles = Vec::new();
for i in 0..threads {
let file_info = Arc::clone(&file_info);
let results = Arc::clone(&results);
let processed = Arc::clone(&processed);
let _user_id = user_id.clone();
let handle = thread::spawn(move || {
let chunk_size = (file_info.len() / threads) + (if i < file_info.len() % threads { 1 } else { 0 });
let start_idx = i * (file_info.len() / threads) + i.min(file_info.len() % threads);
let _end_idx = start_idx + chunk_size;
for (node_id, path_str) in file_info.iter().skip(start_idx).take(chunk_size) {
if let Ok(hash) = compute_file_hash(path_str) {
results.lock().unwrap().insert(node_id.clone(), hash);
}
let mut p = processed.lock().unwrap();
*p += 1;
if *p % 100 == 0 {
print!("\r Hashed {}/{} files...", *p, total);
use std::io::Write;
std::io::stdout().flush().ok();
}
}
});
handles.push(handle);
}
for handle in handles {
handle.join().expect("Thread panicked");
}
println!("\r Hashed {}/{} files...Done", total, total);
let results = results.lock().unwrap();
let conn = Connection::open(&db_path)?;
let tx = conn.unchecked_transaction()?;
for (node_id, hash) in results.iter() {
conn.execute(
"UPDATE file_nodes SET sha256 = ?1, file_uuid = ?1, updated_at = ?2 WHERE node_id = ?3",
rusqlite::params![hash, chrono::Utc::now().timestamp().to_string(), node_id],
)?;
}
tx.commit()?;
println!(" Updated {} hashes in database", results.len());
Ok(())
}
fn scan_recursive(
base: &Path,
current: &Path,
folders: &mut Vec<(String, String, Option<String>)>,
files: &mut Vec<(String, String, u64, String)>,
) -> Result<()> {
let entries: Vec<_> = fs::read_dir(current)?
.filter_map(|e| e.ok())
.filter(|e| e.file_name() != ".DS_Store")
.collect();
for entry in entries {
let path = entry.path();
let path_str = path.to_string_lossy().to_string();
let filename = entry.file_name().to_string_lossy().to_string();
if path.is_dir() {
let parent_id = if path.parent() == Some(base) {
None
} else {
find_parent_folder_id(&path_str, folders)
};
folders.push((path_str.clone(), filename, parent_id));
scan_recursive(base, &path, folders, files)?;
} else {
let metadata = entry.metadata()?;
let size = metadata.len();
let ext = path.extension()
.and_then(|s| s.to_str())
.unwrap_or("")
.to_string();
files.push((path_str, filename, size, ext));
}
}
Ok(())
}
fn compute_file_hash(path: &str) -> Result<String> {
let mut hasher = Sha256::new();
let mut file = fs::File::open(path)?;
let mut buffer = [0u8; 8192];
loop {
let n = std::io::Read::read(&mut file, &mut buffer)?;
if n == 0 {
break;
}
hasher.update(&buffer[..n]);
}
let hash = format!("{:x}", hasher.finalize());
Ok(hash.chars().take(32).collect())
}
fn generate_uuid(path: &str, filename: &str, mac: &str, mtime: u64) -> String {
let mut hasher = Sha256::new();
hasher.update(path.as_bytes());
hasher.update(filename.as_bytes());
hasher.update(mac.as_bytes());
hasher.update(mtime.to_string().as_bytes());
format!("{:x}", hasher.finalize()).chars().take(32).collect()
}
fn get_mac_address() -> Result<String> {
let output = std::process::Command::new("ifconfig")
.arg("en0")
.output()?;
let stdout = String::from_utf8_lossy(&output.stdout);
for line in stdout.lines() {
if line.contains("ether") {
if let Some(mac) = line.split_whitespace().nth(1) {
return Ok(mac.to_string());
}
}
}
Ok("00:00:00:00:00:00".to_string())
}
fn find_parent_folder(
file_path: &str,
_base: &Path,
folders: &[(String, String, Option<String>)],
) -> Option<String> {
let file_dir = Path::new(file_path).parent()?;
for (folder_path, _, folder_id) in folders {
if Path::new(folder_path) == file_dir {
return folder_id.clone();
}
}
None
}
fn find_parent_folder_id(path: &str, folders: &[(String, String, Option<String>)]) -> Option<String> {
let current = Path::new(path);
let parent = current.parent()?;
let parent_str = parent.to_string_lossy();
for (folder_path, _, folder_id) in folders {
if folder_path == &parent_str {
return folder_id.clone();
}
}
None
}
fn insert_node(conn: &Connection, node: &FileNode) -> Result<()> {
conn.execute(
"INSERT OR REPLACE INTO file_nodes (
node_id, label, aliases_json, file_uuid, sha256, parent_id, children_json,
node_type, icon, color, bg_color, file_size, registered_at,
created_at, updated_at, sort_order
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
rusqlite::params![
node.node_id,
node.label,
node.aliases.to_json(),
node.file_uuid,
node.sha256,
node.parent_id,
serde_json::to_string(&node.children)?,
node.node_type.as_str(),
node.icon,
node.color,
node.bg_color,
node.file_size,
node.registered_at,
node.created_at,
node.updated_at,
node.sort_order,
],
)?;
Ok(())
}
fn get_file_icon(filename: &str) -> Option<String> {
let ext = Path::new(filename)
.extension()
.and_then(|s| s.to_str())
.unwrap_or("")
.to_lowercase();
let icon = match ext.as_str() {
"mp4" | "mov" | "avi" | "mkv" | "webm" => "🎬",
"jpg" | "jpeg" | "png" | "gif" | "webp" | "svg" => "🖼️",
"pdf" => "📄",
"doc" | "docx" => "📝",
"xls" | "xlsx" => "📊",
"ppt" | "pptx" => "📽️",
"zip" | "rar" | "7z" | "tar" | "gz" => "📦",
"mp3" | "wav" | "flac" | "aac" => "🎵",
"txt" | "md" => "📃",
_ => "📄",
};
Some(icon.to_string())
}

1733
markbase-core/src/server.rs Normal file

File diff suppressed because it is too large Load Diff

370
markbase-core/src/sync.rs Normal file
View File

@@ -0,0 +1,370 @@
use anyhow::Result;
use chrono::Utc;
use rusqlite::{Connection, params};
use serde::{Deserialize, Serialize};
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PgUser {
pub username: String,
pub password_hash: String,
pub email: Option<String>,
pub status: i32,
pub home_dir: String,
pub permissions: String,
pub uid: i64,
pub gid: i64,
pub last_login: i64,
pub created_at: i64,
pub updated_at: i64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PgGroup {
pub name: String,
pub description: Option<String>,
pub created_at: i64,
pub updated_at: i64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PgUserGroupMapping {
pub username: String,
pub group_name: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PgAdmin {
pub username: String,
pub password_hash: String,
pub email: Option<String>,
pub description: Option<String>,
pub status: i32,
pub permissions: String,
pub filters: Option<String>,
pub role_id: Option<i32>,
pub last_login: i64,
pub created_at: i64,
pub updated_at: i64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncResult {
pub sync_type: String,
pub sync_time: i64,
pub users_synced: usize,
pub users_failed: usize,
pub groups_synced: usize,
pub groups_failed: usize,
pub mappings_synced: usize,
pub mappings_failed: usize,
pub admins_synced: usize,
pub admins_failed: usize,
pub status: String,
pub errors: Vec<String>,
}
impl Default for SyncResult {
fn default() -> Self {
Self {
sync_type: "unknown".to_string(),
sync_time: Utc::now().timestamp(),
users_synced: 0,
users_failed: 0,
groups_synced: 0,
groups_failed: 0,
mappings_synced: 0,
mappings_failed: 0,
admins_synced: 0,
admins_failed: 0,
status: "pending".to_string(),
errors: Vec::new(),
}
}
}
impl SyncResult {
pub fn success() -> Self {
Self {
status: "success".to_string(),
..Default::default()
}
}
pub fn cached() -> Self {
Self {
status: "cached".to_string(),
..Default::default()
}
}
pub fn failed(error: String) -> Self {
Self {
status: "failed".to_string(),
errors: vec![error],
..Default::default()
}
}
pub fn merge(&mut self, other: SyncResult) {
self.users_synced += other.users_synced;
self.users_failed += other.users_failed;
self.groups_synced += other.groups_synced;
self.groups_failed += other.groups_failed;
self.mappings_synced += other.mappings_synced;
self.mappings_failed += other.mappings_failed;
self.errors.extend(other.errors);
if self.users_failed > 0 || self.groups_failed > 0 || self.mappings_failed > 0 {
if self.users_synced > 0 || self.groups_synced > 0 || self.mappings_synced > 0 {
self.status = "partial_success".to_string();
} else {
self.status = "failed".to_string();
}
}
}
}
#[derive(Clone)]
pub struct AuthDb {
pub path: String,
}
impl AuthDb {
pub fn new(path: &str) -> Result<Self> {
if !Path::new(path).exists() {
Self::init_db(path)?;
}
Ok(Self { path: path.to_string() })
}
pub fn init_db(path: &str) -> Result<()> {
let conn = Connection::open(path)?;
conn.execute_batch(include_str!("../data/init_auth_db.sql"))?;
Ok(())
}
pub fn open(&self) -> Result<Connection> {
Ok(Connection::open(&self.path)?)
}
pub fn save_user(&self, user: &PgUser) -> Result<()> {
let conn = self.open()?;
let now = Utc::now().timestamp();
conn.execute(
"INSERT OR REPLACE INTO sftpgo_users
(username, password_hash, email, status, home_dir, permissions,
uid, gid, last_login, created_at, updated_at, last_sync_at, sync_status)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
params![
user.username,
user.password_hash,
user.email,
user.status,
user.home_dir,
user.permissions,
user.uid,
user.gid,
user.last_login,
user.created_at,
user.updated_at,
now,
1, // sync_status = synced
]
)?;
Ok(())
}
pub fn save_group(&self, group: &PgGroup) -> Result<()> {
let conn = self.open()?;
let now = Utc::now().timestamp();
conn.execute(
"INSERT OR REPLACE INTO sftpgo_groups
(name, description, created_at, updated_at, last_sync_at)
VALUES (?1, ?2, ?3, ?4, ?5)",
params![
group.name,
group.description,
group.created_at,
group.updated_at,
now,
]
)?;
Ok(())
}
pub fn save_mapping(&self, mapping: &PgUserGroupMapping) -> Result<()> {
let conn = self.open()?;
let now = Utc::now().timestamp();
conn.execute(
"INSERT OR REPLACE INTO users_groups_mapping
(username, group_name, created_at)
VALUES (?1, ?2, ?3)",
params![
mapping.username,
mapping.group_name,
now,
]
)?;
Ok(())
}
pub fn save_sync_log(&self, result: &SyncResult) -> Result<()> {
let conn = self.open()?;
conn.execute(
"INSERT INTO sync_log
(sync_type, sync_time, users_synced, users_failed,
groups_synced, groups_failed, mappings_synced,
mappings_failed, admins_synced, admins_failed,
status, error_message)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![
result.sync_type,
result.sync_time,
result.users_synced,
result.users_failed,
result.groups_synced,
result.groups_failed,
result.mappings_synced,
result.mappings_failed,
result.admins_synced,
result.admins_failed,
result.status,
result.errors.join(";"),
]
)?;
log::info!(
"Sync log saved: users={}, groups={}, mappings={}, admins={}, status={}",
result.users_synced,
result.groups_synced,
result.mappings_synced,
result.admins_synced,
result.status
);
Ok(())
}
pub fn save_admin(&self, admin: &PgAdmin) -> Result<()> {
let conn = self.open()?;
conn.execute(
"INSERT OR REPLACE INTO sftpgo_admins
(username, password_hash, email, description, status,
permissions, filters, role_id, last_login,
created_at, updated_at, last_sync_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![
admin.username,
admin.password_hash,
admin.email,
admin.description,
admin.status,
admin.permissions,
admin.filters,
admin.role_id,
admin.last_login,
admin.created_at,
admin.updated_at,
Utc::now().timestamp(),
],
)?;
Ok(())
}
pub fn get_admin(&self, username: &str) -> Result<Option<PgAdmin>> {
let conn = self.open()?;
let result = conn.query_row(
"SELECT username, password_hash, email, description, status,
permissions, filters, role_id, last_login,
created_at, updated_at
FROM sftpgo_admins WHERE username = ?1 AND status = 1",
params![username],
|row| Ok(PgAdmin {
username: row.get(0)?,
password_hash: row.get(1)?,
email: row.get(2)?,
description: row.get(3)?,
status: row.get(4)?,
permissions: row.get(5)?,
filters: row.get(6)?,
role_id: row.get(7)?,
last_login: row.get(8)?,
created_at: row.get(9)?,
updated_at: row.get(10)?,
}),
);
match result {
Ok(admin) => Ok(Some(admin)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(e.into()),
}
}
pub fn sync_admins(&self, admins: Vec<PgAdmin>) -> Result<usize> {
let mut synced = 0;
for admin in admins {
match self.save_admin(&admin) {
Ok(_) => synced += 1,
Err(e) => log::warn!("Failed to sync admin {}: {}", admin.username, e),
}
}
Ok(synced)
}
pub fn get_user(&self, username: &str) -> Result<Option<PgUser>> {
let conn = self.open()?;
let result = conn.query_row(
"SELECT username, password_hash, email, status, home_dir, permissions,
uid, gid, last_login, created_at, updated_at
FROM sftpgo_users WHERE username = ?1 AND status = 1",
params![username],
|row| Ok(PgUser {
username: row.get(0)?,
password_hash: row.get(1)?,
email: row.get(2)?,
status: row.get(3)?,
home_dir: row.get(4)?,
permissions: row.get(5)?,
uid: row.get(6)?,
gid: row.get(7)?,
last_login: row.get(8)?,
created_at: row.get(9)?,
updated_at: row.get(10)?,
})
);
match result {
Ok(user) => Ok(Some(user)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(e.into()),
}
}
pub fn get_user_groups(&self, username: &str) -> Result<Vec<String>> {
let conn = self.open()?;
let groups: Vec<String> = conn
.prepare(
"SELECT group_name FROM users_groups_mapping WHERE username = ?1"
)?
.query_map(params![username], |row| row.get(0))?
.collect::<Result<Vec<_>, _>>()?;
Ok(groups)
}
}