MarkBase架构升级：Multi-Volume Virtual Tree + Dual-View Management + Git Remote修正
核心功能： - ✅ Categories/Series双视图管理（category_view.rs + import_markdown.rs） - ✅ FUSE Multi-Volume支持（tree_type参数） - ✅ SSH/SFTP/SCP/rsync协议完整实现（4042行） - ✅ NFS/SMB Module Phase 1-3完成 - ✅ Archive Module Phase 1-4完成（2916行） - ✅ Download Center API完整实现 - ✅ S3兼容API实现（560行） Git配置修正： - ✅ 删除错误origin（gitea.momentry.ddns.net） - ✅ 删除m5max128（指向机器名） - ✅ 设置origin = m5max128gitea.momentry.ddns.net/admin/markbase - ✅ 设置m4minigitea = m4minigitea.momentry.ddns.net/warren/markbase 数据清理： - ✅ 删除38个临时SQLite（保留accusys.sqlite、demo.sqlite） - ✅ 删除.bak、test_*.bin、调试脚本等临时文件 - ✅ 删除临时目录（build/、download files/、raid_test/等） - ✅ 更新.gitignore排除临时文件 架构优化： - 52个文件修改，2434行新增，4739行删除 - Workspace成员整合（16个crate） - 数据库状态：accusys.sqlite保留（主demo测试） 远程同步： - ✅ 准备推送到m5max128gitea（远程Gitea） - ✅ 准备推送到m4minigitea（本地Gitea）
This commit is contained in:
Warren
25
filetree-rocksdb/Cargo.toml
Normal file
25
filetree-rocksdb/Cargo.toml
Normal file
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "filetree-rocksdb"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
rocksdb = "0.24"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
anyhow = "1"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
|
||||
[[bin]]
|
||||
name = "filetree-rocksdb-poc"
|
||||
path = "src/poc.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "sqlite-to-rocksdb-migrate"
|
||||
path = "src/migrate.rs"
|
||||
|
||||
[lib]
|
||||
name = "filetree_rocksdb"
|
||||
path = "src/lib.rs"
|
||||
339
filetree-rocksdb/src/lib.rs
Normal file
339
filetree-rocksdb/src/lib.rs
Normal file
@@ -0,0 +1,339 @@
|
||||
use anyhow::{Context, Result};
|
||||
use rocksdb::{ColumnFamilyDescriptor, Options, WriteBatch, DB};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::str::FromStr;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FileNode {
|
||||
pub node_id: String,
|
||||
pub label: String,
|
||||
pub aliases: Aliases,
|
||||
pub file_uuid: Option<String>,
|
||||
pub sha256: Option<String>,
|
||||
pub parent_id: Option<String>,
|
||||
pub children: Vec<String>,
|
||||
pub node_type: NodeType,
|
||||
pub icon: Option<String>,
|
||||
pub color: Option<String>,
|
||||
pub bg_color: Option<String>,
|
||||
pub file_size: Option<i64>,
|
||||
pub registered_at: Option<String>,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
pub sort_order: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Aliases {
|
||||
#[serde(flatten)]
|
||||
pub map: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl Aliases {
|
||||
pub fn empty() -> Self {
|
||||
Aliases {
|
||||
map: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_json(&self) -> String {
|
||||
serde_json::to_string(&self.map).unwrap_or_else(|_| "{}".to_string())
|
||||
}
|
||||
|
||||
pub fn from_json(s: &str) -> Self {
|
||||
let map: HashMap<String, String> = serde_json::from_str(s).unwrap_or_default();
|
||||
Aliases { map }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NodeType {
|
||||
Folder,
|
||||
File,
|
||||
DynamicLayer,
|
||||
}
|
||||
|
||||
impl NodeType {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
NodeType::Folder => "folder",
|
||||
NodeType::File => "file",
|
||||
NodeType::DynamicLayer => "dynamic_layer",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for NodeType {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"folder" => Ok(NodeType::Folder),
|
||||
"file" => Ok(NodeType::File),
|
||||
"dynamic_layer" => Ok(NodeType::DynamicLayer),
|
||||
_ => Ok(NodeType::Folder),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FileTreeRocksDB {
|
||||
pub user_id: String,
|
||||
pub db: DB,
|
||||
}
|
||||
|
||||
impl FileTreeRocksDB {
|
||||
pub fn user_db_path(user_id: &str) -> String {
|
||||
format!("data/users_rocksdb/{}.rocksdb", user_id)
|
||||
}
|
||||
|
||||
pub fn init_user_db(user_id: &str) -> Result<Self> {
|
||||
let db_path = Self::user_db_path(user_id);
|
||||
let parent = std::path::Path::new(&db_path).parent().unwrap();
|
||||
std::fs::create_dir_all(parent)?;
|
||||
|
||||
let mut opts = Options::default();
|
||||
opts.create_if_missing(true);
|
||||
opts.create_missing_column_families(true);
|
||||
|
||||
let cfs = vec![
|
||||
ColumnFamilyDescriptor::new("file_nodes", Options::default()),
|
||||
ColumnFamilyDescriptor::new("file_registry", Options::default()),
|
||||
ColumnFamilyDescriptor::new("file_locations", Options::default()),
|
||||
ColumnFamilyDescriptor::new("parent_index", Options::default()),
|
||||
];
|
||||
|
||||
let db = DB::open_cf_descriptors(&opts, &db_path, cfs)
|
||||
.with_context(|| format!("Failed to open RocksDB at {}", db_path))?;
|
||||
|
||||
Ok(FileTreeRocksDB {
|
||||
user_id: user_id.to_string(),
|
||||
db,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_user_db(user_id: &str) -> Result<Self> {
|
||||
let db_path = Self::user_db_path(user_id);
|
||||
|
||||
let mut opts = Options::default();
|
||||
opts.create_if_missing(true);
|
||||
opts.create_missing_column_families(true);
|
||||
|
||||
let cfs = vec![
|
||||
ColumnFamilyDescriptor::new("file_nodes", Options::default()),
|
||||
ColumnFamilyDescriptor::new("file_registry", Options::default()),
|
||||
ColumnFamilyDescriptor::new("file_locations", Options::default()),
|
||||
ColumnFamilyDescriptor::new("parent_index", Options::default()),
|
||||
];
|
||||
|
||||
let db = DB::open_cf_descriptors(&opts, &db_path, cfs)
|
||||
.with_context(|| format!("Failed to open RocksDB at {}", db_path))?;
|
||||
|
||||
Ok(FileTreeRocksDB {
|
||||
user_id: user_id.to_string(),
|
||||
db,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn insert_node(&self, node: &FileNode) -> Result<()> {
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
let node_data = serde_json::to_vec(node)?;
|
||||
self.db.put_cf(cf, node.node_id.as_bytes(), node_data)?;
|
||||
|
||||
if let Some(parent_id) = &node.parent_id {
|
||||
let cf_parent = self.db.cf_handle("parent_index").unwrap();
|
||||
let key = format!("children:{}", parent_id);
|
||||
let existing = self.db.get_cf(cf_parent, key.as_bytes())?;
|
||||
|
||||
let mut children: Vec<String> = match existing {
|
||||
Some(data) => serde_json::from_slice(&data)?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
|
||||
if !children.contains(&node.node_id) {
|
||||
children.push(node.node_id.clone());
|
||||
self.db
|
||||
.put_cf(cf_parent, key.as_bytes(), serde_json::to_vec(&children)?)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn insert_node_batch(&self, nodes: &[FileNode]) -> Result<()> {
|
||||
let mut batch = WriteBatch::default();
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
let cf_parent = self.db.cf_handle("parent_index").unwrap();
|
||||
|
||||
for node in nodes {
|
||||
let node_data = serde_json::to_vec(node)?;
|
||||
batch.put_cf(cf, node.node_id.as_bytes(), node_data);
|
||||
}
|
||||
|
||||
self.db.write(batch)?;
|
||||
|
||||
for node in nodes {
|
||||
if let Some(parent_id) = &node.parent_id {
|
||||
let key = format!("children:{}", parent_id);
|
||||
let existing = self.db.get_cf(cf_parent, key.as_bytes())?;
|
||||
|
||||
let mut children: Vec<String> = match existing {
|
||||
Some(data) => serde_json::from_slice(&data)?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
|
||||
if !children.contains(&node.node_id) {
|
||||
children.push(node.node_id.clone());
|
||||
self.db
|
||||
.put_cf(cf_parent, key.as_bytes(), serde_json::to_vec(&children)?)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_node(&self, node_id: &str) -> Result<Option<FileNode>> {
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
let value = self.db.get_cf(cf, node_id.as_bytes())?;
|
||||
|
||||
match value {
|
||||
Some(data) => {
|
||||
let node: FileNode = serde_json::from_slice(&data)?;
|
||||
Ok(Some(node))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_children(&self, parent_id: &str) -> Result<Vec<String>> {
|
||||
let cf = self.db.cf_handle("parent_index").unwrap();
|
||||
let key = format!("children:{}", parent_id);
|
||||
let value = self.db.get_cf(cf, key.as_bytes())?;
|
||||
|
||||
match value {
|
||||
Some(data) => {
|
||||
let children: Vec<String> = serde_json::from_slice(&data)?;
|
||||
Ok(children)
|
||||
}
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_all(&self) -> Result<Vec<FileNode>> {
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
let mut nodes = Vec::new();
|
||||
|
||||
let iter = self.db.iterator_cf(cf, rocksdb::IteratorMode::Start);
|
||||
for item in iter {
|
||||
let (_, value) = item?;
|
||||
let node: FileNode = serde_json::from_slice(&value)?;
|
||||
nodes.push(node);
|
||||
}
|
||||
|
||||
nodes.sort_by(|a, b| {
|
||||
a.sort_order
|
||||
.cmp(&b.sort_order)
|
||||
.then_with(|| a.created_at.cmp(&b.created_at))
|
||||
});
|
||||
|
||||
Ok(nodes)
|
||||
}
|
||||
|
||||
pub fn update_node(&self, node_id: &str, updates: &FileNode) -> Result<()> {
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
let node_data = serde_json::to_vec(updates)?;
|
||||
self.db.put_cf(cf, node_id.as_bytes(), node_data)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_node(&self, node_id: &str) -> Result<()> {
|
||||
let node = self.get_node(node_id)?;
|
||||
|
||||
if let Some(n) = node {
|
||||
if let Some(parent_id) = &n.parent_id {
|
||||
let cf_parent = self.db.cf_handle("parent_index").unwrap();
|
||||
let key = format!("children:{}", parent_id);
|
||||
let existing = self.db.get_cf(cf_parent, key.as_bytes())?;
|
||||
|
||||
let mut children: Vec<String> = match existing {
|
||||
Some(data) => serde_json::from_slice(&data)?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
|
||||
children.retain(|id| id != node_id);
|
||||
self.db
|
||||
.put_cf(cf_parent, key.as_bytes(), serde_json::to_vec(&children)?)?;
|
||||
}
|
||||
}
|
||||
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
self.db.delete_cf(cf, node_id.as_bytes())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn count_nodes(&self) -> Result<usize> {
|
||||
let cf = self.db.cf_handle("file_nodes").unwrap();
|
||||
let mut count = 0;
|
||||
|
||||
let iter = self.db.iterator_cf(cf, rocksdb::IteratorMode::Start);
|
||||
for item in iter {
|
||||
let _ = item?;
|
||||
count += 1;
|
||||
}
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub fn new_folder(label: &str, parent_id: Option<&str>) -> FileNode {
|
||||
FileNode {
|
||||
node_id: Uuid::new_v4().to_string().replace("-", ""),
|
||||
label: label.to_string(),
|
||||
aliases: Aliases::empty(),
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: parent_id.map(|s| s.to_string()),
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::Folder,
|
||||
icon: None,
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_file_node(
|
||||
label: &str,
|
||||
file_uuid: &str,
|
||||
sha256: Option<&str>,
|
||||
original_name: &str,
|
||||
file_size: Option<i64>,
|
||||
mime_type: Option<&str>,
|
||||
parent_id: Option<&str>,
|
||||
) -> FileNode {
|
||||
FileNode {
|
||||
node_id: Uuid::new_v4().to_string().replace("-", ""),
|
||||
label: label.to_string(),
|
||||
aliases: Aliases::empty(),
|
||||
file_uuid: Some(file_uuid.to_string()),
|
||||
sha256: sha256.map(|s| s.to_string()),
|
||||
parent_id: parent_id.map(|s| s.to_string()),
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::File,
|
||||
icon: None,
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size,
|
||||
registered_at: Some(chrono::Utc::now().to_rfc3339()),
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
147
filetree-rocksdb/src/migrate.rs
Normal file
147
filetree-rocksdb/src/migrate.rs
Normal file
@@ -0,0 +1,147 @@
|
||||
use anyhow::{Context, Result};
|
||||
use filetree_rocksdb::FileTreeRocksDB;
|
||||
use rusqlite::Connection;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("=== SQLite → RocksDB Migration Test ===\n");
|
||||
|
||||
let sqlite_path = "data/users/warren.sqlite";
|
||||
let rocksdb_path = "data/users_rocksdb/warren.rocksdb";
|
||||
|
||||
println!("Step 1: Open SQLite database...");
|
||||
let conn =
|
||||
Connection::open(sqlite_path).with_context(|| format!("Failed to open {}", sqlite_path))?;
|
||||
|
||||
let node_count: i64 =
|
||||
conn.query_row("SELECT COUNT(*) FROM file_nodes", [], |row| row.get(0))?;
|
||||
println!(" ✓ SQLite nodes count: {}", node_count);
|
||||
|
||||
println!("\nStep 2: Read all nodes from SQLite...");
|
||||
let start = Instant::now();
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_id, label, aliases_json, file_uuid, sha256, parent_id, children_json,
|
||||
node_type, icon, color, bg_color, file_size, registered_at,
|
||||
created_at, updated_at, sort_order
|
||||
FROM file_nodes",
|
||||
)?;
|
||||
|
||||
let nodes: Vec<filetree_rocksdb::FileNode> = stmt
|
||||
.query_map([], |row| {
|
||||
let children_json: String = row.get(6)?;
|
||||
let children: Vec<String> = serde_json::from_str(&children_json).unwrap_or_default();
|
||||
let node_type_str: String = row.get(7)?;
|
||||
|
||||
Ok(filetree_rocksdb::FileNode {
|
||||
node_id: row.get(0)?,
|
||||
label: row.get(1)?,
|
||||
aliases: filetree_rocksdb::Aliases::from_json(&row.get::<_, String>(2)?),
|
||||
file_uuid: row.get(3)?,
|
||||
sha256: row.get(4)?,
|
||||
parent_id: row.get(5)?,
|
||||
children,
|
||||
node_type: std::str::FromStr::from_str(&node_type_str)
|
||||
.unwrap_or(filetree_rocksdb::NodeType::Folder),
|
||||
icon: row.get(8)?,
|
||||
color: row.get(9)?,
|
||||
bg_color: row.get(10)?,
|
||||
file_size: row.get(11)?,
|
||||
registered_at: row.get(12)?,
|
||||
created_at: row.get(13)?,
|
||||
updated_at: row.get(14)?,
|
||||
sort_order: row.get(15)?,
|
||||
})
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
let read_time = start.elapsed();
|
||||
let read_throughput = nodes.len() as f64 / read_time.as_secs_f64();
|
||||
println!(" ✓ Read time: {:?}", read_time);
|
||||
println!(" ✓ Nodes read: {}", nodes.len());
|
||||
println!(" ✓ Throughput: {:.2} nodes/sec", read_throughput);
|
||||
|
||||
println!("\nStep 3: Initialize RocksDB database...");
|
||||
let start = Instant::now();
|
||||
let rocksdb_tree = FileTreeRocksDB::init_user_db("warren")?;
|
||||
let init_time = start.elapsed();
|
||||
println!(" ✓ Init time: {:?}", init_time);
|
||||
|
||||
println!("\nStep 4: Import nodes to RocksDB (batch insert)...");
|
||||
let start = Instant::now();
|
||||
|
||||
rocksdb_tree.insert_node_batch(&nodes)?;
|
||||
|
||||
let import_time = start.elapsed();
|
||||
let import_throughput = nodes.len() as f64 / import_time.as_secs_f64();
|
||||
println!(" ✓ Import time: {:?}", import_time);
|
||||
println!(" ✓ Throughput: {:.2} nodes/sec", import_throughput);
|
||||
|
||||
println!("\nStep 5: Verify import...");
|
||||
let rocksdb_count = rocksdb_tree.count_nodes()?;
|
||||
println!(" ✓ RocksDB nodes count: {}", rocksdb_count);
|
||||
println!(" ✓ Match: {}", rocksdb_count == nodes.len());
|
||||
|
||||
println!("\nStep 6: Query test (1000 random nodes)...");
|
||||
let test_nodes = &nodes[..1000.min(nodes.len())];
|
||||
let start = Instant::now();
|
||||
|
||||
for node in test_nodes {
|
||||
let _ = rocksdb_tree.get_node(&node.node_id)?;
|
||||
}
|
||||
|
||||
let query_time = start.elapsed();
|
||||
let query_latency = query_time.as_nanos() as f64 / test_nodes.len() as f64;
|
||||
println!(" ✓ Query time: {:?}", query_time);
|
||||
println!(" ✓ Average latency: {:.2} ns", query_latency);
|
||||
|
||||
println!("\nStep 7: Database size comparison...");
|
||||
let sqlite_size = std::fs::metadata(sqlite_path)?.len();
|
||||
let rocksdb_size = get_directory_size(rocksdb_path)?;
|
||||
println!(
|
||||
" ✓ SQLite size: {} bytes ({:.2} MB)",
|
||||
sqlite_size,
|
||||
sqlite_size as f64 / 1024.0 / 1024.0
|
||||
);
|
||||
println!(
|
||||
" ✓ RocksDB size: {} bytes ({:.2} MB)",
|
||||
rocksdb_size,
|
||||
rocksdb_size as f64 / 1024.0 / 1024.0
|
||||
);
|
||||
println!(
|
||||
" ✓ Size ratio: {:.2}x",
|
||||
rocksdb_size as f64 / sqlite_size as f64
|
||||
);
|
||||
|
||||
println!("\n=== Migration Summary ===");
|
||||
println!("SQLite nodes: {}", node_count);
|
||||
println!("Imported nodes: {}", nodes.len());
|
||||
println!("Import throughput: {:.2} nodes/sec", import_throughput);
|
||||
println!("Query latency: {:.2} ns", query_latency);
|
||||
println!(
|
||||
"Size ratio: {:.2}x",
|
||||
rocksdb_size as f64 / sqlite_size as f64
|
||||
);
|
||||
|
||||
println!("\nStep 8: Cleanup...");
|
||||
std::fs::remove_dir_all(rocksdb_path)?;
|
||||
println!(" ✓ Test database removed");
|
||||
|
||||
println!("\n✅ Migration test completed successfully!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_directory_size(path: &str) -> Result<u64> {
|
||||
let mut total_size = 0;
|
||||
for entry in std::fs::read_dir(path)? {
|
||||
let entry = entry?;
|
||||
let metadata = entry.metadata()?;
|
||||
if metadata.is_file() {
|
||||
total_size += metadata.len();
|
||||
} else if metadata.is_dir() {
|
||||
total_size += get_directory_size(entry.path().to_str().unwrap())?;
|
||||
}
|
||||
}
|
||||
Ok(total_size)
|
||||
}
|
||||
120
filetree-rocksdb/src/poc.rs
Normal file
120
filetree-rocksdb/src/poc.rs
Normal file
@@ -0,0 +1,120 @@
|
||||
use anyhow::Result;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("=== FileTree RocksDB POC Performance Test ===\n");
|
||||
|
||||
let user_id = "test_rocksdb";
|
||||
|
||||
println!("Step 1: Initialize RocksDB database...");
|
||||
let start = Instant::now();
|
||||
let tree = filetree_rocksdb::FileTreeRocksDB::init_user_db(user_id)?;
|
||||
let init_time = start.elapsed();
|
||||
println!(" ✓ Init time: {:?}", init_time);
|
||||
|
||||
println!("\nStep 2: Insert 1,000 nodes (single insert)...");
|
||||
let nodes: Vec<filetree_rocksdb::FileNode> = (0..1000)
|
||||
.map(|i| filetree_rocksdb::FileTreeRocksDB::new_folder(&format!("folder_{}", i), None))
|
||||
.collect();
|
||||
|
||||
let start = Instant::now();
|
||||
for node in &nodes {
|
||||
tree.insert_node(node)?;
|
||||
}
|
||||
let single_insert_time = start.elapsed();
|
||||
let single_throughput = nodes.len() as f64 / single_insert_time.as_secs_f64();
|
||||
println!(" ✓ Single insert: {:?}", single_insert_time);
|
||||
println!(" ✓ Throughput: {:.2} nodes/sec", single_throughput);
|
||||
|
||||
println!("\nStep 3: Insert 10,000 nodes (batch insert)...");
|
||||
let nodes_batch: Vec<filetree_rocksdb::FileNode> = (0..10000)
|
||||
.map(|i| {
|
||||
filetree_rocksdb::FileTreeRocksDB::new_folder(&format!("batch_folder_{}", i), None)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let start = Instant::now();
|
||||
tree.insert_node_batch(&nodes_batch)?;
|
||||
let batch_insert_time = start.elapsed();
|
||||
let batch_throughput = nodes_batch.len() as f64 / batch_insert_time.as_secs_f64();
|
||||
println!(" ✓ Batch insert: {:?}", batch_insert_time);
|
||||
println!(" ✓ Throughput: {:.2} nodes/sec", batch_throughput);
|
||||
|
||||
println!("\nStep 4: Query single node (10,000 iterations)...");
|
||||
let test_node_id = &nodes_batch[5000].node_id;
|
||||
let start = Instant::now();
|
||||
for _ in 0..10000 {
|
||||
let _ = tree.get_node(test_node_id)?;
|
||||
}
|
||||
let query_time = start.elapsed();
|
||||
let query_latency = query_time.as_nanos() as f64 / 10000.0;
|
||||
println!(" ✓ Total time: {:?}", query_time);
|
||||
println!(" ✓ Average latency: {:.2} ns", query_latency);
|
||||
|
||||
println!("\nStep 5: Load all nodes...");
|
||||
let start = Instant::now();
|
||||
let all_nodes = tree.load_all()?;
|
||||
let load_time = start.elapsed();
|
||||
println!(" ✓ Load time: {:?}", load_time);
|
||||
println!(" ✓ Nodes loaded: {}", all_nodes.len());
|
||||
|
||||
println!("\nStep 6: Concurrent reads (single process, simulated)...");
|
||||
let db_path = filetree_rocksdb::FileTreeRocksDB::user_db_path(user_id);
|
||||
let start = Instant::now();
|
||||
|
||||
for i in 0..10000 {
|
||||
let node_id = format!("batch_folder_{}", i % 1000);
|
||||
let _ = tree.get_node(&node_id)?;
|
||||
}
|
||||
|
||||
let concurrent_time = start.elapsed();
|
||||
let concurrent_ops = 10000;
|
||||
let concurrent_throughput = concurrent_ops as f64 / concurrent_time.as_secs_f64();
|
||||
println!(" ✓ Concurrent time: {:?}", concurrent_time);
|
||||
println!(" ✓ Total ops: {}", concurrent_ops);
|
||||
println!(" ✓ Throughput: {:.2} ops/sec", concurrent_throughput);
|
||||
|
||||
println!("\nStep 7: Database size...");
|
||||
let db_size = get_directory_size(&db_path)?;
|
||||
println!(
|
||||
" ✓ DB size: {} bytes ({:.2} MB)",
|
||||
db_size,
|
||||
db_size as f64 / 1024.0 / 1024.0
|
||||
);
|
||||
println!(" ✓ Nodes count: {}", tree.count_nodes()?);
|
||||
|
||||
println!("\n=== Performance Summary ===");
|
||||
println!(
|
||||
"Single insert: {:?} ({:.2} nodes/sec)",
|
||||
single_insert_time, single_throughput
|
||||
);
|
||||
println!(
|
||||
"Batch insert: {:?} ({:.2} nodes/sec)",
|
||||
batch_insert_time, batch_throughput
|
||||
);
|
||||
println!("Query latency: {:.2} ns", query_latency);
|
||||
println!("Concurrent reads: {:.2} ops/sec", concurrent_throughput);
|
||||
println!("DB size: {:.2} MB", db_size as f64 / 1024.0 / 1024.0);
|
||||
|
||||
println!("\nStep 8: Cleanup...");
|
||||
std::fs::remove_dir_all(&db_path)?;
|
||||
println!(" ✓ Test database removed");
|
||||
|
||||
println!("\n✅ POC Test completed successfully!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_directory_size(path: &str) -> Result<u64> {
|
||||
let mut total_size = 0;
|
||||
for entry in std::fs::read_dir(path)? {
|
||||
let entry = entry?;
|
||||
let metadata = entry.metadata()?;
|
||||
if metadata.is_file() {
|
||||
total_size += metadata.len();
|
||||
} else if metadata.is_dir() {
|
||||
total_size += get_directory_size(entry.path().to_str().unwrap())?;
|
||||
}
|
||||
}
|
||||
Ok(total_size)
|
||||
}
|
||||
Reference in New Issue
Block a user