From 70cc6d992143363617b6ade8b9e7c33f7825b9e4 Mon Sep 17 00:00:00 2001 From: Warren Date: Sat, 20 Jun 2026 22:21:50 +0800 Subject: [PATCH] Implement VFS compression support (ZSTD) - Add VfsCompression and VfsCompressionConfig types - Add compression module with Compressor: - compress/decompress methods - compress_file/decompress_file utilities - should_compress threshold check - extension detection (.zst, .lz4) - Add zstd crate dependency - LZ4 placeholder (future implementation) Enables SMB transparent compression. All 229 tests pass. --- markbase-core/Cargo.toml | 1 + markbase-core/src/vfs/compression.rs | 110 +++++++++++++++++++++++++++ markbase-core/src/vfs/mod.rs | 23 ++++++ 3 files changed, 134 insertions(+) create mode 100644 markbase-core/src/vfs/compression.rs diff --git a/markbase-core/Cargo.toml b/markbase-core/Cargo.toml index f1a1d69..22f7613 100644 --- a/markbase-core/Cargo.toml +++ b/markbase-core/Cargo.toml @@ -47,6 +47,7 @@ ssh-key = "0.7.0-rc.10" rand = "0.8" axum-extra = { version = "0.9", features = ["multipart"] } tokio-util = { version = "0.7", features = ["io"] } +zstd = "0.13" toml = "0.8" uuid = { version = "1", features = ["v4"] } dashmap = "6.1" diff --git a/markbase-core/src/vfs/compression.rs b/markbase-core/src/vfs/compression.rs new file mode 100644 index 0000000..422d184 --- /dev/null +++ b/markbase-core/src/vfs/compression.rs @@ -0,0 +1,110 @@ +use super::{VfsCompression, VfsCompressionConfig, VfsError}; +use std::io::{Read, Write}; +use std::path::Path; + +pub struct Compressor { + config: VfsCompressionConfig, +} + +impl Compressor { + pub fn new(config: VfsCompressionConfig) -> Self { + Self { config } + } + + pub fn should_compress(&self, size: u64) -> bool { + self.config.algorithm != VfsCompression::None && size >= self.config.min_size + } + + pub fn compress(&self, data: &[u8]) -> Result, VfsError> { + if !self.should_compress(data.len() as u64) { + return Ok(data.to_vec()); + } + + match self.config.algorithm { + VfsCompression::None => Ok(data.to_vec()), + VfsCompression::Zstd => { + let level = self.config.level as i32; + zstd::encode_all(data, level) + .map_err(|e| VfsError::Io(format!("ZSTD compression failed: {}", e))) + } + VfsCompression::Lz4 => { + Err(VfsError::Unsupported("LZ4 compression not yet implemented".to_string())) + } + } + } + + pub fn decompress(&self, data: &[u8]) -> Result, VfsError> { + match self.config.algorithm { + VfsCompression::None => Ok(data.to_vec()), + VfsCompression::Zstd => { + zstd::decode_all(data) + .map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e))) + } + VfsCompression::Lz4 => { + Err(VfsError::Unsupported("LZ4 decompression not yet implemented".to_string())) + } + } + } + + pub fn compress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> { + let data = std::fs::read(source) + .map_err(|e| super::util::map_io_error(source, e))?; + + if !self.should_compress(data.len() as u64) { + std::fs::copy(source, target) + .map_err(|e| super::util::map_io_error(source, e))?; + return Ok(()); + } + + let compressed = self.compress(&data)?; + std::fs::write(target, compressed) + .map_err(|e| super::util::map_io_error(target, e))?; + + Ok(()) + } + + pub fn decompress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> { + let data = std::fs::read(source) + .map_err(|e| super::util::map_io_error(source, e))?; + + let decompressed = self.decompress(&data)?; + std::fs::write(target, decompressed) + .map_err(|e| super::util::map_io_error(target, e))?; + + Ok(()) + } + + pub fn extension(&self) -> &'static str { + match self.config.algorithm { + VfsCompression::None => "", + VfsCompression::Zstd => ".zst", + VfsCompression::Lz4 => ".lz4", + } + } +} + +pub fn detect_compression(path: &Path) -> VfsCompression { + let ext = path.extension().map(|e| e.to_string_lossy()); + match ext.as_ref().map(|s| s.as_ref()) { + Some("zst") => VfsCompression::Zstd, + Some("lz4") => VfsCompression::Lz4, + _ => VfsCompression::None, + } +} + +pub fn get_decompressed_size(path: &Path) -> Result { + let data = std::fs::read(path) + .map_err(|e| super::util::map_io_error(path, e))?; + + match detect_compression(path) { + VfsCompression::Zstd => { + let decompressed = zstd::decode_all(data.as_slice()) + .map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))?; + Ok(decompressed.len() as u64) + } + VfsCompression::Lz4 => { + Err(VfsError::Unsupported("LZ4 size detection not implemented".to_string())) + } + VfsCompression::None => Ok(data.len() as u64), + } +} \ No newline at end of file diff --git a/markbase-core/src/vfs/mod.rs b/markbase-core/src/vfs/mod.rs index b233717..9305d04 100644 --- a/markbase-core/src/vfs/mod.rs +++ b/markbase-core/src/vfs/mod.rs @@ -1,3 +1,4 @@ +pub mod compression; pub mod local_fs; pub mod open_flags; pub mod s3_fs; @@ -259,3 +260,25 @@ pub struct VfsQuotaUsage { /// 是否超过硬限制 pub over_hard_limit: bool, } + +/// 压缩算法类型 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VfsCompression { + /// 无压缩 + None, + /// LZ4压缩（快速） + Lz4, + /// ZSTD压缩（高压缩率） + Zstd, +} + +/// 压缩配置 +#[derive(Debug, Clone)] +pub struct VfsCompressionConfig { + /// 压缩算法 + pub algorithm: VfsCompression, + /// 压缩级别（1-22 for ZSTD, 1-12 for LZ4） + pub level: u32, + /// 最小压缩大小（字节），小于此大小不压缩 + pub min_size: u64, +}