Implement VFS compression support (ZSTD)
- Add VfsCompression and VfsCompressionConfig types - Add compression module with Compressor: - compress/decompress methods - compress_file/decompress_file utilities - should_compress threshold check - extension detection (.zst, .lz4) - Add zstd crate dependency - LZ4 placeholder (future implementation) Enables SMB transparent compression. All 229 tests pass.
This commit is contained in:
@@ -47,6 +47,7 @@ ssh-key = "0.7.0-rc.10"
|
|||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
axum-extra = { version = "0.9", features = ["multipart"] }
|
axum-extra = { version = "0.9", features = ["multipart"] }
|
||||||
tokio-util = { version = "0.7", features = ["io"] }
|
tokio-util = { version = "0.7", features = ["io"] }
|
||||||
|
zstd = "0.13"
|
||||||
toml = "0.8"
|
toml = "0.8"
|
||||||
uuid = { version = "1", features = ["v4"] }
|
uuid = { version = "1", features = ["v4"] }
|
||||||
dashmap = "6.1"
|
dashmap = "6.1"
|
||||||
|
|||||||
110
markbase-core/src/vfs/compression.rs
Normal file
110
markbase-core/src/vfs/compression.rs
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
use super::{VfsCompression, VfsCompressionConfig, VfsError};
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
pub struct Compressor {
|
||||||
|
config: VfsCompressionConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Compressor {
|
||||||
|
pub fn new(config: VfsCompressionConfig) -> Self {
|
||||||
|
Self { config }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn should_compress(&self, size: u64) -> bool {
|
||||||
|
self.config.algorithm != VfsCompression::None && size >= self.config.min_size
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compress(&self, data: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||||
|
if !self.should_compress(data.len() as u64) {
|
||||||
|
return Ok(data.to_vec());
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.config.algorithm {
|
||||||
|
VfsCompression::None => Ok(data.to_vec()),
|
||||||
|
VfsCompression::Zstd => {
|
||||||
|
let level = self.config.level as i32;
|
||||||
|
zstd::encode_all(data, level)
|
||||||
|
.map_err(|e| VfsError::Io(format!("ZSTD compression failed: {}", e)))
|
||||||
|
}
|
||||||
|
VfsCompression::Lz4 => {
|
||||||
|
Err(VfsError::Unsupported("LZ4 compression not yet implemented".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||||
|
match self.config.algorithm {
|
||||||
|
VfsCompression::None => Ok(data.to_vec()),
|
||||||
|
VfsCompression::Zstd => {
|
||||||
|
zstd::decode_all(data)
|
||||||
|
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))
|
||||||
|
}
|
||||||
|
VfsCompression::Lz4 => {
|
||||||
|
Err(VfsError::Unsupported("LZ4 decompression not yet implemented".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> {
|
||||||
|
let data = std::fs::read(source)
|
||||||
|
.map_err(|e| super::util::map_io_error(source, e))?;
|
||||||
|
|
||||||
|
if !self.should_compress(data.len() as u64) {
|
||||||
|
std::fs::copy(source, target)
|
||||||
|
.map_err(|e| super::util::map_io_error(source, e))?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let compressed = self.compress(&data)?;
|
||||||
|
std::fs::write(target, compressed)
|
||||||
|
.map_err(|e| super::util::map_io_error(target, e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decompress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> {
|
||||||
|
let data = std::fs::read(source)
|
||||||
|
.map_err(|e| super::util::map_io_error(source, e))?;
|
||||||
|
|
||||||
|
let decompressed = self.decompress(&data)?;
|
||||||
|
std::fs::write(target, decompressed)
|
||||||
|
.map_err(|e| super::util::map_io_error(target, e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extension(&self) -> &'static str {
|
||||||
|
match self.config.algorithm {
|
||||||
|
VfsCompression::None => "",
|
||||||
|
VfsCompression::Zstd => ".zst",
|
||||||
|
VfsCompression::Lz4 => ".lz4",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn detect_compression(path: &Path) -> VfsCompression {
|
||||||
|
let ext = path.extension().map(|e| e.to_string_lossy());
|
||||||
|
match ext.as_ref().map(|s| s.as_ref()) {
|
||||||
|
Some("zst") => VfsCompression::Zstd,
|
||||||
|
Some("lz4") => VfsCompression::Lz4,
|
||||||
|
_ => VfsCompression::None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_decompressed_size(path: &Path) -> Result<u64, VfsError> {
|
||||||
|
let data = std::fs::read(path)
|
||||||
|
.map_err(|e| super::util::map_io_error(path, e))?;
|
||||||
|
|
||||||
|
match detect_compression(path) {
|
||||||
|
VfsCompression::Zstd => {
|
||||||
|
let decompressed = zstd::decode_all(data.as_slice())
|
||||||
|
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))?;
|
||||||
|
Ok(decompressed.len() as u64)
|
||||||
|
}
|
||||||
|
VfsCompression::Lz4 => {
|
||||||
|
Err(VfsError::Unsupported("LZ4 size detection not implemented".to_string()))
|
||||||
|
}
|
||||||
|
VfsCompression::None => Ok(data.len() as u64),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
pub mod compression;
|
||||||
pub mod local_fs;
|
pub mod local_fs;
|
||||||
pub mod open_flags;
|
pub mod open_flags;
|
||||||
pub mod s3_fs;
|
pub mod s3_fs;
|
||||||
@@ -259,3 +260,25 @@ pub struct VfsQuotaUsage {
|
|||||||
/// 是否超过硬限制
|
/// 是否超过硬限制
|
||||||
pub over_hard_limit: bool,
|
pub over_hard_limit: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 压缩算法类型
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum VfsCompression {
|
||||||
|
/// 无压缩
|
||||||
|
None,
|
||||||
|
/// LZ4压缩(快速)
|
||||||
|
Lz4,
|
||||||
|
/// ZSTD压缩(高压缩率)
|
||||||
|
Zstd,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 压缩配置
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct VfsCompressionConfig {
|
||||||
|
/// 压缩算法
|
||||||
|
pub algorithm: VfsCompression,
|
||||||
|
/// 压缩级别(1-22 for ZSTD, 1-12 for LZ4)
|
||||||
|
pub level: u32,
|
||||||
|
/// 最小压缩大小(字节),小于此大小不压缩
|
||||||
|
pub min_size: u64,
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user