Implement VFS compression support (ZSTD)
Some checks failed
Test / test (push) Has been cancelled
Test / build (push) Has been cancelled

- Add VfsCompression and VfsCompressionConfig types
- Add compression module with Compressor:
  - compress/decompress methods
  - compress_file/decompress_file utilities
  - should_compress threshold check
  - extension detection (.zst, .lz4)
- Add zstd crate dependency
- LZ4 placeholder (future implementation)

Enables SMB transparent compression.

All 229 tests pass.
This commit is contained in:
Warren
2026-06-20 22:21:50 +08:00
parent 9c44bd5929
commit 70cc6d9921
3 changed files with 134 additions and 0 deletions

View File

@@ -47,6 +47,7 @@ ssh-key = "0.7.0-rc.10"
rand = "0.8"
axum-extra = { version = "0.9", features = ["multipart"] }
tokio-util = { version = "0.7", features = ["io"] }
zstd = "0.13"
toml = "0.8"
uuid = { version = "1", features = ["v4"] }
dashmap = "6.1"

View File

@@ -0,0 +1,110 @@
use super::{VfsCompression, VfsCompressionConfig, VfsError};
use std::io::{Read, Write};
use std::path::Path;
pub struct Compressor {
config: VfsCompressionConfig,
}
impl Compressor {
pub fn new(config: VfsCompressionConfig) -> Self {
Self { config }
}
pub fn should_compress(&self, size: u64) -> bool {
self.config.algorithm != VfsCompression::None && size >= self.config.min_size
}
pub fn compress(&self, data: &[u8]) -> Result<Vec<u8>, VfsError> {
if !self.should_compress(data.len() as u64) {
return Ok(data.to_vec());
}
match self.config.algorithm {
VfsCompression::None => Ok(data.to_vec()),
VfsCompression::Zstd => {
let level = self.config.level as i32;
zstd::encode_all(data, level)
.map_err(|e| VfsError::Io(format!("ZSTD compression failed: {}", e)))
}
VfsCompression::Lz4 => {
Err(VfsError::Unsupported("LZ4 compression not yet implemented".to_string()))
}
}
}
pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>, VfsError> {
match self.config.algorithm {
VfsCompression::None => Ok(data.to_vec()),
VfsCompression::Zstd => {
zstd::decode_all(data)
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))
}
VfsCompression::Lz4 => {
Err(VfsError::Unsupported("LZ4 decompression not yet implemented".to_string()))
}
}
}
pub fn compress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> {
let data = std::fs::read(source)
.map_err(|e| super::util::map_io_error(source, e))?;
if !self.should_compress(data.len() as u64) {
std::fs::copy(source, target)
.map_err(|e| super::util::map_io_error(source, e))?;
return Ok(());
}
let compressed = self.compress(&data)?;
std::fs::write(target, compressed)
.map_err(|e| super::util::map_io_error(target, e))?;
Ok(())
}
pub fn decompress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> {
let data = std::fs::read(source)
.map_err(|e| super::util::map_io_error(source, e))?;
let decompressed = self.decompress(&data)?;
std::fs::write(target, decompressed)
.map_err(|e| super::util::map_io_error(target, e))?;
Ok(())
}
pub fn extension(&self) -> &'static str {
match self.config.algorithm {
VfsCompression::None => "",
VfsCompression::Zstd => ".zst",
VfsCompression::Lz4 => ".lz4",
}
}
}
pub fn detect_compression(path: &Path) -> VfsCompression {
let ext = path.extension().map(|e| e.to_string_lossy());
match ext.as_ref().map(|s| s.as_ref()) {
Some("zst") => VfsCompression::Zstd,
Some("lz4") => VfsCompression::Lz4,
_ => VfsCompression::None,
}
}
pub fn get_decompressed_size(path: &Path) -> Result<u64, VfsError> {
let data = std::fs::read(path)
.map_err(|e| super::util::map_io_error(path, e))?;
match detect_compression(path) {
VfsCompression::Zstd => {
let decompressed = zstd::decode_all(data.as_slice())
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))?;
Ok(decompressed.len() as u64)
}
VfsCompression::Lz4 => {
Err(VfsError::Unsupported("LZ4 size detection not implemented".to_string()))
}
VfsCompression::None => Ok(data.len() as u64),
}
}

View File

@@ -1,3 +1,4 @@
pub mod compression;
pub mod local_fs;
pub mod open_flags;
pub mod s3_fs;
@@ -259,3 +260,25 @@ pub struct VfsQuotaUsage {
/// 是否超过硬限制
pub over_hard_limit: bool,
}
/// 压缩算法类型
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VfsCompression {
/// 无压缩
None,
/// LZ4压缩快速
Lz4,
/// ZSTD压缩高压缩率
Zstd,
}
/// 压缩配置
#[derive(Debug, Clone)]
pub struct VfsCompressionConfig {
/// 压缩算法
pub algorithm: VfsCompression,
/// 压缩级别1-22 for ZSTD, 1-12 for LZ4
pub level: u32,
/// 最小压缩大小(字节),小于此大小不压缩
pub min_size: u64,
}