Implement VFS compression support (ZSTD)
- Add VfsCompression and VfsCompressionConfig types - Add compression module with Compressor: - compress/decompress methods - compress_file/decompress_file utilities - should_compress threshold check - extension detection (.zst, .lz4) - Add zstd crate dependency - LZ4 placeholder (future implementation) Enables SMB transparent compression. All 229 tests pass.
This commit is contained in:
110
markbase-core/src/vfs/compression.rs
Normal file
110
markbase-core/src/vfs/compression.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use super::{VfsCompression, VfsCompressionConfig, VfsError};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::Path;
|
||||
|
||||
pub struct Compressor {
|
||||
config: VfsCompressionConfig,
|
||||
}
|
||||
|
||||
impl Compressor {
|
||||
pub fn new(config: VfsCompressionConfig) -> Self {
|
||||
Self { config }
|
||||
}
|
||||
|
||||
pub fn should_compress(&self, size: u64) -> bool {
|
||||
self.config.algorithm != VfsCompression::None && size >= self.config.min_size
|
||||
}
|
||||
|
||||
pub fn compress(&self, data: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||
if !self.should_compress(data.len() as u64) {
|
||||
return Ok(data.to_vec());
|
||||
}
|
||||
|
||||
match self.config.algorithm {
|
||||
VfsCompression::None => Ok(data.to_vec()),
|
||||
VfsCompression::Zstd => {
|
||||
let level = self.config.level as i32;
|
||||
zstd::encode_all(data, level)
|
||||
.map_err(|e| VfsError::Io(format!("ZSTD compression failed: {}", e)))
|
||||
}
|
||||
VfsCompression::Lz4 => {
|
||||
Err(VfsError::Unsupported("LZ4 compression not yet implemented".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||
match self.config.algorithm {
|
||||
VfsCompression::None => Ok(data.to_vec()),
|
||||
VfsCompression::Zstd => {
|
||||
zstd::decode_all(data)
|
||||
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))
|
||||
}
|
||||
VfsCompression::Lz4 => {
|
||||
Err(VfsError::Unsupported("LZ4 decompression not yet implemented".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> {
|
||||
let data = std::fs::read(source)
|
||||
.map_err(|e| super::util::map_io_error(source, e))?;
|
||||
|
||||
if !self.should_compress(data.len() as u64) {
|
||||
std::fs::copy(source, target)
|
||||
.map_err(|e| super::util::map_io_error(source, e))?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let compressed = self.compress(&data)?;
|
||||
std::fs::write(target, compressed)
|
||||
.map_err(|e| super::util::map_io_error(target, e))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn decompress_file(&self, source: &Path, target: &Path) -> Result<(), VfsError> {
|
||||
let data = std::fs::read(source)
|
||||
.map_err(|e| super::util::map_io_error(source, e))?;
|
||||
|
||||
let decompressed = self.decompress(&data)?;
|
||||
std::fs::write(target, decompressed)
|
||||
.map_err(|e| super::util::map_io_error(target, e))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn extension(&self) -> &'static str {
|
||||
match self.config.algorithm {
|
||||
VfsCompression::None => "",
|
||||
VfsCompression::Zstd => ".zst",
|
||||
VfsCompression::Lz4 => ".lz4",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_compression(path: &Path) -> VfsCompression {
|
||||
let ext = path.extension().map(|e| e.to_string_lossy());
|
||||
match ext.as_ref().map(|s| s.as_ref()) {
|
||||
Some("zst") => VfsCompression::Zstd,
|
||||
Some("lz4") => VfsCompression::Lz4,
|
||||
_ => VfsCompression::None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_decompressed_size(path: &Path) -> Result<u64, VfsError> {
|
||||
let data = std::fs::read(path)
|
||||
.map_err(|e| super::util::map_io_error(path, e))?;
|
||||
|
||||
match detect_compression(path) {
|
||||
VfsCompression::Zstd => {
|
||||
let decompressed = zstd::decode_all(data.as_slice())
|
||||
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))?;
|
||||
Ok(decompressed.len() as u64)
|
||||
}
|
||||
VfsCompression::Lz4 => {
|
||||
Err(VfsError::Unsupported("LZ4 size detection not implemented".to_string()))
|
||||
}
|
||||
VfsCompression::None => Ok(data.len() as u64),
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod compression;
|
||||
pub mod local_fs;
|
||||
pub mod open_flags;
|
||||
pub mod s3_fs;
|
||||
@@ -259,3 +260,25 @@ pub struct VfsQuotaUsage {
|
||||
/// 是否超过硬限制
|
||||
pub over_hard_limit: bool,
|
||||
}
|
||||
|
||||
/// 压缩算法类型
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum VfsCompression {
|
||||
/// 无压缩
|
||||
None,
|
||||
/// LZ4压缩(快速)
|
||||
Lz4,
|
||||
/// ZSTD压缩(高压缩率)
|
||||
Zstd,
|
||||
}
|
||||
|
||||
/// 压缩配置
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VfsCompressionConfig {
|
||||
/// 压缩算法
|
||||
pub algorithm: VfsCompression,
|
||||
/// 压缩级别(1-22 for ZSTD, 1-12 for LZ4)
|
||||
pub level: u32,
|
||||
/// 最小压缩大小(字节),小于此大小不压缩
|
||||
pub min_size: u64,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user