Files
markbase/markbase-core/src/vfs/mod.rs
Warren ebe976eee4
Some checks failed
Test / test (push) Has been cancelled
Test / build (push) Has been cancelled
Implement Write/Read Cache (Phase 3)
2026-06-22 04:42:55 +08:00

669 lines
21 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
pub mod cache;
pub mod compression;
pub mod dedup;
pub mod local_fs;
pub mod open_flags;
pub mod raid;
pub mod s3_fs;
pub mod smb_fs;
#[cfg(feature = "smb-server")]
pub mod smb_server_backend;
pub mod util;
#[cfg(feature = "async-vfs")]
pub mod async_fs;
#[cfg(feature = "async-vfs")]
pub mod async_s3_fs;
#[cfg(feature = "async-vfs")]
pub mod async_smb_fs;
use std::path::{Path, PathBuf};
use std::time::SystemTime;
/// VFS 错误类型
#[derive(Debug, Clone)]
pub enum VfsError {
NotFound(String),
PermissionDenied(String),
AlreadyExists(String),
NotEmpty(String),
NotADirectory(String),
IsADirectory(String),
Unsupported(String),
Io(String),
UnexpectedEof,
}
impl std::fmt::Display for VfsError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VfsError::NotFound(p) => write!(f, "No such file or directory: {}", p),
VfsError::PermissionDenied(p) => write!(f, "Permission denied: {}", p),
VfsError::AlreadyExists(p) => write!(f, "File already exists: {}", p),
VfsError::NotEmpty(p) => write!(f, "Directory not empty: {}", p),
VfsError::NotADirectory(p) => write!(f, "Not a directory: {}", p),
VfsError::IsADirectory(p) => write!(f, "Is a directory: {}", p),
VfsError::Unsupported(msg) => write!(f, "Unsupported: {}", msg),
VfsError::Io(msg) => write!(f, "IO error: {}", msg),
VfsError::UnexpectedEof => write!(f, "Unexpected end of file"),
}
}
}
impl std::error::Error for VfsError {}
/// 文件统计信息(类似 libc::stat
#[derive(Debug, Clone)]
pub struct VfsStat {
pub size: u64,
pub mode: u32,
pub uid: u32,
pub gid: u32,
pub atime: SystemTime,
pub mtime: SystemTime,
pub is_dir: bool,
pub is_symlink: bool,
}
impl VfsStat {
pub fn new() -> Self {
Self {
size: 0,
mode: 0,
uid: 0,
gid: 0,
atime: SystemTime::UNIX_EPOCH,
mtime: SystemTime::UNIX_EPOCH,
is_dir: false,
is_symlink: false,
}
}
}
impl Default for VfsStat {
fn default() -> Self {
Self::new()
}
}
/// 目录条目
#[derive(Debug, Clone)]
pub struct VfsDirEntry {
pub name: String,
pub long_name: String,
pub stat: VfsStat,
}
/// 打开文件的抽象
pub trait VfsFile: Send {
fn read(&mut self, buf: &mut [u8]) -> Result<usize, VfsError>;
fn write(&mut self, buf: &[u8]) -> Result<usize, VfsError>;
fn seek(&mut self, pos: std::io::SeekFrom) -> Result<u64, VfsError>;
fn flush(&mut self) -> Result<(), VfsError>;
fn stat(&mut self) -> Result<VfsStat, VfsError>;
fn set_len(&mut self, size: u64) -> Result<(), VfsError>;
/// Write all bytes (convenience, default loops write() until done)
fn write_all(&mut self, mut buf: &[u8]) -> Result<(), VfsError> {
while !buf.is_empty() {
let n = self.write(buf)?;
if n == 0 {
return Err(VfsError::Io("write returned 0".to_string()));
}
buf = &buf[n..];
}
Ok(())
}
/// Read exactly `buf.len()` bytes (convenience, loops read() until done)
fn read_exact(&mut self, mut buf: &mut [u8]) -> Result<(), VfsError> {
while !buf.is_empty() {
let n = self.read(buf)?;
if n == 0 {
return Err(VfsError::UnexpectedEof);
}
buf = &mut buf[n..];
}
Ok(())
}
}
/// VFS 后端 trait所有文件系统操作
pub trait VfsBackend: Send + Sync {
/// Clone boxed
fn clone_boxed(&self) -> Box<dyn VfsBackend>;
/// 读取目录内容
fn read_dir(&self, path: &Path) -> Result<Vec<VfsDirEntry>, VfsError>;
/// 打开文件(读/写)
fn open_file(
&self,
path: &Path,
flags: &open_flags::OpenFlags,
) -> Result<Box<dyn VfsFile>, VfsError>;
/// 获取文件/目录元数据
fn stat(&self, path: &Path) -> Result<VfsStat, VfsError>;
fn lstat(&self, path: &Path) -> Result<VfsStat, VfsError>;
/// 创建目录
fn create_dir(&self, path: &Path, mode: u32) -> Result<(), VfsError>;
/// 递归创建目录
fn create_dir_all(&self, path: &Path, mode: u32) -> Result<(), VfsError>;
/// 删除空目录
fn remove_dir(&self, path: &Path) -> Result<(), VfsError>;
/// 递归删除目录及其所有内容
fn remove_dir_all(&self, path: &Path) -> Result<(), VfsError> {
// Default: read entries and remove one by one
let entries = self.read_dir(path)?;
for entry in entries {
let child = path.join(&entry.name);
if entry.stat.is_dir {
self.remove_dir_all(&child)?;
} else {
self.remove_file(&child)?;
}
}
self.remove_dir(path)
}
/// 删除文件
fn remove_file(&self, path: &Path) -> Result<(), VfsError>;
/// 重命名
fn rename(&self, from: &Path, to: &Path) -> Result<(), VfsError>;
/// 设置文件属性
fn set_stat(&self, path: &Path, stat: &VfsStat) -> Result<(), VfsError>;
/// 原子性设置 atime 和 mtime默认实现调用 stat + set_stat有 race condition
fn set_times(&self, path: &Path, atime: SystemTime, mtime: SystemTime) -> Result<(), VfsError> {
let mut stat = self.stat(path)?;
stat.atime = atime;
stat.mtime = mtime;
self.set_stat(path, &stat)
}
/// 原子性设置 atime默认实现调用 stat + set_stat有 race condition
fn set_atime(&self, path: &Path, atime: SystemTime) -> Result<(), VfsError> {
let mut stat = self.stat(path)?;
stat.atime = atime;
self.set_stat(path, &stat)
}
/// 原子性设置 mtime默认实现调用 stat + set_stat有 race condition
fn set_mtime(&self, path: &Path, mtime: SystemTime) -> Result<(), VfsError> {
let mut stat = self.stat(path)?;
stat.mtime = mtime;
self.set_stat(path, &stat)
}
/// 读取符号链接目标
fn read_link(&self, path: &Path) -> Result<PathBuf, VfsError>;
/// 创建符号链接
fn create_symlink(&self, target: &Path, link: &Path) -> Result<(), VfsError>;
/// 规范化路径
fn real_path(&self, path: &Path) -> Result<PathBuf, VfsError>;
/// 检查路径是否存在
fn exists(&self, path: &Path) -> bool;
/// 创建硬链接
fn hard_link(&self, original: &Path, link: &Path) -> Result<(), VfsError>;
/// 复制文件高效实现fallback 到 read+write
fn copy(&self, from: &Path, to: &Path) -> Result<(), VfsError> {
let flags = open_flags::OpenFlags::new().read();
let mut src = self.open_file(from, &flags)?;
let write_flags = open_flags::OpenFlags::new().write().create().truncate().mode(0o644);
let mut dst = self.open_file(to, &write_flags)?;
let mut buf = vec![0u8; 65536];
loop {
match src.read(&mut buf) {
Ok(0) => break,
Ok(n) => dst.write_all(&buf[..n])?,
Err(e) => return Err(e),
}
}
dst.flush()?;
Ok(())
}
// ===== Snapshot support (ZFS-style) =====
/// 创建快照
fn create_snapshot(&self, _path: &Path, _name: &str) -> Result<(), VfsError> {
Err(VfsError::Unsupported("create_snapshot".to_string()))
}
/// 列出快照
fn list_snapshots(&self, _path: &Path) -> Result<Vec<String>, VfsError> {
Err(VfsError::Unsupported("list_snapshots".to_string()))
}
/// 删除快照
fn delete_snapshot(&self, _path: &Path, _name: &str) -> Result<(), VfsError> {
Err(VfsError::Unsupported("delete_snapshot".to_string()))
}
/// 从快照恢复
fn restore_snapshot(&self, _path: &Path, _name: &str) -> Result<(), VfsError> {
Err(VfsError::Unsupported("restore_snapshot".to_string()))
}
/// 获取快照信息
fn snapshot_info(&self, _path: &Path, _name: &str) -> Result<VfsSnapshotInfo, VfsError> {
Err(VfsError::Unsupported("snapshot_info".to_string()))
}
// ===== Quota support =====
/// 设置配额限制(字节)
fn set_quota(&self, _path: &Path, _quota: &VfsQuota) -> Result<(), VfsError> {
Err(VfsError::Unsupported("set_quota".to_string()))
}
/// 获取配额信息
fn get_quota(&self, _path: &Path) -> Result<VfsQuota, VfsError> {
Err(VfsError::Unsupported("get_quota".to_string()))
}
/// 获取配额使用情况
fn get_quota_usage(&self, _path: &Path) -> Result<VfsQuotaUsage, VfsError> {
Err(VfsError::Unsupported("get_quota_usage".to_string()))
}
/// 检查配额(写入前检查)
fn check_quota(&self, _path: &Path, _size: u64) -> Result<bool, VfsError> {
Ok(true) // Default: no quota, always allow
}
// ===== Previous versions (shadow copy) =====
/// 列出文件的所有历史版本
fn list_previous_versions(&self, _path: &Path) -> Result<Vec<VfsPreviousVersion>, VfsError> {
Err(VfsError::Unsupported("list_previous_versions".to_string()))
}
/// 打开历史版本文件(通过 @GMT- token
fn open_previous_version(&self, _path: &Path, _gmt_token: &str) -> Result<Box<dyn VfsFile>, VfsError> {
Err(VfsError::Unsupported("open_previous_version".to_string()))
}
/// 从历史版本恢复文件
fn restore_previous_version(&self, _path: &Path, _gmt_token: &str) -> Result<(), VfsError> {
Err(VfsError::Unsupported("restore_previous_version".to_string()))
}
// ===== ACL support (NFSv4/SMB) =====
/// 获取文件ACL
fn get_acl(&self, _path: &Path) -> Result<VfsAcl, VfsError> {
Err(VfsError::Unsupported("get_acl".to_string()))
}
/// 设置文件ACL
fn set_acl(&self, _path: &Path, _acl: &VfsAcl) -> Result<(), VfsError> {
Err(VfsError::Unsupported("set_acl".to_string()))
}
/// 检查ACL权限
fn check_acl(&self, _path: &Path, _principal: &str, _mask: VfsAceMask) -> Result<bool, VfsError> {
Ok(true) // Default: no ACL, always allow
}
/// 添加ACE
fn add_ace(&self, _path: &Path, _ace: &VfsAce) -> Result<(), VfsError> {
Err(VfsError::Unsupported("add_ace".to_string()))
}
/// 移除ACE
fn remove_ace(&self, _path: &Path, _ace_index: usize) -> Result<(), VfsError> {
Err(VfsError::Unsupported("remove_ace".to_string()))
}
}
/// 快照信息
#[derive(Debug, Clone)]
pub struct VfsSnapshotInfo {
/// 快照名称
pub name: String,
/// 创建时间
pub created: SystemTime,
/// 快照大小(字节)
pub size: u64,
/// 是否只读
pub read_only: bool,
}
/// 配额设置
#[derive(Debug, Clone)]
pub struct VfsQuota {
/// 空间限制字节0表示无限制
pub space_limit: u64,
/// 文件数量限制0表示无限制
pub file_limit: u64,
/// 用户ID可选
pub user_id: Option<String>,
/// 软限制(字节),超过时警告
pub soft_limit: u64,
/// 宽限期(秒)
pub grace_period: u64,
}
/// 配额使用情况
#[derive(Debug, Clone)]
pub struct VfsQuotaUsage {
/// 已使用空间(字节)
pub space_used: u64,
/// 文件数量
pub files_used: u64,
/// 是否超过软限制
pub over_soft_limit: bool,
/// 是否超过硬限制
pub over_hard_limit: bool,
}
/// 历史版本信息SMB shadow copy
#[derive(Debug, Clone)]
pub struct VfsPreviousVersion {
/// 快照名称
pub snapshot_name: String,
/// GMT token (@GMT-YYYY.MM.DD-HH.MM.SS)
pub gmt_token: String,
/// 创建时间
pub created: SystemTime,
/// 版本大小(字节)
pub size: u64,
}
/// ACL访问控制条目类型NFSv4/SMB
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VfsAceType {
/// 允许访问
Allow,
/// 拒绝访问
Deny,
/// 审计SMB
Audit,
/// 警报SMB
Alarm,
}
/// ACL继承标志NFSv4/SMB
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VfsAceFlag {
/// 文件继承
FileInherit,
/// 目录继承
DirectoryInherit,
/// 无继承(仅当前对象)
NoPropagateInherit,
/// 仅继承(不应用于当前对象)
InheritOnly,
/// 已继承
Inherited,
/// 成功审计SMB
SuccessfulAccess,
/// 失败审计SMB
FailedAccess,
}
/// ACL权限掩码NFSv4
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VfsAceMask {
/// 读数据
ReadData,
/// 写数据
WriteData,
/// 执行
Execute,
/// 列目录(读数据+目录)
ListDirectory,
/// 添加文件(写数据+目录)
AddFile,
/// 添加子目录
AddSubdirectory,
/// 删除子项
DeleteChild,
/// 删除
Delete,
/// 读属性
ReadAttributes,
/// 写属性
WriteAttributes,
/// 读ACL
ReadNfsAcl,
/// 写ACL
WriteNfsAcl,
/// 读取所有权
ReadOwner,
/// 写入所有权
WriteOwner,
/// 同步
Synchronize,
/// 完全控制(所有权限)
FullControl,
}
/// ACL访问控制条目ACE
#[derive(Debug, Clone)]
pub struct VfsAce {
/// ACE类型
pub ace_type: VfsAceType,
/// ACE标志
pub flags: Vec<VfsAceFlag>,
/// 权限掩码
pub mask: Vec<VfsAceMask>,
/// 主体(用户/组SID或名称
pub principal: String,
}
/// ACL列表NFSv4/SMB
#[derive(Debug, Clone, Default)]
pub struct VfsAcl {
/// ACE列表
pub aces: Vec<VfsAce>,
/// 默认ACL仅目录
pub default_acl: Option<Box<VfsAcl>>,
}
/// 压缩算法类型
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VfsCompression {
/// 无压缩
None,
/// LZ4压缩快速
Lz4,
/// ZSTD压缩高压缩率
Zstd,
}
/// 压缩配置
#[derive(Debug, Clone)]
pub struct VfsCompressionConfig {
/// 压缩算法
pub algorithm: VfsCompression,
/// 压缩级别1-22 for ZSTD, 1-12 for LZ4
pub level: u32,
/// 最小压缩大小(字节),小于此大小不压缩
pub min_size: u64,
}
/// 去重配置
#[derive(Debug, Clone)]
pub struct VfsDedupConfig {
/// 块大小字节默认4KB
pub block_size: usize,
/// 最小文件大小(字节),小于此大小不去重
pub min_file_size: u64,
/// 去重存储路径
pub store_path: PathBuf,
}
impl Default for VfsDedupConfig {
fn default() -> Self {
Self {
block_size: 4096,
min_file_size: 1024,
store_path: PathBuf::from(".dedup"),
}
}
}
/// RAID级别ZFS RAID-Z
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VfsRaidLevel {
/// 单磁盘无RAID
Single,
/// RAID-Z1单奇偶校验类似RAID 5
RaidZ1,
/// RAID-Z2双奇偶校验类似RAID 6
RaidZ2,
/// RAID-Z3三奇偶校验
RaidZ3,
}
impl std::fmt::Display for VfsRaidLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VfsRaidLevel::Single => write!(f, "Single"),
VfsRaidLevel::RaidZ1 => write!(f, "RAID-Z1"),
VfsRaidLevel::RaidZ2 => write!(f, "RAID-Z2"),
VfsRaidLevel::RaidZ3 => write!(f, "RAID-Z3"),
}
}
}
/// RAID配置
#[derive(Debug, Clone)]
pub struct VfsRaidConfig {
/// RAID级别
pub level: VfsRaidLevel,
/// 条带大小字节默认64KB
pub stripe_size: usize,
/// 磁盘列表路径
pub disk_paths: Vec<PathBuf>,
}
impl Default for VfsRaidConfig {
fn default() -> Self {
Self {
level: VfsRaidLevel::Single,
stripe_size: 65536,
disk_paths: Vec::new(),
}
}
}
// ===== Async VfsBackend Design (Phase 1 - Framework) =====
/// Async VFS 文件 trait用于异步操作
///
/// 设计要点:
/// 1. 使用 `async fn` in traits (Rust 1.75+)
/// 2. 所有方法返回 `Pin<Box<dyn Future>>`
/// 3. 与 VfsFile 保持一致的接口
#[cfg(feature = "async-vfs")]
pub trait AsyncVfsFile: Send + Sync {
fn read<'a>(&'a mut self, buf: &'a mut [u8]) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<usize, VfsError>> + Send + 'a>>;
fn write<'a>(&'a mut self, buf: &'a [u8]) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<usize, VfsError>> + Send + 'a>>;
fn seek<'a>(&'a mut self, pos: std::io::SeekFrom) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<u64, VfsError>> + Send + 'a>>;
fn flush<'a>(&'a mut self) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
}
/// Async VFS 后端 trait用于异步文件系统操作
///
/// 设计要点:
/// 1. 使用 `async fn` in traits (Rust 1.75+)
/// 2. 所有方法返回 `Pin<Box<dyn Future>>`
/// 3. 与 VfsBackend 保持一致的接口
/// 4. 用于 WebDAV/SMB/SSH 异步处理
#[cfg(feature = "async-vfs")]
pub trait AsyncVfsBackend: Send + Sync {
fn clone_boxed(&self) -> Box<dyn AsyncVfsBackend>;
fn read_dir<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Vec<VfsDirEntry>, VfsError>> + Send + 'a>>;
fn open_file<'a>(&'a self, path: &'a Path, flags: &'a open_flags::OpenFlags) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Box<dyn AsyncVfsFile>, VfsError>> + Send + 'a>>;
fn stat<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<VfsStat, VfsError>> + Send + 'a>>;
fn create_dir<'a>(&'a self, path: &'a Path, mode: u32) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
fn remove_dir<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
fn remove_file<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
fn exists<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + 'a>>;
}
// ===== Async VfsBackend Implementation Notes =====
//
// Phase 2: AsyncLocalFs (tokio::fs)
// - 使用 tokio::fs::File 替代 std::fs::File
// - 使用 tokio::fs::read_dir 替代 std::fs::read_dir
// - 使用 tokio::fs::create_dir 替代 std::fs::create_dir
//
// Phase 3: AsyncS3Vfs (ureq is blocking, need async client)
// - 使用 async-s3 或 rusoto
// - 或者使用 spawn_blocking 包装现有 ureq 调用
//
// Phase 4: AsyncSmbVfs
// - smb-server crate 使用 async internally
// - 需要 async wrapper
//
// Phase 5: WebDAV Integration
// - VfsDavFs 改为 AsyncVfsBackend
// - dav-server 已经是 async
// - 直接使用 async 方法
//
// 预估工作量:
// - AsyncVfsBackend trait: 1 hour
// - AsyncLocalFs: 3 hours
// - AsyncS3Vfs: 2 hours
// - AsyncSmbVfs: 2 hours
// - WebDAV integration: 3 hours
// - Tests: 2 hours
// Total: ~13 hours (multi-day project)
//
// ===== Phase 5 WebDAV Async Integration Design =====
//
// 现状分析:
// 1. dav-server DavFileSystem trait 方法返回 Pin<Box<dyn Future>>
// 2. 当前 VfsDavFs::open() 返回 Box::pin(ready(...))
// 3. 这是 "同步包装为 Future",不是真正的 async
// 4. DavFileSystem trait API 已变化2026-06-21 session发现
// - read_dir(path, ReadDirMeta) 而非 read_dir(path, depth)
// - have_props(path) 返回 Pin<Box<dyn Future>>
// - get_props/get_prop/patch_props 新方法
// - get_quota/set_accessed/set_modified 新方法
// - DavFile 需要 write_buf 方法
// - DavMetaData modified()/is_dir() 返回 Pin<Box<dyn Future>>
// - DavDirEntry name()/is_dir()/metadata() 返回 Pin<Box<dyn Future>>
//
// Phase 5 阻塞因素:
// 1. dav-server API 签名与预期不匹配20+ 编译错误)
// 2. 需要 match 完整 DavFileSystem trait 所有方法(~30个
// 3. AsyncVfsFile trait 方法签名需调整
// 4. 估算工作量:~8小时而非原估计3小时
//
// 实现方案选择:
// 方案Aspawn_blocking wrapper推荐
// - 创建 AsyncVfsDavFs 包装现有 VfsDavFs
// - 所有 DavFileSystem 方法使用 spawn_blocking 调用同步版本
// - 工作量:~2小时
// - 优点:快速实现,兼容现有 API
// - 缺点:仍为伪异步(阻塞线程池)
//
// 方案B完整重写 DavFileSystem长期
// - 完全匹配 dav-server API
// - 使用真正的 AsyncVfsBackend async 方法
// - 工作量:~8小时
// - 优点:真正的异步
// - 缺点:需要完全理解 dav-server API
//
// 推荐方案Aspawn_blocking wrapper
//
// 预估工作量Phase 5 方案A ~2小时方案B ~8小时