#define FUSE_USE_VERSION 31 #include #include #include #include #include #include #include #include #include #include #include static sqlite3 *db = NULL; static const char *db_path = "/Users/accusys/markbase/data/users/warren.sqlite"; static pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER; // Optimized cache with mmap support typedef struct { char node_id[64]; char file_path[512]; long file_size; int access_count; time_t last_access; void *mmap_ptr; // mmap pointer for large files int mmap_fd; } FileCacheEntry; #define CACHE_SIZE 200 #define MMAP_THRESHOLD 1048576 // 1MB threshold for mmap static FileCacheEntry file_cache[CACHE_SIZE]; static int cache_count = 0; static pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER; // Node info cache typedef struct { char node_id[64]; char node_type[20]; long file_size; char parent_id[64]; } NodeCacheEntry; #define NODE_CACHE_SIZE 500 static NodeCacheEntry node_cache[NODE_CACHE_SIZE]; static int node_cache_count = 0; // Large read buffer (64KB) #define READ_BUFFER_SIZE 65536 static int init_db() { pthread_mutex_lock(&db_mutex); int result = sqlite3_open_v2(db_path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, NULL); pthread_mutex_unlock(&db_mutex); if (result != SQLITE_OK) { fprintf(stderr, "Cannot open database\n"); return -1; } printf("Database opened (read-only optimized)\n"); return 0; } static FileCacheEntry* cache_lookup(const char *node_id) { pthread_mutex_lock(&cache_mutex); for (int i = 0; i < cache_count; i++) { if (strcmp(file_cache[i].node_id, node_id) == 0) { file_cache[i].access_count++; file_cache[i].last_access = time(NULL); pthread_mutex_unlock(&cache_mutex); return &file_cache[i]; } } pthread_mutex_unlock(&cache_mutex); return NULL; } static void cache_insert(const char *node_id, const char *file_path, long file_size) { pthread_mutex_lock(&cache_mutex); for (int i = 0; i < cache_count; i++) { if (strcmp(file_cache[i].node_id, node_id) == 0) { pthread_mutex_unlock(&cache_mutex); return; } } if (cache_count >= CACHE_SIZE) { int lru_index = 0; time_t oldest_time = file_cache[0].last_access; for (int i = 1; i < cache_count; i++) { if (file_cache[i].last_access < oldest_time) { oldest_time = file_cache[i].last_access; lru_index = i; } } // Unmap if mapped if (file_cache[lru_index].mmap_ptr) { munmap(file_cache[lru_index].mmap_ptr, file_cache[lru_index].file_size); close(file_cache[lru_index].mmap_fd); } strcpy(file_cache[lru_index].node_id, node_id); strcpy(file_cache[lru_index].file_path, file_path); file_cache[lru_index].file_size = file_size; file_cache[lru_index].access_count = 1; file_cache[lru_index].last_access = time(NULL); file_cache[lru_index].mmap_ptr = NULL; file_cache[lru_index].mmap_fd = -1; } else { strcpy(file_cache[cache_count].node_id, node_id); strcpy(file_cache[cache_count].file_path, file_path); file_cache[cache_count].file_size = file_size; file_cache[cache_count].access_count = 1; file_cache[cache_count].last_access = time(NULL); file_cache[cache_count].mmap_ptr = NULL; file_cache[cache_count].mmap_fd = -1; cache_count++; } pthread_mutex_unlock(&cache_mutex); } static NodeCacheEntry* node_cache_lookup(const char *node_id) { pthread_mutex_lock(&cache_mutex); for (int i = 0; i < node_cache_count; i++) { if (strcmp(node_cache[i].node_id, node_id) == 0) { pthread_mutex_unlock(&cache_mutex); return &node_cache[i]; } } pthread_mutex_unlock(&cache_mutex); return NULL; } static void node_cache_insert(const char *node_id, const char *node_type, long file_size, const char *parent_id) { pthread_mutex_lock(&cache_mutex); if (node_cache_count < NODE_CACHE_SIZE) { strcpy(node_cache[node_cache_count].node_id, node_id); strcpy(node_cache[node_cache_count].node_type, node_type); node_cache[node_cache_count].file_size = file_size; if (parent_id) strcpy(node_cache[node_cache_count].parent_id, parent_id); else node_cache[node_cache_count].parent_id[0] = '\0'; node_cache_count++; } pthread_mutex_unlock(&cache_mutex); } static void *mb_init(struct fuse_conn_info *conn, struct fuse_config *cfg) { (void) conn; cfg->kernel_cache = 1; init_db(); // Pre-cache top 200 largest files (likely to be accessed) pthread_mutex_lock(&db_mutex); const char *sql = "SELECT f.file_uuid, l.location, f.file_size " "FROM file_nodes f " "JOIN file_locations l ON f.file_uuid = l.file_uuid " "WHERE f.file_size > ? " "ORDER BY f.file_size DESC LIMIT 200"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { sqlite3_bind_int64(stmt, 1, 1048576); // Files > 1MB int cached = 0; while (sqlite3_step(stmt) == SQLITE_ROW) { const char *file_uuid = (const char*)sqlite3_column_text(stmt, 0); const char *location = (const char*)sqlite3_column_text(stmt, 1); long file_size = sqlite3_column_int64(stmt, 2); cache_insert(file_uuid, location, file_size); cached++; } sqlite3_finalize(stmt); printf("Pre-cached %d large files (>1MB)\n", cached); } pthread_mutex_unlock(&db_mutex); return NULL; } static void mb_destroy(void *userdata) { (void) userdata; // Unmap all mapped files pthread_mutex_lock(&cache_mutex); for (int i = 0; i < cache_count; i++) { if (file_cache[i].mmap_ptr) { munmap(file_cache[i].mmap_ptr, file_cache[i].file_size); close(file_cache[i].mmap_fd); } } pthread_mutex_unlock(&cache_mutex); printf("Cache stats: %d files, %d nodes\n", cache_count, node_cache_count); if (db) { pthread_mutex_lock(&db_mutex); sqlite3_close(db); pthread_mutex_unlock(&db_mutex); } } static char* find_node_id(const char *path) { if (strcmp(path, "/") == 0) return NULL; char *path_copy = strdup(path); char *components[20]; int depth = 0; char *token = strtok(path_copy + 1, "/"); while (token && depth < 20) { components[depth++] = strdup(token); token = strtok(NULL, "/"); } free(path_copy); if (depth == 0) return NULL; char *current_parent_id = NULL; pthread_mutex_lock(&db_mutex); for (int level = 0; level < depth; level++) { const char *sql; sqlite3_stmt *stmt; if (level == 0) { sql = "SELECT node_id, node_type, file_size, parent_id " "FROM file_nodes WHERE label = ? AND (parent_id IS NULL OR parent_id = '')"; } else { sql = "SELECT node_id, node_type, file_size, parent_id " "FROM file_nodes WHERE label = ? AND parent_id = ?"; } if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { pthread_mutex_unlock(&db_mutex); for (int i = 0; i < depth; i++) free(components[i]); if (current_parent_id) free(current_parent_id); return NULL; } sqlite3_bind_text(stmt, 1, components[level], -1, SQLITE_STATIC); if (level > 0 && current_parent_id) { sqlite3_bind_text(stmt, 2, current_parent_id, -1, SQLITE_STATIC); } if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); pthread_mutex_unlock(&db_mutex); for (int i = 0; i < depth; i++) free(components[i]); if (current_parent_id) free(current_parent_id); return NULL; } const char *found_node_id = (const char*)sqlite3_column_text(stmt, 0); const char *node_type = (const char*)sqlite3_column_text(stmt, 1); long file_size = sqlite3_column_int64(stmt, 2); const char *parent_id = (const char*)sqlite3_column_text(stmt, 3); node_cache_insert(found_node_id, node_type, file_size, parent_id); if (current_parent_id) free(current_parent_id); current_parent_id = strdup(found_node_id); sqlite3_finalize(stmt); } pthread_mutex_unlock(&db_mutex); for (int i = 0; i < depth; i++) free(components[i]); return current_parent_id; } static int mb_getattr(const char *path, struct stat *stbuf, struct fuse_file_info *fi) { (void) fi; memset(stbuf, 0, sizeof(struct stat)); if (strcmp(path, "/") == 0) { stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; return 0; } char *node_id = find_node_id(path); if (!node_id) return -ENOENT; NodeCacheEntry *cached_node = node_cache_lookup(node_id); if (cached_node) { if (strcmp(cached_node->node_type, "folder") == 0) { stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; } else { stbuf->st_mode = S_IFREG | 0444; stbuf->st_nlink = 1; stbuf->st_size = cached_node->file_size; } free(node_id); return 0; } pthread_mutex_lock(&db_mutex); const char *sql = "SELECT node_type, file_size FROM file_nodes WHERE node_id = ?"; sqlite3_stmt *stmt; int result = -ENOENT; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { sqlite3_bind_text(stmt, 1, node_id, -1, SQLITE_STATIC); if (sqlite3_step(stmt) == SQLITE_ROW) { const char *node_type = (const char*)sqlite3_column_text(stmt, 0); long file_size = sqlite3_column_int64(stmt, 1); if (strcmp(node_type, "folder") == 0) { stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; } else { stbuf->st_mode = S_IFREG | 0444; stbuf->st_nlink = 1; stbuf->st_size = file_size; } result = 0; } sqlite3_finalize(stmt); } pthread_mutex_unlock(&db_mutex); free(node_id); return result; } static int mb_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags) { (void) offset; (void) fi; (void) flags; filler(buf, ".", NULL, 0, FUSE_FILL_DIR_DEFAULTS); filler(buf, "..", NULL, 0, FUSE_FILL_DIR_DEFAULTS); pthread_mutex_lock(&db_mutex); if (strcmp(path, "/") == 0) { const char *sql = "SELECT label FROM file_nodes WHERE parent_id IS NULL OR parent_id = ''"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { while (sqlite3_step(stmt) == SQLITE_ROW) { const char *label = (const char*)sqlite3_column_text(stmt, 0); if (label) filler(buf, label, NULL, 0, FUSE_FILL_DIR_DEFAULTS); } sqlite3_finalize(stmt); } pthread_mutex_unlock(&db_mutex); return 0; } char *parent_node_id = find_node_id(path); if (!parent_node_id) { pthread_mutex_unlock(&db_mutex); return -ENOENT; } const char *sql = "SELECT label FROM file_nodes WHERE parent_id = ?"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { sqlite3_bind_text(stmt, 1, parent_node_id, -1, SQLITE_STATIC); while (sqlite3_step(stmt) == SQLITE_ROW) { const char *label = (const char*)sqlite3_column_text(stmt, 0); if (label) filler(buf, label, NULL, 0, FUSE_FILL_DIR_DEFAULTS); } sqlite3_finalize(stmt); } pthread_mutex_unlock(&db_mutex); free(parent_node_id); return 0; } static int mb_open(const char *path, struct fuse_file_info *fi) { if ((fi->flags & O_ACCMODE) != O_RDONLY) return -EACCES; char *node_id = find_node_id(path); if (!node_id) return -ENOENT; free(node_id); return 0; } // Optimized read with mmap for large files static int mb_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { (void) fi; char *node_id = find_node_id(path); if (!node_id) return -ENOENT; FileCacheEntry *cached = cache_lookup(node_id); // Use mmap for large files (>1MB) if (cached && cached->file_size > MMAP_THRESHOLD) { pthread_mutex_lock(&cache_mutex); // mmap if not already mapped if (!cached->mmap_ptr) { int fd = open(cached->file_path, O_RDONLY); if (fd >= 0) { void *ptr = mmap(NULL, cached->file_size, PROT_READ, MAP_PRIVATE, fd, 0); if (ptr != MAP_FAILED) { cached->mmap_ptr = ptr; cached->mmap_fd = fd; } else { close(fd); } } } if (cached->mmap_ptr) { // Read from mmap if (offset >= cached->file_size) { pthread_mutex_unlock(&cache_mutex); free(node_id); return 0; } size_t bytes_to_read = size; if (offset + size > cached->file_size) { bytes_to_read = cached->file_size - offset; } memcpy(buf, cached->mmap_ptr + offset, bytes_to_read); pthread_mutex_unlock(&cache_mutex); free(node_id); return bytes_to_read; } pthread_mutex_unlock(&cache_mutex); } // Use fread for small files or if mmap failed if (cached && strcmp(cached->file_path, "") != 0) { FILE *fp = fopen(cached->file_path, "rb"); if (fp) { if (fseek(fp, offset, SEEK_SET) == 0) { // Read in large chunks (64KB) size_t total_read = 0; while (total_read < size) { size_t chunk_size = (size - total_read > READ_BUFFER_SIZE) ? READ_BUFFER_SIZE : size - total_read; size_t bytes_read = fread(buf + total_read, 1, chunk_size, fp); total_read += bytes_read; if (bytes_read < chunk_size) break; // EOF or error } fclose(fp); free(node_id); return total_read; } fclose(fp); } } // Query from database if not cached pthread_mutex_lock(&db_mutex); const char *sql = "SELECT location FROM file_locations WHERE file_uuid = ? LIMIT 1"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { pthread_mutex_unlock(&db_mutex); free(node_id); return -EIO; } sqlite3_bind_text(stmt, 1, node_id, -1, SQLITE_STATIC); if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); pthread_mutex_unlock(&db_mutex); free(node_id); return -ENOENT; } const char *file_path = (const char*)sqlite3_column_text(stmt, 0); char *path_copy = strdup(file_path); sqlite3_finalize(stmt); pthread_mutex_unlock(&db_mutex); NodeCacheEntry *node_info = node_cache_lookup(node_id); if (node_info) { cache_insert(node_id, path_copy, node_info->file_size); } free(node_id); FILE *fp = fopen(path_copy, "rb"); if (!fp) { free(path_copy); return -ENOENT; } if (fseek(fp, offset, SEEK_SET) != 0) { fclose(fp); free(path_copy); return -EIO; } size_t bytes_read = fread(buf, 1, size, fp); fclose(fp); free(path_copy); return bytes_read; } static const struct fuse_operations mb_oper = { .init = mb_init, .destroy = mb_destroy, .getattr = mb_getattr, .readdir = mb_readdir, .open = mb_open, .read = mb_read, }; int main(int argc, char *argv[]) { printf("MarkBase FUSE v11.0 - Throughput Optimized\n"); printf("==========================================\n"); printf("Optimizations:\n"); printf(" - mmap for large files (>1MB)\n"); printf(" - Large read buffer (64KB)\n"); printf(" - Pre-cache 200 large files\n"); printf(" - Kernel caching enabled\n"); printf(" - Thread-safe SQLite\n"); printf("\n"); return fuse_main(argc, argv, &mb_oper, NULL); }