markbase/docs/fuse_poc/markbase_v11_fast.c

#define FUSE_USE_VERSION 31
#include <fuse3/fuse.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <sqlite3.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>
#include <pthread.h>
#include <sys/mman.h>

static sqlite3 *db = NULL;
static const char *db_path = "/Users/accusys/markbase/data/users/warren.sqlite";
static pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER;

// Optimized cache with mmap support
typedef struct {
    char node_id[64];
    char file_path[512];
    long file_size;
    int access_count;
    time_t last_access;
    void *mmap_ptr;  // mmap pointer for large files
    int mmap_fd;
} FileCacheEntry;

#define CACHE_SIZE 200
#define MMAP_THRESHOLD 1048576  // 1MB threshold for mmap

static FileCacheEntry file_cache[CACHE_SIZE];
static int cache_count = 0;
static pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER;

// Node info cache
typedef struct {
    char node_id[64];
    char node_type[20];
    long file_size;
    char parent_id[64];
} NodeCacheEntry;

#define NODE_CACHE_SIZE 500
static NodeCacheEntry node_cache[NODE_CACHE_SIZE];
static int node_cache_count = 0;

// Large read buffer (64KB)
#define READ_BUFFER_SIZE 65536

static int init_db() {
    pthread_mutex_lock(&db_mutex);
    int result = sqlite3_open_v2(db_path, &db, 
                                  SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, NULL);
    pthread_mutex_unlock(&db_mutex);
    
    if (result != SQLITE_OK) {
        fprintf(stderr, "Cannot open database\n");
        return -1;
    }
    
    printf("Database opened (read-only optimized)\n");
    return 0;
}

static FileCacheEntry* cache_lookup(const char *node_id) {
    pthread_mutex_lock(&cache_mutex);
    for (int i = 0; i < cache_count; i++) {
        if (strcmp(file_cache[i].node_id, node_id) == 0) {
            file_cache[i].access_count++;
            file_cache[i].last_access = time(NULL);
            pthread_mutex_unlock(&cache_mutex);
            return &file_cache[i];
        }
    }
    pthread_mutex_unlock(&cache_mutex);
    return NULL;
}

static void cache_insert(const char *node_id, const char *file_path, long file_size) {
    pthread_mutex_lock(&cache_mutex);
    
    for (int i = 0; i < cache_count; i++) {
        if (strcmp(file_cache[i].node_id, node_id) == 0) {
            pthread_mutex_unlock(&cache_mutex);
            return;
        }
    }
    
    if (cache_count >= CACHE_SIZE) {
        int lru_index = 0;
        time_t oldest_time = file_cache[0].last_access;
        
        for (int i = 1; i < cache_count; i++) {
            if (file_cache[i].last_access < oldest_time) {
                oldest_time = file_cache[i].last_access;
                lru_index = i;
            }
        }
        
        // Unmap if mapped
        if (file_cache[lru_index].mmap_ptr) {
            munmap(file_cache[lru_index].mmap_ptr, file_cache[lru_index].file_size);
            close(file_cache[lru_index].mmap_fd);
        }
        
        strcpy(file_cache[lru_index].node_id, node_id);
        strcpy(file_cache[lru_index].file_path, file_path);
        file_cache[lru_index].file_size = file_size;
        file_cache[lru_index].access_count = 1;
        file_cache[lru_index].last_access = time(NULL);
        file_cache[lru_index].mmap_ptr = NULL;
        file_cache[lru_index].mmap_fd = -1;
    } else {
        strcpy(file_cache[cache_count].node_id, node_id);
        strcpy(file_cache[cache_count].file_path, file_path);
        file_cache[cache_count].file_size = file_size;
        file_cache[cache_count].access_count = 1;
        file_cache[cache_count].last_access = time(NULL);
        file_cache[cache_count].mmap_ptr = NULL;
        file_cache[cache_count].mmap_fd = -1;
        cache_count++;
    }
    
    pthread_mutex_unlock(&cache_mutex);
}

static NodeCacheEntry* node_cache_lookup(const char *node_id) {
    pthread_mutex_lock(&cache_mutex);
    for (int i = 0; i < node_cache_count; i++) {
        if (strcmp(node_cache[i].node_id, node_id) == 0) {
            pthread_mutex_unlock(&cache_mutex);
            return &node_cache[i];
        }
    }
    pthread_mutex_unlock(&cache_mutex);
    return NULL;
}

static void node_cache_insert(const char *node_id, const char *node_type, 
                               long file_size, const char *parent_id) {
    pthread_mutex_lock(&cache_mutex);
    
    if (node_cache_count < NODE_CACHE_SIZE) {
        strcpy(node_cache[node_cache_count].node_id, node_id);
        strcpy(node_cache[node_cache_count].node_type, node_type);
        node_cache[node_cache_count].file_size = file_size;
        if (parent_id) strcpy(node_cache[node_cache_count].parent_id, parent_id);
        else node_cache[node_cache_count].parent_id[0] = '\0';
        node_cache_count++;
    }
    
    pthread_mutex_unlock(&cache_mutex);
}

static void *mb_init(struct fuse_conn_info *conn, struct fuse_config *cfg) {
    (void) conn;
    cfg->kernel_cache = 1;
    
    init_db();
    
    // Pre-cache top 200 largest files (likely to be accessed)
    pthread_mutex_lock(&db_mutex);
    const char *sql = "SELECT f.file_uuid, l.location, f.file_size "
                      "FROM file_nodes f "
                      "JOIN file_locations l ON f.file_uuid = l.file_uuid "
                      "WHERE f.file_size > ? "
                      "ORDER BY f.file_size DESC LIMIT 200";
    sqlite3_stmt *stmt;
    
    if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) {
        sqlite3_bind_int64(stmt, 1, 1048576);  // Files > 1MB
        
        int cached = 0;
        while (sqlite3_step(stmt) == SQLITE_ROW) {
            const char *file_uuid = (const char*)sqlite3_column_text(stmt, 0);
            const char *location = (const char*)sqlite3_column_text(stmt, 1);
            long file_size = sqlite3_column_int64(stmt, 2);
            
            cache_insert(file_uuid, location, file_size);
            cached++;
        }
        sqlite3_finalize(stmt);
        printf("Pre-cached %d large files (>1MB)\n", cached);
    }
    pthread_mutex_unlock(&db_mutex);
    
    return NULL;
}

static void mb_destroy(void *userdata) {
    (void) userdata;
    
    // Unmap all mapped files
    pthread_mutex_lock(&cache_mutex);
    for (int i = 0; i < cache_count; i++) {
        if (file_cache[i].mmap_ptr) {
            munmap(file_cache[i].mmap_ptr, file_cache[i].file_size);
            close(file_cache[i].mmap_fd);
        }
    }
    pthread_mutex_unlock(&cache_mutex);
    
    printf("Cache stats: %d files, %d nodes\n", cache_count, node_cache_count);
    
    if (db) {
        pthread_mutex_lock(&db_mutex);
        sqlite3_close(db);
        pthread_mutex_unlock(&db_mutex);
    }
}

static char* find_node_id(const char *path) {
    if (strcmp(path, "/") == 0) return NULL;
    
    char *path_copy = strdup(path);
    char *components[20];
    int depth = 0;
    
    char *token = strtok(path_copy + 1, "/");
    while (token && depth < 20) {
        components[depth++] = strdup(token);
        token = strtok(NULL, "/");
    }
    free(path_copy);
    
    if (depth == 0) return NULL;
    
    char *current_parent_id = NULL;
    
    pthread_mutex_lock(&db_mutex);
    
    for (int level = 0; level < depth; level++) {
        const char *sql;
        sqlite3_stmt *stmt;
        
        if (level == 0) {
            sql = "SELECT node_id, node_type, file_size, parent_id "
                  "FROM file_nodes WHERE label = ? AND (parent_id IS NULL OR parent_id = '')";
        } else {
            sql = "SELECT node_id, node_type, file_size, parent_id "
                  "FROM file_nodes WHERE label = ? AND parent_id = ?";
        }
        
        if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
            pthread_mutex_unlock(&db_mutex);
            for (int i = 0; i < depth; i++) free(components[i]);
            if (current_parent_id) free(current_parent_id);
            return NULL;
        }
        
        sqlite3_bind_text(stmt, 1, components[level], -1, SQLITE_STATIC);
        if (level > 0 && current_parent_id) {
            sqlite3_bind_text(stmt, 2, current_parent_id, -1, SQLITE_STATIC);
        }
        
        if (sqlite3_step(stmt) != SQLITE_ROW) {
            sqlite3_finalize(stmt);
            pthread_mutex_unlock(&db_mutex);
            for (int i = 0; i < depth; i++) free(components[i]);
            if (current_parent_id) free(current_parent_id);
            return NULL;
        }
        
        const char *found_node_id = (const char*)sqlite3_column_text(stmt, 0);
        const char *node_type = (const char*)sqlite3_column_text(stmt, 1);
        long file_size = sqlite3_column_int64(stmt, 2);
        const char *parent_id = (const char*)sqlite3_column_text(stmt, 3);
        
        node_cache_insert(found_node_id, node_type, file_size, parent_id);
        
        if (current_parent_id) free(current_parent_id);
        current_parent_id = strdup(found_node_id);
        
        sqlite3_finalize(stmt);
    }
    
    pthread_mutex_unlock(&db_mutex);
    
    for (int i = 0; i < depth; i++) free(components[i]);
    return current_parent_id;
}

static int mb_getattr(const char *path, struct stat *stbuf, struct fuse_file_info *fi) {
    (void) fi;
    memset(stbuf, 0, sizeof(struct stat));
    
    if (strcmp(path, "/") == 0) {
        stbuf->st_mode = S_IFDIR | 0755;
        stbuf->st_nlink = 2;
        return 0;
    }
    
    char *node_id = find_node_id(path);
    if (!node_id) return -ENOENT;
    
    NodeCacheEntry *cached_node = node_cache_lookup(node_id);
    if (cached_node) {
        if (strcmp(cached_node->node_type, "folder") == 0) {
            stbuf->st_mode = S_IFDIR | 0755;
            stbuf->st_nlink = 2;
        } else {
            stbuf->st_mode = S_IFREG | 0444;
            stbuf->st_nlink = 1;
            stbuf->st_size = cached_node->file_size;
        }
        free(node_id);
        return 0;
    }
    
    pthread_mutex_lock(&db_mutex);
    const char *sql = "SELECT node_type, file_size FROM file_nodes WHERE node_id = ?";
    sqlite3_stmt *stmt;
    
    int result = -ENOENT;
    if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) {
        sqlite3_bind_text(stmt, 1, node_id, -1, SQLITE_STATIC);
        if (sqlite3_step(stmt) == SQLITE_ROW) {
            const char *node_type = (const char*)sqlite3_column_text(stmt, 0);
            long file_size = sqlite3_column_int64(stmt, 1);
            
            if (strcmp(node_type, "folder") == 0) {
                stbuf->st_mode = S_IFDIR | 0755;
                stbuf->st_nlink = 2;
            } else {
                stbuf->st_mode = S_IFREG | 0444;
                stbuf->st_nlink = 1;
                stbuf->st_size = file_size;
            }
            result = 0;
        }
        sqlite3_finalize(stmt);
    }
    pthread_mutex_unlock(&db_mutex);
    
    free(node_id);
    return result;
}

static int mb_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
                     off_t offset, struct fuse_file_info *fi,
                     enum fuse_readdir_flags flags) {
    (void) offset;
    (void) fi;
    (void) flags;
    
    filler(buf, ".", NULL, 0, FUSE_FILL_DIR_DEFAULTS);
    filler(buf, "..", NULL, 0, FUSE_FILL_DIR_DEFAULTS);
    
    pthread_mutex_lock(&db_mutex);
    
    if (strcmp(path, "/") == 0) {
        const char *sql = "SELECT label FROM file_nodes WHERE parent_id IS NULL OR parent_id = ''";
        sqlite3_stmt *stmt;
        if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) {
            while (sqlite3_step(stmt) == SQLITE_ROW) {
                const char *label = (const char*)sqlite3_column_text(stmt, 0);
                if (label) filler(buf, label, NULL, 0, FUSE_FILL_DIR_DEFAULTS);
            }
            sqlite3_finalize(stmt);
        }
        pthread_mutex_unlock(&db_mutex);
        return 0;
    }
    
    char *parent_node_id = find_node_id(path);
    if (!parent_node_id) {
        pthread_mutex_unlock(&db_mutex);
        return -ENOENT;
    }
    
    const char *sql = "SELECT label FROM file_nodes WHERE parent_id = ?";
    sqlite3_stmt *stmt;
    if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) {
        sqlite3_bind_text(stmt, 1, parent_node_id, -1, SQLITE_STATIC);
        while (sqlite3_step(stmt) == SQLITE_ROW) {
            const char *label = (const char*)sqlite3_column_text(stmt, 0);
            if (label) filler(buf, label, NULL, 0, FUSE_FILL_DIR_DEFAULTS);
        }
        sqlite3_finalize(stmt);
    }
    
    pthread_mutex_unlock(&db_mutex);
    free(parent_node_id);
    return 0;
}

static int mb_open(const char *path, struct fuse_file_info *fi) {
    if ((fi->flags & O_ACCMODE) != O_RDONLY) return -EACCES;
    char *node_id = find_node_id(path);
    if (!node_id) return -ENOENT;
    free(node_id);
    return 0;
}

// Optimized read with mmap for large files
static int mb_read(const char *path, char *buf, size_t size, off_t offset,
                   struct fuse_file_info *fi) {
    (void) fi;
    
    char *node_id = find_node_id(path);
    if (!node_id) return -ENOENT;
    
    FileCacheEntry *cached = cache_lookup(node_id);
    
    // Use mmap for large files (>1MB)
    if (cached && cached->file_size > MMAP_THRESHOLD) {
        pthread_mutex_lock(&cache_mutex);
        
        // mmap if not already mapped
        if (!cached->mmap_ptr) {
            int fd = open(cached->file_path, O_RDONLY);
            if (fd >= 0) {
                void *ptr = mmap(NULL, cached->file_size, PROT_READ, MAP_PRIVATE, fd, 0);
                if (ptr != MAP_FAILED) {
                    cached->mmap_ptr = ptr;
                    cached->mmap_fd = fd;
                } else {
                    close(fd);
                }
            }
        }
        
        if (cached->mmap_ptr) {
            // Read from mmap
            if (offset >= cached->file_size) {
                pthread_mutex_unlock(&cache_mutex);
                free(node_id);
                return 0;
            }
            
            size_t bytes_to_read = size;
            if (offset + size > cached->file_size) {
                bytes_to_read = cached->file_size - offset;
            }
            
            memcpy(buf, cached->mmap_ptr + offset, bytes_to_read);
            
            pthread_mutex_unlock(&cache_mutex);
            free(node_id);
            return bytes_to_read;
        }
        
        pthread_mutex_unlock(&cache_mutex);
    }
    
    // Use fread for small files or if mmap failed
    if (cached && strcmp(cached->file_path, "") != 0) {
        FILE *fp = fopen(cached->file_path, "rb");
        if (fp) {
            if (fseek(fp, offset, SEEK_SET) == 0) {
                // Read in large chunks (64KB)
                size_t total_read = 0;
                while (total_read < size) {
                    size_t chunk_size = (size - total_read > READ_BUFFER_SIZE) ? 
                                         READ_BUFFER_SIZE : size - total_read;
                    size_t bytes_read = fread(buf + total_read, 1, chunk_size, fp);
                    total_read += bytes_read;
                    if (bytes_read < chunk_size) break;  // EOF or error
                }
                fclose(fp);
                free(node_id);
                return total_read;
            }
            fclose(fp);
        }
    }
    
    // Query from database if not cached
    pthread_mutex_lock(&db_mutex);
    const char *sql = "SELECT location FROM file_locations WHERE file_uuid = ? LIMIT 1";
    sqlite3_stmt *stmt;
    
    if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
        pthread_mutex_unlock(&db_mutex);
        free(node_id);
        return -EIO;
    }
    
    sqlite3_bind_text(stmt, 1, node_id, -1, SQLITE_STATIC);
    
    if (sqlite3_step(stmt) != SQLITE_ROW) {
        sqlite3_finalize(stmt);
        pthread_mutex_unlock(&db_mutex);
        free(node_id);
        return -ENOENT;
    }
    
    const char *file_path = (const char*)sqlite3_column_text(stmt, 0);
    char *path_copy = strdup(file_path);
    
    sqlite3_finalize(stmt);
    pthread_mutex_unlock(&db_mutex);
    
    NodeCacheEntry *node_info = node_cache_lookup(node_id);
    if (node_info) {
        cache_insert(node_id, path_copy, node_info->file_size);
    }
    
    free(node_id);
    
    FILE *fp = fopen(path_copy, "rb");
    if (!fp) {
        free(path_copy);
        return -ENOENT;
    }
    
    if (fseek(fp, offset, SEEK_SET) != 0) {
        fclose(fp);
        free(path_copy);
        return -EIO;
    }
    
    size_t bytes_read = fread(buf, 1, size, fp);
    fclose(fp);
    free(path_copy);
    
    return bytes_read;
}

static const struct fuse_operations mb_oper = {
    .init    = mb_init,
    .destroy = mb_destroy,
    .getattr = mb_getattr,
    .readdir = mb_readdir,
    .open    = mb_open,
    .read    = mb_read,
};

int main(int argc, char *argv[]) {
    printf("MarkBase FUSE v11.0 - Throughput Optimized\n");
    printf("==========================================\n");
    printf("Optimizations:\n");
    printf("  - mmap for large files (>1MB)\n");
    printf("  - Large read buffer (64KB)\n");
    printf("  - Pre-cache 200 large files\n");
    printf("  - Kernel caching enabled\n");
    printf("  - Thread-safe SQLite\n");
    printf("\n");
    
    return fuse_main(argc, argv, &mb_oper, NULL);
}