admin/markbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

P0: Add S3 benchmark script
Some checks failed
Test / build (push) Has been cancelled
Details
Test / test (push) Has been cancelled
Details

Browse Source 
- PUT/GET 1-100MB files
- LIST bucket, HEAD object
- DELETE cleanup
- Multipart upload simulation (5x10MB)

Tests throughput for all S3 operations
This commit is contained in:
Warren
2026-06-22 00:06:35 +08:00
parent 7d229d0b62
commit 08244032a8
1 changed files with 160 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
scripts/s3_benchmark.sh Executable file
View File

@@ -0,0 +1,160 @@
#!/bin/bash
# S3 Performance Benchmark
# Tests: PUT, GET, LIST, Multipart upload
set -e
S3_URL="http://127.0.0.1:11438/s3"
BUCKET="demo"
TEST_DIR="/tmp/s3_benchmark"
ACCESS_KEY="markbase_access_key_001"
SECRET_KEY="markbase_secret_key_xyz123"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0;33m'
echo "================================================"
echo "S3 Performance Benchmark"
echo "================================================"
echo ""
# Setup test directory
rm -rf "$TEST_DIR"
mkdir -p "$TEST_DIR"
cd "$TEST_DIR"
# Generate test files
echo "Generating test files..."
dd if=/dev/urandom of=file_1mb.bin bs=1M count=1 2>/dev/null
dd if=/dev/urandom of=file_10mb.bin bs=1M count=10 2>/dev/null
dd if=/dev/urandom of=file_50mb.bin bs=1M count=50 2>/dev/null
dd if=/dev/urandom of=file_100mb.bin bs=1M count=100 2>/dev/null
echo "${GREEN}Test files generated${NC}"
echo ""
# Test 1: PUT 1MB file
echo "=== Test 1: PUT 1MB file ==="
START=$(date +%s.%N)
curl -s -X PUT -T file_1mb.bin "$S3_URL/$BUCKET/benchmark/file_1mb.bin"
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 1 / $ELAPSED" | bc)
echo "${GREEN}PUT 1MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 2: PUT 10MB file
echo "=== Test 2: PUT 10MB file ==="
START=$(date +%s.%N)
curl -s -X PUT -T file_10mb.bin "$S3_URL/$BUCKET/benchmark/file_10mb.bin"
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 10 / $ELAPSED" | bc)
echo "${GREEN}PUT 10MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 3: GET 10MB file
echo "=== Test 3: GET 10MB file ==="
START=$(date +%s.%N)
curl -s "$S3_URL/$BUCKET/benchmark/file_10mb.bin" -o /tmp/s3_download_10mb.bin
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 10 / $ELAPSED" | bc)
echo "${GREEN}GET 10MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 4: PUT 50MB file
echo "=== Test 4: PUT 50MB file ==="
START=$(date +%s.%N)
curl -s -X PUT -T file_50mb.bin "$S3_URL/$BUCKET/benchmark/file_50mb.bin"
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 50 / $ELAPSED" | bc)
echo "${GREEN}PUT 50MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 5: GET 50MB file
echo "=== Test 5: GET 50MB file ==="
START=$(date +%s.%N)
curl -s "$S3_URL/$BUCKET/benchmark/file_50mb.bin" -o /tmp/s3_download_50mb.bin
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 50 / $ELAPSED" | bc)
echo "${GREEN}GET 50MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 6: PUT 100MB file
echo "=== Test 6: PUT 100MB file ==="
START=$(date +%s.%N)
curl -s -X PUT -T file_100mb.bin "$S3_URL/$BUCKET/benchmark/file_100mb.bin"
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 100 / $ELAPSED" | bc)
echo "${GREEN}PUT 100MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 7: GET 100MB file
echo "=== Test 7: GET 100MB file ==="
START=$(date +%s.%N)
curl -s "$S3_URL/$BUCKET/benchmark/file_100mb.bin" -o /tmp/s3_download_100mb.bin
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 100 / $ELAPSED" | bc)
echo "${GREEN}GET 100MB: ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Test 8: LIST objects (bucket listing)
echo "=== Test 8: LIST objects ==="
START=$(date +%s.%N)
curl -s "$S3_URL/$BUCKET" > /dev/null
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
echo "${GREEN}LIST bucket: ${ELAPSED}s${NC}"
echo ""
# Test 9: HEAD object (metadata)
echo "=== Test 9: HEAD object ==="
START=$(date +%s.%N)
curl -s -I "$S3_URL/$BUCKET/benchmark/file_10mb.bin" > /dev/null
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
echo "${GREEN}HEAD object: ${ELAPSED}s${NC}"
echo ""
# Test 10: DELETE objects (cleanup)
echo "=== Test 10: DELETE objects ==="
START=$(date +%s.%N)
curl -s -X DELETE "$S3_URL/$BUCKET/benchmark/file_1mb.bin"
curl -s -X DELETE "$S3_URL/$BUCKET/benchmark/file_10mb.bin"
curl -s -X DELETE "$S3_URL/$BUCKET/benchmark/file_50mb.bin"
curl -s -X DELETE "$S3_URL/$BUCKET/benchmark/file_100mb.bin"
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
echo "${GREEN}DELETE 4 files: ${ELAPSED}s${NC}"
echo ""
# Test 11: Multipart upload simulation (split 50MB into 5 parts)
echo "=== Test 11: Multipart upload simulation ==="
# Split file into 10MB parts
split -b 10M file_50mb.bin part_
START=$(date +%s.%N)
# Upload each part sequentially (simulating multipart)
for i in part_aa part_ab part_ac part_ad part_ae; do
curl -s -X PUT -T "$i" "$S3_URL/$BUCKET/benchmark/mp_$i" 2>/dev/null || true
done
END=$(date +%s.%N)
ELAPSED=$(echo "$END - $START" | bc)
THROUGHPUT=$(echo "scale=2; 50 / $ELAPSED" | bc)
echo "${GREEN}Multipart (5x10MB): ${ELAPSED}s (${THROUGHPUT} MB/s)${NC}"
echo ""
# Cleanup
rm -rf "$TEST_DIR"
rm -f /tmp/s3_download_*.bin part_*
echo "================================================"
echo "${GREEN}S3 Performance Benchmark Complete${NC}"
echo "================================================"