From 6847b8f36467455e783139778e6cf9ecf1838269 Mon Sep 17 00:00:00 2001 From: Mathew Date: Wed, 13 May 2026 18:19:52 +1000 Subject: [PATCH 1/9] add --version & tweak cow --- src/core/Makefile | 7 +++- src/core/config.h | 6 ++- src/core/db.cpp | 21 +++++----- src/core/settings.cpp | 89 ++++++++++++++++++++++++++----------------- src/server/Makefile | 7 +++- 5 files changed, 82 insertions(+), 48 deletions(-) diff --git a/src/core/Makefile b/src/core/Makefile index 00ce1c7..ded4a7d 100644 --- a/src/core/Makefile +++ b/src/core/Makefile @@ -6,6 +6,11 @@ CC = g++ _CFLAGS = -g -c -Wall -fPIC -c $(CFLAGS) LFLAGS = -lpthread -shared +GIT_DESCRIBE := $(shell git describe --tags --always --dirty 2>/dev/null || echo 0.0.0) +GIT_REV := $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown) + +_CFLAGS += -DSCACHE_VERSION=\"$(GIT_DESCRIBE)\" -DSCACHE_REVISION=\"$(GIT_REV)\" + ifeq ($(CONFIG), Debug) _CFLAGS += -DDEBUG_BUILD -O0 else @@ -53,4 +58,4 @@ timer.o: timer.cpp clean: - rm -f $(OBJS) $(OUT) \ No newline at end of file + rm -f $(OBJS) $(OUT) diff --git a/src/core/config.h b/src/core/config.h index 71bbb7f..18f5241 100644 --- a/src/core/config.h +++ b/src/core/config.h @@ -19,6 +19,10 @@ #define DB_ENABLE_COPY_ON_WRITE true #endif +#ifndef SCACHE_BUILD_DATE +#define SCACHE_BUILD_DATE __DATE__ " " __TIME__ +#endif + #define DEFAULT_LISTING_LIMIT 10000 #define HASH_SEED 13 @@ -33,4 +37,4 @@ #ifndef SCACHE_REVISION #define SCACHE_REVISION "" -#endif \ No newline at end of file +#endif diff --git a/src/core/db.cpp b/src/core/db.cpp index 3a01b23..a227e82 100644 --- a/src/core/db.cpp +++ b/src/core/db.cpp @@ -1393,7 +1393,17 @@ static pid_t db_index_flush(bool copyOnWrite){ cache_entry* ce; block_free_node *free_node; int temp; - + + //If we are forking we can do so now + if(copyOnWrite){ + pid = fork(); + if(pid != 0) return pid; // includes -1 + signal_handler_remove(); + } + + // NOTE: This work is intentionally after the fork so the parent process does not + // run the potentially expensive blockfile copy when copy-on-write flush is enabled. + // buffer contains the target temp file (${blockfile}.temp) snprintf(buffer, sizeof(buffer), "%s.temp", db.path_blockfile); @@ -1403,16 +1413,9 @@ static pid_t db_index_flush(bool copyOnWrite){ // ensure all data is on disk fdatasync(db.fd_blockfile); - // create a hard link from the currentl block file to ${blockfile}.temp + // create a hard link / copy from the current block file to ${blockfile}.temp force_link(db.path_blockfile, buffer); - //If we are forking we can do so now - if(copyOnWrite){ - pid = fork(); - if(pid != 0) return pid; // includes -1 - signal_handler_remove(); - } - // Open temporary index file snprintf(buffer, sizeof(buffer), "%s/index.temp", db.path_root); int fd = open(buffer, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, S_IRUSR | S_IWUSR); diff --git a/src/core/settings.cpp b/src/core/settings.cpp index 11263ed..5bba498 100644 --- a/src/core/settings.cpp +++ b/src/core/settings.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "config.h" #include "debug.h" #include "settings.h" @@ -41,11 +42,23 @@ static void print_usage() { "\n" "General settings:\n" "\n" -" -m file --make-pid file - output a PID file (default: no)\n" -" -d - daemonize (default: no)\n" -" -o - redirect output to /dev/null if daemonized (default: yes)\n" -"\n" -"Problems? You can reach the author at .\n"); + " -m file --make-pid file - output a PID file (default: no)\n" + " -d - daemonize (default: no)\n" + " -o - redirect output to /dev/null if daemonized (default: yes)\n" + " --version - print version info and exit\n" + "\n" + "Problems? You can reach the author at .\n"); +} + +static void print_version_and_exit() { + const char* build_type = +#ifdef DEBUG_BUILD + "debug"; +#else + "release"; +#endif + printf("scache %s (%s) built %s [%s]\n", SCACHE_VERSION, SCACHE_REVISION, SCACHE_BUILD_DATE, build_type); + exit(0); } char* string_allocate(const char* s) @@ -155,28 +168,29 @@ static void parse_binds(const char* optarg_const, scache_binds* target) free(optarg); } -void settings_parse_arguments(int argc, char** argv) { - static struct option long_options[] = - { - /* These options set a flag. */ - /* These options set a value */ - { "make-pid", required_argument, 0, 'm' }, - { "leave-pid", no_argument, 0, 'M' }, - { "database-max-size", required_argument, 0, 's' }, - { "database-file-path", required_argument, 0, 'r' }, - { "database-lru-clear", required_argument, 0, 'l' }, - { "bind", required_argument, 0, 'b' }, - { "monitor", required_argument, 0, 'B' }, - { 0, 0, 0, 0 } - }; + void settings_parse_arguments(int argc, char** argv) { + static struct option long_options[] = + { + /* These options set a flag. */ + /* These options set a value */ + { "make-pid", required_argument, 0, 'm' }, + { "leave-pid", no_argument, 0, 'M' }, + { "database-max-size", required_argument, 0, 's' }, + { "database-file-path", required_argument, 0, 'r' }, + { "database-lru-clear", required_argument, 0, 'l' }, + { "bind", required_argument, 0, 'b' }, + { "monitor", required_argument, 0, 'B' }, + { "version", no_argument, 0, 'v' }, + { 0, 0, 0, 0 } + }; - int r = 0, option_index = 0; - while ((r = getopt_long(argc, argv, "dom:s:r:l:b:B:", long_options, &option_index)) != -1) { - switch (r) { - case 0: - if (long_options[option_index].flag != 0) - break; - printf("option %s", long_options[option_index].name); + int r = 0, option_index = 0; + while ((r = getopt_long(argc, argv, "dom:s:r:l:b:B:v", long_options, &option_index)) != -1) { + switch (r) { + case 0: + if (long_options[option_index].flag != 0) + break; + printf("option %s", long_options[option_index].name); if (optarg) printf(" with arg %s", optarg); printf("\n"); @@ -205,15 +219,18 @@ void settings_parse_arguments(int argc, char** argv) { case 'B': parse_binds(optarg, &settings.bind_monitor); break; - case 'l': - settings.db_lru_clear = atof(optarg)/100; - break; - default: - case '?': - print_usage(); - exit(EXIT_FAILURE); - } - } + case 'l': + settings.db_lru_clear = atof(optarg)/100; + break; + case 'v': + print_version_and_exit(); + break; + default: + case '?': + print_usage(); + exit(EXIT_FAILURE); + } + } if (settings.db_file_path == NULL) { settings.db_file_path = strdup("/var/lib/scache/"); @@ -225,4 +242,4 @@ void settings_cleanup() { free(settings.db_file_path); settings.db_file_path = NULL; } -} \ No newline at end of file +} diff --git a/src/server/Makefile b/src/server/Makefile index 45f3ebf..f83ab2c 100644 --- a/src/server/Makefile +++ b/src/server/Makefile @@ -6,6 +6,11 @@ CC = g++ _CFLAGS = -g -c -Wall -I../core -fPIC $(CFLAGS) LFLAGS += -lpthread +GIT_DESCRIBE := $(shell git describe --tags --always --dirty 2>/dev/null || echo 0.0.0) +GIT_REV := $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown) + +_CFLAGS += -DSCACHE_VERSION=\"$(GIT_DESCRIBE)\" -DSCACHE_REVISION=\"$(GIT_REV)\" + ifeq ($(CONFIG),Debug) _CFLAGS += -DDEBUG_BUILD -O0 else @@ -20,4 +25,4 @@ scache.o: ../core/libscache.a scache.cpp clean: - rm -f $(OBJS) $(OUT) \ No newline at end of file + rm -f $(OBJS) $(OUT) From ee07d9ab80b87aa015822464f66a889a5ad73764 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sat, 16 May 2026 17:00:04 +0000 Subject: [PATCH 2/9] reduce flushing --- src/core/config.h | 6 ++++++ src/core/db.cpp | 30 +++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/core/config.h b/src/core/config.h index 18f5241..c6f5961 100644 --- a/src/core/config.h +++ b/src/core/config.h @@ -19,6 +19,12 @@ #define DB_ENABLE_COPY_ON_WRITE true #endif +#ifndef DB_FLUSH_MIN_INTERVAL_MS +// Minimum interval between index flushes. This reduces flush storms under cache churn. +// A value of 0 disables rate limiting. +#define DB_FLUSH_MIN_INTERVAL_MS 1000 +#endif + #ifndef SCACHE_BUILD_DATE #define SCACHE_BUILD_DATE __DATE__ " " __TIME__ #endif diff --git a/src/core/db.cpp b/src/core/db.cpp index a227e82..3a22d5e 100644 --- a/src/core/db.cpp +++ b/src/core/db.cpp @@ -31,6 +31,7 @@ LRU #include #include #include +#include #include "db.h" #include "debug.h" #include "hash.h" @@ -66,9 +67,16 @@ struct db_details db { }; pid_t current_flush = 0; +static uint64_t last_flush_ms = 0; +static bool flush_pending = false; static pid_t db_index_flush(bool copyOnWrite = true); +static inline uint64_t db_now_ms() { + // current_time is maintained by the timer subsystem (updated frequently). + return ((uint64_t)current_time.tv_sec * 1000ULL) + ((uint64_t)current_time.tv_usec / 1000ULL); +} + //Buffers static char filename_buffer[MAX_PATH]; static uint16_t nextit = 0; @@ -315,7 +323,13 @@ void db_lru_cleanup_percent(int* bytes_to_remove) { cache_entry* l = db.lru_head; //Skip if currently deleting - if(l->writing || l->deleted) continue; + if(l->writing || l->deleted) { + // Hardening: this should not happen (writing entries should not be in LRU; + // deleted entries should have been removed). If it does, ensure forward + // progress and avoid a tight infinite loop. + db_lru_remove_node(l); + continue; + } *bytes_to_remove -= l->data_length; @@ -470,6 +484,19 @@ void db_lru_gc() { // Check for running flush if(currently_flushing(WNOHANG)) return; + // Debounce/rate-limit flushes to avoid flush storms under heavy churn. + // If rate limited, mark as pending so the next db_lru_gc call after the interval can flush. + uint64_t now_ms = db_now_ms(); + if (flush_pending) { + // fall through; we want to flush if interval has elapsed + } + if (DB_FLUSH_MIN_INTERVAL_MS > 0 && last_flush_ms != 0 && + (now_ms - last_flush_ms) < (uint64_t)DB_FLUSH_MIN_INTERVAL_MS) { + flush_pending = true; + return; + } + flush_pending = false; + // Flush index pid_t pid = db_index_flush(DB_ENABLE_COPY_ON_WRITE); if(pid == -1){ @@ -477,6 +504,7 @@ void db_lru_gc() { return; } current_flush = pid; + last_flush_ms = now_ms; } static void db_clear_directory(const char* directory) { From 861a1cf3804864d136f3fec893e6fac5d40bffb6 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sat, 16 May 2026 17:00:55 +0000 Subject: [PATCH 3/9] implementation plan (future) --- IMPLEMENTATION_PLAN.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 IMPLEMENTATION_PLAN.md diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..f7ff384 --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,41 @@ +# Cache Thrash Improvements (Plan) + +This repo currently does a full index flush (and a blockfile snapshot) from the write path via `db_lru_gc()`. Under high churn this can devolve into near-continuous flush attempts. The changes implemented in this PR address flush storms and a potential infinite loop in LRU cleanup. The remaining items below are larger design changes. + +## 1. Remove Full Blockfile Copy During Flush + +Problem: +Each flush currently writes `index.save` and also updates `blockfile.db.save`. Today the "snapshot" of the blockfile is performed via `force_link()` which shells out to `cp`, which is O(size of blockfile) I/O every flush. This scales poorly and will dominate performance as the cache grows. + +Direction: +Make `blockfile.db` the durable canonical store and make `index.save` the only atomic metadata file. On startup: +1. Load `index.save`. +2. Reconstruct `free_blocks` by scanning all loaded entries and marking blocks in-use, then pushing the remaining blocks into the free list. + +Notes: +This eliminates `blockfile.db.save` and avoids copying the blockfile during steady-state operation. + +## 3. Move Expiration Work Off the Write Path + +Problem: +`db_lru_gc()` calls `db_expire_cursor()` which can scan thousands of entries per invocation. When `db_lru_gc()` is triggered from inserts, this couples cache maintenance cost directly to write throughput and amplifies latency under load. + +Direction: +1. Split `db_lru_gc()` into: + - a cheap write-path step: eviction only when over limit + - a periodic maintenance step: expiration cursor work +2. Trigger maintenance periodically (timer-driven or a dedicated background loop) rather than per-insert. + +## 4. Add Hysteresis to Eviction Thresholds + +Problem: +When `db.db_size_bytes` hovers around `settings.max_size`, the system can oscillate between inserting and evicting, repeatedly triggering maintenance and flushes. + +Direction: +Introduce two thresholds: +1. High watermark: start eviction when `db_size_bytes > max_size`. +2. Low watermark: evict down to `max_size * (1 - clear_pct)` (or another tuned value). + +Optional enhancement: +Do not start a new eviction pass until the high watermark is exceeded again. This reduces "thrash" oscillations and improves steady-state throughput. + From d7e97556ffed8b6050ee983f76f5cbee4791a865 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sat, 16 May 2026 17:20:54 +0000 Subject: [PATCH 4/9] new blockfile system --- src/core/db.cpp | 166 +++++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 80 deletions(-) diff --git a/src/core/db.cpp b/src/core/db.cpp index 3a22d5e..acb5bc2 100644 --- a/src/core/db.cpp +++ b/src/core/db.cpp @@ -348,19 +348,8 @@ void db_lru_cleanup_percent(int* bytes_to_remove) { DEBUG("[#] LRU attempted to remove %d bytes, %d bytes remaining\n", debug_bytes, *bytes_to_remove); } -static void force_link(const char* fileThatExists, const char* fileThatDoesNotExist){ - char buffer[8096]; - //TODO: reflink - /*int ret = link(fileThatExists, fileThatDoesNotExist); - char buffer[8096]; - if(ret != 0){ - PWARN("Unable to hard link");*/ - //Really bad! Temporary. - snprintf(buffer, sizeof(buffer), "cp %s %s", fileThatExists, fileThatDoesNotExist); - printf("Executing %s\n", buffer); - system(buffer); - //} -} +// force_link() has been removed; blockfile.db is now the durable canonical store +// and no blockfile copy is performed during flush. static int db_expire_cursor_table(db_table* table) { int ret = 0; @@ -625,11 +614,12 @@ static bool db_load_from_save(){ char *bp = NULL; bool ret = false; size_t len = 0; - uint32_t u1, u2, u3, u4; + uint32_t u2, u3, u4; // u1 no longer used (old 'f:' format ignored) db_table* table = NULL; cache_entry* entry; int d1; ssize_t read; + uint8_t* block_in_use = NULL; snprintf(buffer, sizeof(buffer), "%s/index.save", db.path_root); int fd = open(buffer, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR); @@ -637,15 +627,13 @@ static bool db_load_from_save(){ return false; //it's ok } - snprintf(buffer, sizeof(buffer), "%s/blockfile.db.save", db.path_root); - db.fd_blockfile = open(buffer, O_RDWR | O_LARGEFILE , S_IRUSR | S_IWUSR); - if(db.fd_blockfile == -1){ - PWARN("Unable to open saved block file"); + // db.fd_blockfile and db.blocks_exist must already be set by db_open() + // before calling this function. We verify the blockfile is open. + if(db.fd_blockfile < 0){ + PWARN("Blockfile not open; cannot load from save"); close_fd(fd, "file descriptor"); return ret; } - off64_t size = lseek64(db.fd_blockfile, 0L, SEEK_END); - db.blocks_exist = (uint32_t)(size / BLOCK_LENGTH); FILE* fp = fdopen(fd, "r"); if(fp == NULL){ @@ -653,16 +641,23 @@ static bool db_load_from_save(){ goto close_fd2; } + // Bitset to track which blocks are in-use by loaded entries. + // Allocated based on current db.blocks_exist. + if (db.blocks_exist > 0) { + block_in_use = (uint8_t*)calloc((db.blocks_exist + 7) / 8, 1); + if (block_in_use == NULL) { + PWARN("Failed to allocate block_in_use bitset"); + goto close_fd2; // fp not open yet, use close_fd2 to close fd + } + } + while ((read = getline(&bp, &len, fp)) != -1) { if(read <= 2 || bp[1] != ':') continue; switch(bp[0]){ case 'f': - if(sscanf(bp, "f:%u", &u1) != 1){ - WARN("Free block parsing error\n"); - continue; - } - db_block_free(u1); - break; + // Old format: free block list. Ignore; we reconstruct free_blocks below. + // Optionally parse and discard to avoid warning. + break; case 't': if(sscanf(bp, "t:%s", buffer2) != 1){ WARN("Table parsing error\n"); @@ -705,6 +700,10 @@ static bool db_load_from_save(){ free(entry); continue; } + // Mark this block as in-use + if (block_in_use != NULL) { + block_in_use[d1 / 8] |= (1 << (d1 % 8)); + } } db_load_from_save_insert(table, entry); @@ -717,6 +716,16 @@ static bool db_load_from_save(){ } + // Reconstruct free_blocks: any block not marked in-use is free + if (block_in_use != NULL) { + for (uint32_t i = 0; i < db.blocks_exist; i++) { + if ((block_in_use[i / 8] & (1 << (i % 8))) == 0) { + db_block_free(i); + } + } + free(block_in_use); + } + if(bp != NULL){ free(bp); } @@ -726,21 +735,36 @@ static bool db_load_from_save(){ table = NULL; } - // move over block file (via link to preserve save) - snprintf(buffer, sizeof(buffer), "%s/blockfile.db.save", db.path_root); - unlink(db.path_blockfile); - force_link(buffer, db.path_blockfile); - ret = true; -close_fd: - fclose(fp); -close_fd2: - if(fp == NULL) close_fd(fd, "file desriptor (cache file)"); - fd = db.fd_blockfile; - db.fd_blockfile = -1; - close_fd(fd, "file descriptor (blockfile)"); + // Reconstruct free_blocks: any block not marked in-use is free + if (block_in_use != NULL) { + for (uint32_t i = 0; i < db.blocks_exist; i++) { + if ((block_in_use[i / 8] & (1 << (i % 8))) == 0) { + db_block_free(i); + } + } + free(block_in_use); + } + + if(bp != NULL){ + free(bp); + } + + if(table != NULL){ + db_table_deref(table, true); + table = NULL; + } + if (fp != NULL) { + fclose(fp); + } +close_fd2: + if(fp == NULL && fd >= 0) { + close_fd(fd, "file descriptor (index)"); + } + + // Note: db.fd_blockfile is NOT closed here; it remains open as the live blockfile. return ret; } @@ -759,15 +783,16 @@ bool db_open(const char* path) { db.tables = kh_init(table); db.table_gc = kh_begin(db.tables); - //Load from index if available + // Set blockfile path and open it BEFORE loading from save. + // db_load_from_save() expects db.fd_blockfile and db.blocks_exist to be valid. snprintf(db.path_blockfile, SHORT_PATH, "%s/blockfile.db", path); - - if(!db_load_from_save()){ - PWARN("Unable to load index from disk, will blank database"); - will_black = true; - } - //Block file + // Delete any stale blockfile.db.save from the old format + // (best-effort; ignore errors if it doesn't exist) + char save_path[MAX_PATH]; + snprintf(save_path, sizeof(save_path), "%s/blockfile.db.save", path); + unlink(save_path); + db.fd_blockfile = open(db.path_blockfile, O_CREAT | O_RDWR | O_LARGEFILE , S_IRUSR | S_IWUSR); if (db.fd_blockfile < 0) { PFATAL("Failed to open blockfile: %s", db.path_blockfile); @@ -785,6 +810,12 @@ bool db_open(const char* path) { db.blocks_exist = (uint32_t)(size / BLOCK_LENGTH); lseek64(db.fd_blockfile, 0L, SEEK_SET); + //Load from index if available (expects db.fd_blockfile and db.blocks_exist to be set) + if(!db_load_from_save()){ + PWARN("Unable to load index from disk, will blank database"); + will_black = true; + } + // Mark all blocks that already exist in the block file as non-allocated if(will_black){ if(size > (BLOCK_MAX_LOAD * BLOCK_LENGTH)) { @@ -795,11 +826,13 @@ bool db_open(const char* path) { db.blocks_exist = (uint32_t)(size / BLOCK_LENGTH); } if((size % BLOCK_LENGTH) != 0){ - WARN("Block file was strange size of %d", size); - - if(ftruncate(db.fd_blockfile, (size / BLOCK_LENGTH) * BLOCK_LENGTH) == -1){ + WARN("Block file was strange size of %d", (int)size); + + off64_t truncated_size = (size / BLOCK_LENGTH) * BLOCK_LENGTH; + if(ftruncate(db.fd_blockfile, truncated_size) == -1){ PFATAL("Failed to truncate blockfile %s", db.path_blockfile); } + size = truncated_size; } for (off64_t i = 0; i < size; i += BLOCK_LENGTH) { db_block_free((uint32_t)(i / BLOCK_LENGTH)); @@ -1416,10 +1449,9 @@ static bool full_write(int fd, const char* buffer, int buffer_length){ static pid_t db_index_flush(bool copyOnWrite){ pid_t pid = 0; - char buffer[2048], buffer2[2048], buffer3[2048], buffer4[2048]; + char buffer[2048], buffer2[2048]; db_table* table; cache_entry* ce; - block_free_node *free_node; int temp; //If we are forking we can do so now @@ -1429,21 +1461,12 @@ static pid_t db_index_flush(bool copyOnWrite){ signal_handler_remove(); } - // NOTE: This work is intentionally after the fork so the parent process does not - // run the potentially expensive blockfile copy when copy-on-write flush is enabled. - - // buffer contains the target temp file (${blockfile}.temp) - snprintf(buffer, sizeof(buffer), "%s.temp", db.path_blockfile); - - // remove any other ${blockfile}.temp files (i.e an interrupted operation) - unlink(buffer); + // NOTE: The blockfile is now the durable canonical store. + // We only write index.save here; no blockfile copy is performed. // ensure all data is on disk fdatasync(db.fd_blockfile); - // create a hard link / copy from the current block file to ${blockfile}.temp - force_link(db.path_blockfile, buffer); - // Open temporary index file snprintf(buffer, sizeof(buffer), "%s/index.temp", db.path_root); int fd = open(buffer, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, S_IRUSR | S_IWUSR); @@ -1452,15 +1475,6 @@ static pid_t db_index_flush(bool copyOnWrite){ goto close; } - // Write free blocks - free_node = db.free_blocks; - while(free_node != NULL){ - temp = snprintf(buffer, sizeof(buffer), "f:%u\n", free_node->block_number); - assert(temp > 0); - if(!full_write(fd, buffer, temp)) goto close_fd; - free_node = free_node->next; - } - //Write tables and cache entries for (khiter_t ke = kh_begin(db.tables); ke < kh_end(db.tables); ++ke) { if (kh_exist(db.tables, ke)) { @@ -1492,22 +1506,14 @@ static pid_t db_index_flush(bool copyOnWrite){ close_fd(fd, "file descriptor (index)"); fd = -1; - // index.temp -> db.index + // index.temp -> index.save temp = snprintf(buffer, sizeof(buffer), "%s/index.temp", db.path_root); assert(temp > 0); temp = snprintf(buffer2, sizeof(buffer2), "%s/index.save", db.path_root); unlink(buffer2); - // blockfile.temp -> blockfile.save - temp = snprintf(buffer3, sizeof(buffer3), "%s.temp", db.path_blockfile); - assert(temp > 0); - temp = snprintf(buffer4, sizeof(buffer4), "%s.save", db.path_blockfile); - assert(temp > 0); - - // execute the renames of tmp files to actual + // execute the rename of tmp file to actual rename(buffer, buffer2); - rename(buffer3, buffer4); - unlink(buffer3); unlink(buffer); pid = 0; From d42511e299098bbc3ee86cf2c2e7f51e96312135 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sat, 16 May 2026 17:38:37 +0000 Subject: [PATCH 5/9] simple test runner --- run-test.sh | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 run-test.sh diff --git a/run-test.sh b/run-test.sh new file mode 100644 index 0000000..7195ba5 --- /dev/null +++ b/run-test.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# run-test.sh - Build and run tests for simple-cache +# Usage: ./run-test.sh [Release|Debug] +# Default: Release + +set -e # Exit on error + +# Parse configuration argument (case insensitive) +CONFIG="${1:-Release}" +CONFIG=$(echo "$CONFIG" | tr '[:lower:]' '[:upper:]') + +# Validate configuration +if [ "$CONFIG" != "RELEASE" ] && [ "$CONFIG" != "DEBUG" ]; then + echo "Error: Invalid configuration '$1'" + echo "Usage: $0 [Release|Debug]" + exit 1 +fi + +echo "==========================================" +echo "Building simple-cache in ${CONFIG} mode" +echo "==========================================" + +# Clean previous builds +echo "Cleaning previous builds..." +make clean + +# Build the project with the specified configuration +echo "Building project (CONFIG=${CONFIG})..." +make CONFIG=${CONFIG} + +# Build the tests +echo "Building tests (CONFIG=${CONFIG})..." +make CONFIG=${CONFIG} tests + +echo "" +echo "==========================================" +echo "Running tests..." +echo "==========================================" + +# Change to tests directory and run tests +cd tests +./tests ../src/server/scache ../testcases +TEST_RESULT=$? + +cd .. + +echo "" +echo "==========================================" +if [ $TEST_RESULT -eq 0 ]; then + echo "All tests passed successfully!" +else + echo "Tests failed with exit code: $TEST_RESULT" +fi +echo "==========================================" + +exit $TEST_RESULT From 5448d3cb91c60fee218bda5dab78a4e40a541676 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sat, 16 May 2026 18:01:01 +0000 Subject: [PATCH 6/9] db loading fixes and test --- src/core/db.cpp | 31 ++--- src/core/read_buffer.h | 5 + tests/Makefile | 3 +- tests/tests.cpp | 2 + tests/tests_db_load.cpp | 252 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 272 insertions(+), 21 deletions(-) create mode 100644 tests/tests_db_load.cpp diff --git a/src/core/db.cpp b/src/core/db.cpp index acb5bc2..b4ff7aa 100644 --- a/src/core/db.cpp +++ b/src/core/db.cpp @@ -737,25 +737,6 @@ static bool db_load_from_save(){ ret = true; - // Reconstruct free_blocks: any block not marked in-use is free - if (block_in_use != NULL) { - for (uint32_t i = 0; i < db.blocks_exist; i++) { - if ((block_in_use[i / 8] & (1 << (i % 8))) == 0) { - db_block_free(i); - } - } - free(block_in_use); - } - - if(bp != NULL){ - free(bp); - } - - if(table != NULL){ - db_table_deref(table, true); - table = NULL; - } - if (fp != NULL) { fclose(fp); } @@ -783,6 +764,18 @@ bool db_open(const char* path) { db.tables = kh_init(table); db.table_gc = kh_begin(db.tables); + // Reset block tracking state (in case db_open is called multiple times) + db.free_blocks = NULL; + db.blocks_free = 0; + db.lru_head = NULL; + db.lru_tail = NULL; + db.db_size_bytes = 0; + db.db_keys = 0; + db.db_stats_inserts = 0; + db.db_stats_gets = 0; + db.db_stats_deletes = 0; + db.db_stats_operations = 0; + // Set blockfile path and open it BEFORE loading from save. // db_load_from_save() expects db.fd_blockfile and db.blocks_exist to be valid. snprintf(db.path_blockfile, SHORT_PATH, "%s/blockfile.db", path); diff --git a/src/core/read_buffer.h b/src/core/read_buffer.h index bf9e09c..f3e107c 100644 --- a/src/core/read_buffer.h +++ b/src/core/read_buffer.h @@ -1,3 +1,6 @@ +#ifndef READ_BUFFER_H +#define READ_BUFFER_H + #include #include @@ -148,3 +151,5 @@ Helper to Iterate over circular buffer ret = needs_more_read; \ } \ } while (0); + +#endif /* READ_BUFFER_H */ diff --git a/tests/Makefile b/tests/Makefile index 65b34ff..9d187ef 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -30,6 +30,5 @@ tests_rbuffer.o: tests_rbuffer.cpp tests_system_simple.o: tests_system_simple.cpp $(CC) $(_CFLAGS) tests_system_simple.cpp -std=c++11 - clean: - rm -f $(OBJS) $(OUT) \ No newline at end of file + rm -f $(OBJS) $(OUT) tests_db_load \ No newline at end of file diff --git a/tests/tests.cpp b/tests/tests.cpp index 6b4c929..3b774f0 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -2,6 +2,7 @@ #include "minunit.h" #include "tests_rbuffer.cpp" #include "tests_system_simple.cpp" +#include "tests_db_load.cpp" #include /* atoi */ int tests_run = 0; @@ -34,5 +35,6 @@ int main(int argc, char *argv[]) port = atoi(argv[3]); } TESTSET("system_simple", test_simple(argv[1], argv[2], port)); + TESTSET("db_load", test_db_load_all()); return final_result != 0; } \ No newline at end of file diff --git a/tests/tests_db_load.cpp b/tests/tests_db_load.cpp new file mode 100644 index 0000000..638145f --- /dev/null +++ b/tests/tests_db_load.cpp @@ -0,0 +1,252 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minunit.h" +#include "../src/core/db.h" +#include "../src/core/db_structures.h" +#include "../src/core/config.h" + +static char test_dir[MAX_PATH]; +static char index_save_path[MAX_PATH]; +static char blockfile_path[MAX_PATH]; + +static void setup_test_dir() { + static int counter = 0; + snprintf(test_dir, sizeof(test_dir), "/tmp/test_db_load_%d", counter++); + mkdir(test_dir, 0755); + snprintf(index_save_path, sizeof(index_save_path), "%s/index.save", test_dir); + snprintf(blockfile_path, sizeof(blockfile_path), "%s/blockfile.db", test_dir); +} + +static void cleanup_test_dir() { + // Remove test files + unlink(index_save_path); + unlink(blockfile_path); + + // Remove any temp files that might have been created + char temp_path[MAX_PATH]; + snprintf(temp_path, sizeof(temp_path), "%s/index.temp", test_dir); + unlink(temp_path); + + rmdir(test_dir); +} + +static bool write_file(const char* path, const char* content) { + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (fd < 0) return false; + if (write(fd, content, strlen(content)) < 0) { close(fd); return false; } + close(fd); + return true; +} + +static bool create_blockfile(int num_blocks) { + int fd = open(blockfile_path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (fd < 0) return false; + if (ftruncate(fd, (off64_t)num_blocks * BLOCK_LENGTH) < 0) { close(fd); return false; } + close(fd); + return true; +} + +static bool run_db_open() { + char path_no_slash[MAX_PATH]; + snprintf(path_no_slash, sizeof(path_no_slash), "%s", test_dir); + int len = (int)strlen(path_no_slash); + if (len > 0 && path_no_slash[len-1] == '/') { + path_no_slash[len-1] = '\0'; + } + return db_open(path_no_slash); +} + +/* GOOD CASES */ + +static const char * test_empty_dir() { + setup_test_dir(); + bool result = run_db_open(); + mu_assert("empty_dir: db_open should succeed", result == true); + db_details* details = db_get_details(); + mu_assert("empty_dir: tables empty", kh_size(details->tables) == 0); + mu_assert("empty_dir: db_keys 0", details->db_keys == 0); + mu_assert("empty_dir: db_size_bytes 0", details->db_size_bytes == 0); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_one_block_entry() { + setup_test_dir(); + mu_assert("one_block: create blockfile", create_blockfile(2)); + const char* index_content = "t:test_table\nb:0:100:0:0:key1\n"; + mu_assert("one_block: write index", write_file(index_save_path, index_content)); + mu_assert("one_block: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("one_block: 1 table", kh_size(details->tables) == 1); + mu_assert("one_block: 1 key", details->db_keys == 1); + mu_assert("one_block: 100 bytes", details->db_size_bytes == 100); + mu_assert("one_block: blocks_exist 2", details->blocks_exist == 2); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_multiple_entries_free_reconstruction() { + setup_test_dir(); + mu_assert("multi_entry: create blockfile", create_blockfile(5)); + const char* index_content = "t:table1\nb:0:50:0:0:key1\nb:2:75:0:0:key2\nb:4:100:0:0:key3\n"; + mu_assert("multi_entry: write index", write_file(index_save_path, index_content)); + mu_assert("multi_entry: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("multi_entry: 1 table", kh_size(details->tables) == 1); + mu_assert("multi_entry: 3 keys", details->db_keys == 3); + mu_assert("multi_entry: 225 bytes", details->db_size_bytes == 225); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_old_format_with_f_lines() { + setup_test_dir(); + mu_assert("old_fmt: create blockfile", create_blockfile(3)); + const char* index_content = "f:0\nf:1\nt:table1\nb:0:50:0:0:key1\nf:2\n"; + mu_assert("old_fmt: write index", write_file(index_save_path, index_content)); + mu_assert("old_fmt: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("old_fmt: 1 table", kh_size(details->tables) == 1); + mu_assert("old_fmt: 1 key", details->db_keys == 1); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_multiple_tables() { + setup_test_dir(); + mu_assert("multi_tbl: create blockfile", create_blockfile(10)); + const char* index_content = "t:table1\nb:0:100:0:0:key1\nb:1:200:0:0:key2\nt:table2\nb:2:150:0:0:key3\nb:3:300:0:0:key4\n"; + mu_assert("multi_tbl: write index", write_file(index_save_path, index_content)); + mu_assert("multi_tbl: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("multi_tbl: 2 tables", kh_size(details->tables) == 2); + mu_assert("multi_tbl: 4 keys", details->db_keys == 4); + mu_assert("multi_tbl: 750 bytes", details->db_size_bytes == 750); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_entry_with_expiry() { + setup_test_dir(); + mu_assert("expiry: create blockfile", create_blockfile(1)); + time_t future_time = time(NULL) + 3600; + char index_content[256]; + snprintf(index_content, sizeof(index_content), "t:table1\nb:0:100:%lu:0:key1\n", future_time); + mu_assert("expiry: write index", write_file(index_save_path, index_content)); + mu_assert("expiry: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("expiry: 1 table", kh_size(details->tables) == 1); + mu_assert("expiry: 1 key", details->db_keys == 1); + db_close(); + cleanup_test_dir(); + return 0; +} + +/* ERROR CASES */ + +static const char * test_malformed_line_no_colon() { + setup_test_dir(); + mu_assert("malformed: create blockfile", create_blockfile(1)); + const char* index_content = "t:table1\nTHIS_IS_NOT_VALID\nb:0:100:0:0:key1\n"; + mu_assert("malformed: write index", write_file(index_save_path, index_content)); + mu_assert("malformed: db_open should not crash", run_db_open() == true); + db_details* details = db_get_details(); + mu_assert("malformed: 1 table (skip bad line)", kh_size(details->tables) == 1); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_entry_nonexistent_block() { + setup_test_dir(); + mu_assert("no_block: create blockfile", create_blockfile(1)); // only block 0 + const char* index_content = "t:table1\nb:5:100:0:0:key1\n"; // block 5 doesn't exist + mu_assert("no_block: write index", write_file(index_save_path, index_content)); + mu_assert("no_block: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("no_block: 0 keys (entry skipped)", details->db_keys == 0); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_corrupt_entry_line() { + setup_test_dir(); + mu_assert("corrupt: create blockfile", create_blockfile(2)); + const char* index_content = "t:table1\nb:not_a_number:100:0:0:key1\n"; + mu_assert("corrupt: write index", write_file(index_save_path, index_content)); + mu_assert("corrupt: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("corrupt: 0 keys (corrupt entry skipped)", details->db_keys == 0); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_entry_before_table() { + setup_test_dir(); + mu_assert("before_tbl: create blockfile", create_blockfile(1)); + const char* index_content = "b:0:100:0:0:key1\nt:table1\nb:0:100:0:0:key2\n"; + mu_assert("before_tbl: write index", write_file(index_save_path, index_content)); + mu_assert("before_tbl: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("before_tbl: 1 key (first entry skipped)", details->db_keys == 1); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_empty_index_save() { + setup_test_dir(); + mu_assert("empty_idx: create blockfile", create_blockfile(1)); + mu_assert("empty_idx: write index", write_file(index_save_path, "\n\n")); + mu_assert("empty_idx: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("empty_idx: 0 tables", kh_size(details->tables) == 0); + db_close(); + cleanup_test_dir(); + return 0; +} + +static const char * test_empty_blockfile() { + setup_test_dir(); + int fd = open(blockfile_path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + mu_assert("empty_bf: create empty blockfile", fd >= 0); + close(fd); + const char* index_content = "t:table1\n"; + mu_assert("empty_bf: write index", write_file(index_save_path, index_content)); + mu_assert("empty_bf: db_open", run_db_open()); + db_details* details = db_get_details(); + mu_assert("empty_bf: blocks_exist 0", details->blocks_exist == 0); + db_close(); + cleanup_test_dir(); + return 0; +} + +const char* test_db_load_all() { + mu_run_test(test_empty_dir); + mu_run_test(test_one_block_entry); + mu_run_test(test_multiple_entries_free_reconstruction); + mu_run_test(test_old_format_with_f_lines); + mu_run_test(test_multiple_tables); + mu_run_test(test_entry_with_expiry); + mu_run_test(test_malformed_line_no_colon); + mu_run_test(test_entry_nonexistent_block); + mu_run_test(test_corrupt_entry_line); + mu_run_test(test_entry_before_table); + mu_run_test(test_empty_index_save); + mu_run_test(test_empty_blockfile); + return 0; +} From 7dce40b58a7ca572f41882963e917b1df867a0b6 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sat, 16 May 2026 18:01:18 +0000 Subject: [PATCH 7/9] gitignore and agents --- .gitignore | 4 +++- AGENTS.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 AGENTS.md diff --git a/.gitignore b/.gitignore index 5e4c7ed..ceb79cf 100644 --- a/.gitignore +++ b/.gitignore @@ -173,4 +173,6 @@ gcc_Release.h *.o *.a src/server/scache -tests/tests \ No newline at end of file +tests/tests +core.* +tests/tests_db_load diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..abdd55e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,13 @@ +# simple-cache a C++ HTTP caching daemon + +Core Constraints: + - High Performance + - High Scalability + - Low overhead + - Trusted clients but crash safe + +Testing: run-test.sh` + +Building: `make` + +Running: ./src/server/scache \ No newline at end of file From 66542c34256477d2cef7e9976d3a2dbda4348dfa Mon Sep 17 00:00:00 2001 From: Mathew Date: Sun, 17 May 2026 01:12:47 +0000 Subject: [PATCH 8/9] cleanup --- src/core/connection.cpp | 19 ++++++++++++++----- src/core/connection_structures.h | 4 +--- src/core/db.cpp | 14 ++++++++++++-- src/core/http.cpp | 2 +- src/core/http_parse_cache.cpp | 4 ++++ src/core/http_parse_mon.cpp | 5 ++++- tests/Makefile | 8 +------- tests/scenario.cpp | 28 +++++++++++++++------------- tests/tests_db_load.cpp | 10 +++++----- 9 files changed, 57 insertions(+), 37 deletions(-) diff --git a/src/core/connection.cpp b/src/core/connection.cpp index b4442a5..fed52fd 100644 --- a/src/core/connection.cpp +++ b/src/core/connection.cpp @@ -1,4 +1,6 @@ -#define _GNU_SOURCE +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include #include @@ -35,6 +37,10 @@ #include "db.h" #endif +const char *state_action_string[] = { + "close_connection", "registered_write", "needs_more_read", "continue_processing" +}; + /* Globals */ listener_collection scache_listeners = { .listeners = NULL, .listener_count = 0 }; @@ -310,9 +316,11 @@ bool connection_remove(scache_connection* conn) { return true; } -static unsigned int connection_any() {\ +#ifdef DEBUG_BUILD +static unsigned int connection_any() { return connections.size(); } +#endif static void* connection_handle_accept(void *arg) { @@ -453,17 +461,19 @@ void close_socket(int fd){ } void close_fd(int fd, const char* descriptor_type){ - int ret; #ifdef DEBUG_BUILD + int ret; if(scache_listeners.listeners != NULL){ for (uint32_t i = 0; i < scache_listeners.listener_count; i++) { assert(scache_listeners.listeners[i].fd != fd); } } -#endif ret = close(fd); assert(ret == 0); +#else + close(fd); +#endif DEBUG("[#%d] Closed %s\n", fd, descriptor_type); } @@ -472,7 +482,6 @@ void monitoring_check(); void connection_event_loop(void (*connection_handler)(scache_connection* connection), int monitoring_fd) { epfd = epoll_create1(EPOLL_CLOEXEC); struct epoll_event events[NUM_EVENTS]; - int max_listener = 0; int res; int efd; pthread_t tid[2]; diff --git a/src/core/connection_structures.h b/src/core/connection_structures.h index 9cf38c6..a05a0d1 100644 --- a/src/core/connection_structures.h +++ b/src/core/connection_structures.h @@ -26,9 +26,7 @@ typedef enum { close_connection, registered_write, needs_more_read, continue_processing } state_action; -static const char *state_action_string[] = { - "close_connection", "registered_write", "needs_more_read", "continue_processing" -}; +extern const char *state_action_string[]; typedef enum { cache_listener, mon_listener, cache_connection, mon_connection diff --git a/src/core/db.cpp b/src/core/db.cpp index b4ff7aa..903ddf1 100644 --- a/src/core/db.cpp +++ b/src/core/db.cpp @@ -318,7 +318,9 @@ void db_entry_incref(cache_entry* entry, bool table = true) { } void db_lru_cleanup_percent(int* bytes_to_remove) { +#ifdef DEBUG_BUILD int debug_bytes = *bytes_to_remove; +#endif while (db.lru_head != NULL && *bytes_to_remove > 0) { cache_entry* l = db.lru_head; @@ -345,7 +347,9 @@ void db_lru_cleanup_percent(int* bytes_to_remove) { } } +#ifdef DEBUG_BUILD DEBUG("[#] LRU attempted to remove %d bytes, %d bytes remaining\n", debug_bytes, *bytes_to_remove); +#endif } // force_link() has been removed; blockfile.db is now the durable canonical store @@ -496,11 +500,12 @@ void db_lru_gc() { last_flush_ms = now_ms; } +#if 0 static void db_clear_directory(const char* directory) { char file_buffer[MAX_PATH]; struct dirent *next_file; DIR *theFolder = opendir(directory); - while (next_file = readdir(theFolder)) + while ((next_file = readdir(theFolder))) { if (next_file->d_name[0] == '.') continue; @@ -513,6 +518,7 @@ static void db_clear_directory(const char* directory) { PFATAL("Unable to close directory."); } } +#endif void db_init_folders() { mkdir(db.path_single, 0777); @@ -695,7 +701,7 @@ static bool db_load_from_save(){ } }else{ // Test size of blockfile - if(d1 >= db.blocks_exist){ + if((uint32_t)d1 >= db.blocks_exist){ DEBUG("skipping as block %d does not exist\n", d1); free(entry); continue; @@ -1388,6 +1394,7 @@ void db_target_write_allocate(struct cache_target* target, uint32_t data_length) } } +#if 0 static void db_close_table_key_space() { db_table* table; @@ -1412,7 +1419,9 @@ static void db_close_table_key_space() { } kh_destroy(table, db.tables); } +#endif +#if 0 static void db_close_blockfile() { block_free_node* bf = db.free_blocks; block_free_node* bf2; @@ -1423,6 +1432,7 @@ static void db_close_blockfile() { } db.free_blocks = NULL; } +#endif static bool full_write(int fd, const char* buffer, int buffer_length){ assert(buffer != NULL); diff --git a/src/core/http.cpp b/src/core/http.cpp index d50115a..9518b17 100644 --- a/src/core/http.cpp +++ b/src/core/http.cpp @@ -156,7 +156,7 @@ Initialize the http_templates_length structure with the length of the static http_templates. */ void http_templates_init() { - for (int i = 0; i < NUMBER_OF_HTTPTEMPLATE; i++) { + for (size_t i = 0; i < NUMBER_OF_HTTPTEMPLATE; i++) { http_templates_length[i] = strlen(http_templates[i]); } } diff --git a/src/core/http_parse_cache.cpp b/src/core/http_parse_cache.cpp index 2edf4be..5d51bd5 100644 --- a/src/core/http_parse_cache.cpp +++ b/src/core/http_parse_cache.cpp @@ -93,7 +93,9 @@ static bool http_key_lookup(scache_connection* connection, int n) { return http_write_response_after_eol(connection, HTTPTEMPLATE_FULLINVALIDMETHOD); } +#ifndef NDEBUG int type = connection->method; +#endif connection->method |= REQUEST_CACHE_LEVELKEY; assert(REQUEST_IS(connection->method, REQUEST_CACHE_LEVELKEY)); assert(REQUEST_IS(connection->method, connection->method)); @@ -208,7 +210,9 @@ static inline state_action http_read_requeststarturl1(scache_connection* connect char* key = (char*)malloc(sizeof(char) * (n + 1)); rbuf_copyn(&connection->input, key, n - 1); key[n - 1] = 0;//Null terminate the key +#ifndef NDEBUG int type = connection->method; +#endif connection->method |= REQUEST_CACHE_LEVELTABLE; assert(REQUEST_IS(connection->method, REQUEST_CACHE_LEVELTABLE)); assert(REQUEST_IS(connection->method, connection->method)); diff --git a/src/core/http_parse_mon.cpp b/src/core/http_parse_mon.cpp index 759300f..e7ff0fa 100644 --- a/src/core/http_parse_mon.cpp +++ b/src/core/http_parse_mon.cpp @@ -67,6 +67,7 @@ static state_action http_write_response_after_eol(scache_connection* connection, return needs_more_read; } +#if 0 static state_action http_write_response(scache_connection* connection, int http_template) { CONNECTION_HANDLER(connection, http_respond_writeonly); connection->output_buffer = http_templates[http_template]; @@ -75,6 +76,7 @@ static state_action http_write_response(scache_connection* connection, int http_ bool res = connection_register_write(connection); return res?registered_write:close_connection; } +#endif static state_action http_headers_response_after_eol(scache_connection* connection, int http_template) { @@ -156,6 +158,7 @@ state_action http_handle_mon_eolwrite_initial(scache_connection* connection) { +#if 0 static state_action http_headers_response_count(scache_connection* connection, int http_template) { connection->state = 2; CONNECTION_HANDLER(connection, http_handle_mon_eolwrite_initial); @@ -163,6 +166,7 @@ static state_action http_headers_response_count(scache_connection* connection, i connection->output_length = http_templates_length[http_template]; return needs_more_read; } +#endif static inline state_action http_read_requeststartmethod_mon(scache_connection* connection, char* buffer, int n) { //Check if this is never going to be valid, too long @@ -558,7 +562,6 @@ void monitoring_init(){ void monitoring_close(){ scache_connection* conn; - int flag = 1; static scache_connection* close_head = mon_head; while(close_head != NULL){ diff --git a/tests/Makefile b/tests/Makefile index 9d187ef..046bb75 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,4 @@ -OBJS = scenario.o tests.o tests_rbuffer.o tests_system_simple.o +OBJS = scenario.o tests.o SOURCE = scenario.cpp tests.cpp tests_rbuffer.cpp tests_system_simple.cpp HEADER = OUT = tests @@ -24,11 +24,5 @@ test_rbuffer_standalone.o: test_rbuffer_standalone.cpp tests.o: tests.cpp $(CC) $(_CFLAGS) tests.cpp -std=c++11 -tests_rbuffer.o: tests_rbuffer.cpp - $(CC) $(_CFLAGS) tests_rbuffer.cpp -std=c++11 - -tests_system_simple.o: tests_system_simple.cpp - $(CC) $(_CFLAGS) tests_system_simple.cpp -std=c++11 - clean: rm -f $(OBJS) $(OUT) tests_db_load \ No newline at end of file diff --git a/tests/scenario.cpp b/tests/scenario.cpp index 34fe6b9..6b70409 100644 --- a/tests/scenario.cpp +++ b/tests/scenario.cpp @@ -217,7 +217,7 @@ int remove_cr(char* buffer, int n){ int unit_connect(int port){ int sockfd; - struct sockaddr_in servaddr, cliaddr; + struct sockaddr_in servaddr; sockfd = socket(AF_INET, SOCK_STREAM, 0); @@ -229,7 +229,7 @@ int unit_connect(int port){ int res; struct timeval start_time; - int err = gettimeofday(&start_time, NULL); + gettimeofday(&start_time, NULL); struct timeval current_time; do { @@ -309,7 +309,7 @@ bool run_unit(std::string& request, std::string& expect, int sockfd){ if (strncmp(recv_buffer, buffer, n) != 0){ int read_length = buffer - expect.c_str(); *(recv_buffer + n) = 0;//Incase we arent comparing it all - if (expect.length() < (n + read_length)){ + if (expect.length() < (size_t)(n + read_length)){ printf("Expected (insufficient bytes): %s\n", expect.c_str()); } else{ @@ -331,7 +331,11 @@ pid_t start_server(const char* binary_path, int port, const char* db, const char char execcmd[512]; int res; - char* pidfile = tempnam(NULL, NULL); + char pidfile_template[] = "/tmp/scache-pid-XXXXXX"; + int pidfd = mkstemp(pidfile_template); + close(pidfd); + unlink(pidfile_template); + char* pidfile = strdup(pidfile_template); if (access(binary_path, X_OK)){ WARN("%s not executable or does not exist", binary_path); @@ -524,13 +528,13 @@ bool run_scenario(const char* binary, const char* testcases, const char* filenam printf("Running scenarios \"%s\"\n", testcases); char testcase_path[1024]; sprintf(testcase_path,"%s/%s", testcases, filename); - char* db = tempnam(NULL, NULL); - int res = mkdir(db, 0777); - bool result; - if (res < 0){ - free(db); - PFATAL("Failed to create temporary directory: %s", db); + char db_template[] = "/tmp/scache-db-XXXXXX"; + char* db = mkdtemp(db_template); + if (db == NULL){ + PFATAL("Failed to create temporary directory"); } + int res = 0; + bool result; pid_t pid; if (run_server){ // start the scache executable @@ -556,12 +560,10 @@ bool run_scenario(const char* binary, const char* testcases, const char* filenam sprintf(testcase_path, "rm -Rf \"%s\"", db); res = system(testcase_path); if (res < 0){ - free(db); PFATAL("Failed to clean up temporary directory: %s", db); } end: - free(db); return result; } @@ -574,7 +576,7 @@ bool run_scenarios(const char* binary, const char* testcases, const char* direct if (theFolder == NULL){ FATAL("%s does not exist", directory_buffer); } - while (next_file = readdir(theFolder)) + while ((next_file = readdir(theFolder))) { if (*(next_file->d_name) == '.') continue; diff --git a/tests/tests_db_load.cpp b/tests/tests_db_load.cpp index 638145f..f213294 100644 --- a/tests/tests_db_load.cpp +++ b/tests/tests_db_load.cpp @@ -12,9 +12,9 @@ #include "../src/core/db_structures.h" #include "../src/core/config.h" -static char test_dir[MAX_PATH]; -static char index_save_path[MAX_PATH]; -static char blockfile_path[MAX_PATH]; +static char test_dir[MAX_PATH + 32]; +static char index_save_path[MAX_PATH * 2]; +static char blockfile_path[MAX_PATH * 2]; static void setup_test_dir() { static int counter = 0; @@ -30,7 +30,7 @@ static void cleanup_test_dir() { unlink(blockfile_path); // Remove any temp files that might have been created - char temp_path[MAX_PATH]; + char temp_path[MAX_PATH * 2]; snprintf(temp_path, sizeof(temp_path), "%s/index.temp", test_dir); unlink(temp_path); @@ -54,7 +54,7 @@ static bool create_blockfile(int num_blocks) { } static bool run_db_open() { - char path_no_slash[MAX_PATH]; + char path_no_slash[MAX_PATH * 2]; snprintf(path_no_slash, sizeof(path_no_slash), "%s", test_dir); int len = (int)strlen(path_no_slash); if (len > 0 && path_no_slash[len-1] == '/') { From d13803a849f14a75d5cc0d70a0845fcd38557458 Mon Sep 17 00:00:00 2001 From: Mathew Date: Sun, 17 May 2026 04:15:13 +0000 Subject: [PATCH 9/9] add more tests --- IMPLEMENTATION_PLAN.md | 13 - tests/php/DataConsistencyHelper.php | 328 ++++++++++++++++++ tests/php/run-consistency-tests.sh | 136 ++++++++ .../test_10_content_expired_while_reading.php | 294 ++++++++++++++++ tests/php/test_11_size_switch.php | 287 +++++++++++++++ tests/php/test_12_concurrent_put_delete.php | 215 ++++++++++++ tests/php/test_13_head_during_write.php | 197 +++++++++++ .../test_14_crash_recovery_consistency.php | 313 +++++++++++++++++ .../php/test_15_lru_eviction_consistency.php | 173 +++++++++ tests/php/test_16_concurrent_readers.php | 233 +++++++++++++ .../test_17_table_operations_consistency.php | 295 ++++++++++++++++ tests/php/test_5_access_while_writing.php | 181 ++++++++++ tests/php/test_6_access_while_replacing.php | 307 ++++++++++++++++ .../php/test_7_interrupted_while_writing.php | 205 +++++++++++ .../test_8_interrupted_while_replacing.php | 192 ++++++++++ .../test_9_content_deleted_while_reading.php | 182 ++++++++++ 16 files changed, 3538 insertions(+), 13 deletions(-) create mode 100644 tests/php/DataConsistencyHelper.php create mode 100644 tests/php/run-consistency-tests.sh create mode 100644 tests/php/test_10_content_expired_while_reading.php create mode 100644 tests/php/test_11_size_switch.php create mode 100644 tests/php/test_12_concurrent_put_delete.php create mode 100644 tests/php/test_13_head_during_write.php create mode 100644 tests/php/test_14_crash_recovery_consistency.php create mode 100644 tests/php/test_15_lru_eviction_consistency.php create mode 100644 tests/php/test_16_concurrent_readers.php create mode 100644 tests/php/test_17_table_operations_consistency.php create mode 100644 tests/php/test_5_access_while_writing.php create mode 100644 tests/php/test_6_access_while_replacing.php create mode 100644 tests/php/test_7_interrupted_while_writing.php create mode 100644 tests/php/test_8_interrupted_while_replacing.php create mode 100644 tests/php/test_9_content_deleted_while_reading.php diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index f7ff384..384a82a 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -2,19 +2,6 @@ This repo currently does a full index flush (and a blockfile snapshot) from the write path via `db_lru_gc()`. Under high churn this can devolve into near-continuous flush attempts. The changes implemented in this PR address flush storms and a potential infinite loop in LRU cleanup. The remaining items below are larger design changes. -## 1. Remove Full Blockfile Copy During Flush - -Problem: -Each flush currently writes `index.save` and also updates `blockfile.db.save`. Today the "snapshot" of the blockfile is performed via `force_link()` which shells out to `cp`, which is O(size of blockfile) I/O every flush. This scales poorly and will dominate performance as the cache grows. - -Direction: -Make `blockfile.db` the durable canonical store and make `index.save` the only atomic metadata file. On startup: -1. Load `index.save`. -2. Reconstruct `free_blocks` by scanning all loaded entries and marking blocks in-use, then pushing the remaining blocks into the free list. - -Notes: -This eliminates `blockfile.db.save` and avoids copying the blockfile during steady-state operation. - ## 3. Move Expiration Work Off the Write Path Problem: diff --git a/tests/php/DataConsistencyHelper.php b/tests/php/DataConsistencyHelper.php new file mode 100644 index 0000000..681bd65 --- /dev/null +++ b/tests/php/DataConsistencyHelper.php @@ -0,0 +1,328 @@ += (int)$m[1]) break; + } else break; + } + } + fclose($sock); + return $response; +} + +function extractBody($response) +{ + $bs = strpos($response, "\r\n\r\n"); + return $bs !== false ? substr($response, $bs + 4) : ''; +} + +function isHttp200($r) +{ + return $r !== null && strpos($r, '200 OK') !== false; +} +function isHttp404($r) +{ + return $r !== null && (strpos($r, '404') !== false || strpos($r, 'Not Found') !== false); +} + +// ============================================================ +// Async PUT with pause (non-blocking socket) +// ============================================================ + +function startAsyncPutWithPause($host, $port, $table, $key, $content) +{ + $sf = tempnam(sys_get_temp_dir(), 'scache_'); + $pid = pcntl_fork(); + if ($pid == -1) { + echo "FAILED: fork\n"; + exit(1); + } + if ($pid == 0) { + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + if (!$sock) { + file_put_contents($sf, "error:$errstr"); + exit(1); + } + stream_set_blocking($sock, false); + $hdr = "PUT /$table/$key HTTP/1.1\r\nHost: $host:$port\r\nConnection: close\r\nContent-Length: " . strlen($content) . "\r\n\r\n"; + $w = 0; + $hl = strlen($hdr); + while ($w < $hl) { + $n = @fwrite($sock, substr($hdr, $w)); + if ($n === false || $n === 0) { + usleep(10000); + continue; + } + $w += $n; + } + file_put_contents($sf, 'ready'); + for ($i = 0; $i < 300; $i++) { + $s = @file_get_contents($sf); + if ($s === 'go' || strpos($s, 'kill') === 0) break; + usleep(50000); + } + $s = @file_get_contents($sf); + if (strpos($s, 'kill') === 0) { + fclose($sock); + file_put_contents($sf, 'done:interrupted'); + exit(0); + } + $w = 0; + $bl = strlen($content); + $st = time(); + while ($w < $bl) { + $n = @fwrite($sock, substr($content, $w)); + if ($n === false || $n === 0) { + if (time() - $st > 5) break; + usleep(10000); + continue; + } + $w += $n; + } + $resp = ''; + $st = time(); + while (time() - $st < 3) { + $d = @fread($sock, 4096); + if ($d !== false && $d !== '') { + $resp .= $d; + if (strpos($resp, "\r\n\r\n") !== false) { + if (preg_match('/Content-Length: (\d+)/i', $resp, $m)) { + $he = strpos($resp, "\r\n\r\n") + 4; + if (strlen($resp) - $he >= (int)$m[1]) break; + } else break; + } + } + usleep(10000); + } + fclose($sock); + file_put_contents($sf, 'done:' . $resp); + exit(0); + } + for ($i = 0; $i < 300; $i++) { + clearstatcache(); + if (!file_exists($sf)) { + usleep(50000); + continue; + } + $s = @file_get_contents($sf); + if ($s === 'ready' || strpos($s, 'error:') === 0) break; + usleep(50000); + } + return ['pid' => $pid, 'signalFile' => $sf]; +} + +function signalAsyncPutContinue($info) +{ + file_put_contents($info['signalFile'], 'go'); +} +function signalAsyncPutKill($info) +{ + file_put_contents($info['signalFile'], 'kill'); +} + +function waitAsyncPut($info) +{ + pcntl_waitpid($info['pid'], $status); + for ($i = 0; $i < 300; $i++) { + clearstatcache(); + $c = @file_get_contents($info['signalFile']); + if ($c === false) { + usleep(50000); + continue; + } + if (strpos($c, 'done:') === 0) { + @unlink($info['signalFile']); + return substr($c, 5); + } + if (strpos($c, 'error:') === 0) { + @unlink($info['signalFile']); + return null; + } + usleep(50000); + } + @unlink($info['signalFile']); + return null; +} + +// ============================================================ +// Async GET (slow consumer) +// ============================================================ + +function startAsyncGet($host, $port, $table, $key) +{ + $rf = tempnam(sys_get_temp_dir(), 'scache_get_'); + $pid = pcntl_fork(); + if ($pid == -1) { + echo "FAILED: fork\n"; + exit(1); + } + if ($pid == 0) { + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + if (!$sock) { + file_put_contents($rf, "error:$errstr"); + exit(1); + } + stream_set_blocking($sock, false); + $req = "GET /$table/$key HTTP/1.1\r\nHost: $host:$port\r\nConnection: close\r\n\r\n"; + $w = 0; + $rl = strlen($req); + while ($w < $rl) { + $n = @fwrite($sock, substr($req, $w)); + if ($n === false || $n === 0) { + usleep(10000); + continue; + } + $w += $n; + } + file_put_contents($rf, 'sent'); + $resp = ''; + $st = time(); + while (time() - $st < 5) { + $d = @fread($sock, 256); + if ($d !== false && $d !== '') { + $resp .= $d; + if (strpos($resp, "\r\n\r\n") !== false) { + if (preg_match('/Content-Length: (\d+)/i', $resp, $m)) { + $he = strpos($resp, "\r\n\r\n") + 4; + if (strlen($resp) - $he >= (int)$m[1]) break; + } else break; + } + } + usleep(10000); + } + fclose($sock); + file_put_contents($rf, 'done:' . extractBody($resp)); + exit(0); + } + for ($i = 0; $i < 300; $i++) { + clearstatcache(); + $s = @file_get_contents($rf); + if ($s === 'sent') break; + usleep(50000); + } + return ['pid' => $pid, 'resultFile' => $rf]; +} + +function waitAsyncGet($info) +{ + pcntl_waitpid($info['pid'], $status); + for ($i = 0; $i < 300; $i++) { + clearstatcache(); + $c = @file_get_contents($info['resultFile']); + if ($c === false) { + usleep(50000); + continue; + } + if (strpos($c, 'done:') === 0) { + @unlink($info['resultFile']); + return substr($c, 5); + } + if (strpos($c, 'error:') === 0) { + @unlink($info['resultFile']); + return null; + } + usleep(50000); + } + @unlink($info['resultFile']); + return null; +} + +// ============================================================ +// Utility +// ============================================================ + +function waitForChild($pid) +{ + pcntl_waitpid($pid, $status); + return $status; +} + +function getServerPid($pf) +{ + if (!file_exists($pf)) return null; + $p = (int)trim(file_get_contents($pf)); + return $p > 0 ? $p : null; +} + +function killServer($pf, $sig = SIGKILL) +{ + $p = getServerPid($pf); + if ($p) { + posix_kill($p, $sig); + for ($i = 0; $i < 100; $i++) { + if (!posix_kill($p, 0)) break; + usleep(100000); + } + } + if (file_exists($pf)) @unlink($pf); +} + +function testHeader($name) +{ + echo "\n========================================\nTEST: $name\n========================================\n"; +} +function testResult($name, $passed) +{ + echo ($passed ? "PASS" : "FAIL") . ": $name\n"; + return $passed; +} diff --git a/tests/php/run-consistency-tests.sh b/tests/php/run-consistency-tests.sh new file mode 100644 index 0000000..f6e36d0 --- /dev/null +++ b/tests/php/run-consistency-tests.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# run-consistency-tests.sh - Run PHP data consistency tests for simple-cache +# Usage: ./run-consistency-tests.sh [port] +# Default port: 8081 + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +PORT="${1:-8081}" +HOST="127.0.0.1" +PIDFILE="/tmp/scache-consistency-test.pid" +DBDIR="/tmp/scache-consistency-test-db" +SCACHE_BIN="$PROJECT_DIR/src/server/scache" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +PASSED=0 +FAILED=0 + +echo "==========================================" +echo "simple-cache Data Consistency Tests" +echo "==========================================" +echo "Host: $HOST:$PORT" +echo "DB: $DBDIR" +echo "" + +# Build scache if needed +if [ ! -x "$SCACHE_BIN" ]; then + echo -e "${YELLOW}Building simple-cache...${NC}" + cd "$PROJECT_DIR" + make clean && make + if [ ! -x "$SCACHE_BIN" ]; then + echo -e "${RED}Failed to build simple-cache${NC}" + exit 1 + fi +fi + +# Clean up any previous run +cleanup() { + if [ -f "$PIDFILE" ]; then + PID=$(cat "$PIDFILE" 2>/dev/null) + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + kill -9 "$PID" 2>/dev/null || true + fi + rm -f "$PIDFILE" + fi + rm -rf "$DBDIR" +} +cleanup + +# Start server +start_server() { + local extra_args="${1:-}" + echo -e "${YELLOW}Starting scache on $HOST:$PORT...${NC}" + rm -rf "$DBDIR" + mkdir -p "$DBDIR" + + # Start scache with correct CLI flags (daemon mode) + $SCACHE_BIN \ + -r "$DBDIR" \ + -b "$HOST:$PORT" \ + -m "$PIDFILE" \ + -d \ + $extra_args + + # Wait for server to be ready (daemon writes real PID after fork) + for i in $(seq 1 30); do + if [ -f "$PIDFILE" ]; then + PID=$(cat "$PIDFILE") + # PID must be non-zero (daemon parent writes 0, child writes real PID) + if [ -n "$PID" ] && [ "$PID" -gt 0 ] 2>/dev/null && kill -0 "$PID" 2>/dev/null; then + sleep 0.5 + # Verify it's listening + if curl -s -o /dev/null -w "%{http_code}" "http://$HOST:$PORT/" 2>/dev/null | grep -q "200\|404"; then + echo -e "${GREEN}Server started (PID: $PID)${NC}" + return 0 + fi + fi + fi + sleep 0.5 + done + + echo -e "${RED}Failed to start server${NC}" + return 1 +} + +# Run a single test +run_test() { + local test_file="$1" + local test_name="$(basename "$test_file" .php)" + + echo -n " $test_name ... " + + if php "$test_file" "$HOST" "$PORT" 2>&1; then + echo -e "${GREEN}PASS${NC}" + PASSED=$((PASSED + 1)) + return 0 + else + echo -e "${RED}FAIL${NC}" + FAILED=$((FAILED + 1)) + return 1 + fi +} + +# Start server with default config +start_server "" || exit 1 + +echo "" +echo "Running tests..." +echo "" + +# Run all consistency test files +cd "$SCRIPT_DIR" +for test_file in test_5_*.php test_6_*.php test_7_*.php test_8_*.php test_9_*.php test_10_*.php test_11_*.php test_12_*.php test_13_*.php test_14_*.php test_15_*.php test_16_*.php test_17_*.php; do + if [ -f "$test_file" ]; then + run_test "$test_file" + fi +done + +echo "" +echo "==========================================" +echo -e "Results: ${GREEN}$PASSED passed${NC}, ${RED}$FAILED failed${NC}" +echo "==========================================" + +# Cleanup +cleanup + +if [ "$FAILED" -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/tests/php/test_10_content_expired_while_reading.php b/tests/php/test_10_content_expired_while_reading.php new file mode 100644 index 0000000..28f4d1c --- /dev/null +++ b/tests/php/test_10_content_expired_while_reading.php @@ -0,0 +1,294 @@ + $size) { + $content = generateKnownContent($size); + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + $request = "PUT /t11_multi/$key HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: $size\r\n\r\n$content"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 5); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + + $passed = strpos($response, '200 OK') !== false; + if (!$passed) { + testResult("PUT size $size (switch #$idx)", false); + $allPassed = false; + continue; + } + + // Verify + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + $request = "GET /t11_multi/$key HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 5); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + $bodyStart = strpos($response, "\r\n\r\n"); + $body = $bodyStart !== false ? substr($response, $bodyStart + 4) : ''; + $passed = verifyContent($body, $content, "GET after switch #$idx (size $size)"); + testResult("Switch #$idx: size $size", $passed); + $allPassed = $allPassed && $passed; +} + +echo "\n"; +if ($allPassed) { + echo "test_11_size_switch: ALL TESTS PASSED\n"; + exit(0); +} else { + echo "test_11_size_switch: SOME TESTS FAILED\n"; + exit(1); +} \ No newline at end of file diff --git a/tests/php/test_12_concurrent_put_delete.php b/tests/php/test_12_concurrent_put_delete.php new file mode 100644 index 0000000..971f0e3 --- /dev/null +++ b/tests/php/test_12_concurrent_put_delete.php @@ -0,0 +1,215 @@ + 0 && posix_kill($pid, 0)) { + usleep(500000); + // Verify it's listening + $sock = @fsockopen($host, $port, $errno, $errstr, 2); + if ($sock) { + fclose($sock); + return true; + } + } + } + usleep(500000); + } + echo "FAILED: Could not restart server\n"; + return false; +} + +// ============================================================ +// PUT several keys, kill, restart, verify all exist +// ============================================================ +testHeader('Crash recovery: PUT keys, kill, restart, verify'); + +$keys = [ + 'k_small1' => generateKnownContent(100), + 'k_small2' => generateKnownContent(200), + 'k_large1' => generateKnownContent(5000), + 'k_small3' => generateKnownContent(300), + 'k_large2' => generateKnownContent(6000), +]; + +foreach ($keys as $key => $content) { + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + assertOrDie($sock !== false, "Could not connect: $errstr"); + $request = "PUT /t14_persist/$key HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: " . strlen($content) . "\r\n\r\n$content"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 5); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + + if (strpos($response, '200 OK') === false) { + testResult("PUT $key", false); + $allPassed = false; + } +} + +// Wait a moment for any pending flush +sleep(1); + +// Kill server hard +echo " Killing server with SIGKILL...\n"; +killServer($pidFile, SIGKILL); + +// Restart +echo " Restarting server...\n"; +$restarted = restartServer($host, $port, $pidFile, $dbDir, $scacheBin); +assertOrDie($restarted, "Server restart failed"); + +// Verify all keys survived +foreach ($keys as $key => $expectedContent) { + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + assertOrDie($sock !== false, "Could not connect: $errstr"); + $request = "GET /t14_persist/$key HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 5); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + + $bodyStart = strpos($response, "\r\n\r\n"); + $body = $bodyStart !== false ? substr($response, $bodyStart + 4) : ''; + $passed = verifyContent($body, $expectedContent, "GET $key after restart"); + testResult("Key $key survives restart", $passed); + $allPassed = $allPassed && $passed; +} + +// ============================================================ +// Async PUT with pause, kill, restart, verify key not present +// ============================================================ +testHeader('Crash recovery: Interrupted write not persisted'); + +$content = generateKnownContent(SMALL_SIZE); + +// Start async PUT with pause +$info = startAsyncPutWithPause($host, $port, 't14_interrupt', 'k1', $content); + +// Kill server while PUT is paused (writing=true, not in index) +echo " Killing server during paused PUT...\n"; +killServer($pidFile, SIGKILL); + +// Clean up the paused child +signalAsyncPutKill($info); +waitAsyncPut($info); + +// Restart +echo " Restarting server...\n"; +$restarted = restartServer($host, $port, $pidFile, $dbDir, $scacheBin); +assertOrDie($restarted, "Server restart failed"); + +// GET - should return 404 (partial write not persisted) +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "GET /t14_interrupt/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$passed = strpos($response, '404') !== false || strpos($response, 'Not Found') !== false; +testResult('Interrupted write not persisted after restart', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// PUT → DELETE → kill → restart → verify key gone +// ============================================================ +testHeader('Crash recovery: Deleted key stays deleted'); + +$content = generateKnownContent(SMALL_SIZE); + +// PUT +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "PUT /t14_delete/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: " . strlen($content) . "\r\n\r\n$content"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); +$passed = strpos($response, '200 OK') !== false; +testResult('PUT succeeds', $passed); +$allPassed = $allPassed && $passed; + +// DELETE +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "DELETE /t14_delete/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); +$passed = strpos($response, '200 OK') !== false || strpos($response, 'DELETED') !== false; +testResult('DELETE succeeds', $passed); +$allPassed = $allPassed && $passed; + +// Wait for flush +sleep(1); + +// Kill and restart +echo " Killing server...\n"; +killServer($pidFile, SIGKILL); +echo " Restarting server...\n"; +$restarted = restartServer($host, $port, $pidFile, $dbDir, $scacheBin); +assertOrDie($restarted, "Server restart failed"); + +// GET - should return 404 +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "GET /t14_delete/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$passed = strpos($response, '404') !== false || strpos($response, 'Not Found') !== false; +testResult('Deleted key stays deleted after restart', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// PUT → replace (complete) → kill → restart → verify new value +// ============================================================ +testHeader('Crash recovery: Replaced value persists'); + +$originalContent = generateKnownContent(SMALL_SIZE); +$replacementContent = generateKnownContent(SMALL_SIZE + 100); + +// PUT original +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "PUT /t14_replace/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: " . strlen($originalContent) . "\r\n\r\n$originalContent"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +// PUT replacement (complete) +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "PUT /t14_replace/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: " . strlen($replacementContent) . "\r\n\r\n$replacementContent"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); +$passed = strpos($response, '200 OK') !== false; +testResult('Replace PUT succeeds', $passed); +$allPassed = $allPassed && $passed; + +// Wait for flush +sleep(1); + +// Kill and restart +echo " Killing server...\n"; +killServer($pidFile, SIGKILL); +echo " Restarting server...\n"; +$restarted = restartServer($host, $port, $pidFile, $dbDir, $scacheBin); +assertOrDie($restarted, "Server restart failed"); + +// GET - should return replacement content +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "GET /t14_replace/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$bodyStart = strpos($response, "\r\n\r\n"); +$body = $bodyStart !== false ? substr($response, $bodyStart + 4) : ''; +$passed = verifyContent($body, $replacementContent, 'GET after restart'); +testResult('Replaced value persists after restart', $passed); +$allPassed = $allPassed && $passed; + +echo "\n"; +if ($allPassed) { + echo "test_14_crash_recovery_consistency: ALL TESTS PASSED\n"; + exit(0); +} else { + echo "test_14_crash_recovery_consistency: SOME TESTS FAILED\n"; + exit(1); +} \ No newline at end of file diff --git a/tests/php/test_15_lru_eviction_consistency.php b/tests/php/test_15_lru_eviction_consistency.php new file mode 100644 index 0000000..738bee1 --- /dev/null +++ b/tests/php/test_15_lru_eviction_consistency.php @@ -0,0 +1,173 @@ +0 protection in db_lru_cleanup_percent) + * + * NOTE: Requires server started with --database-max-size. + * The test runner should start a separate server instance for this test. + */ + +require_once __DIR__ . '/DataConsistencyHelper.php'; + +$host = $argv[1] ?? '127.0.0.1'; +$port = (int)($argv[2] ?? 8081); + +$allPassed = true; + +// ============================================================ +// Fill cache beyond limit, verify oldest entries evicted +// ============================================================ +testHeader('LRU: Fill beyond limit, oldest evicted'); + +// We'll use small content to fit many entries +// The server should be started with --database-max-size 50000 (50KB) +// Each entry is ~100 bytes of data + overhead + +// PUT 20 entries of ~3000 bytes each = ~60KB, should trigger LRU +$numEntries = 20; +$entrySize = 3000; + +echo " Putting $numEntries entries of $entrySize bytes each...\n"; + +for ($i = 0; $i < $numEntries; $i++) { + $content = generateKnownContent($entrySize); + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + if (!$sock) { + echo " Connection failed at entry $i, server may have crashed\n"; + break; + } + $request = "PUT /t15_fill/k$i HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: $entrySize\r\n\r\n$content"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 5); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + + if (strpos($response, '200 OK') === false) { + echo " PUT failed at entry $i\n"; + } +} + +// Check which keys survived - the oldest (lowest index) should be evicted +$survived = 0; +$evicted = 0; +for ($i = 0; $i < $numEntries; $i++) { + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + if (!$sock) break; + $request = "GET /t15_fill/k$i HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 3); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + + if (strpos($response, '200 OK') !== false) { + $survived++; + } else { + $evicted++; + } +} + +echo " Survived: $survived, Evicted: $evicted\n"; +$passed = $evicted > 0; // At least some entries should be evicted +testResult('Some entries evicted when cache exceeds limit', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// LRU eviction with active reader (refcount protection) +// ============================================================ +testHeader('LRU: Active reader protected from eviction'); + +// PUT a key that we'll read +$protectedContent = generateKnownContent(2000); +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +assertOrDie($sock !== false, "Could not connect: $errstr"); +$request = "PUT /t15_protect/protected HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: 2000\r\n\r\n$protectedContent"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); +$passed = strpos($response, '200 OK') !== false; +testResult('PUT protected key succeeds', $passed); +$allPassed = $allPassed && $passed; + +// Start async GET on the protected key (slow consumer) +$getInfo = startAsyncGet($host, $port, 't15_protect', 'protected'); + +// Now fill cache with many entries to trigger LRU eviction +echo " Filling cache to trigger LRU while GET is in progress...\n"; +for ($i = 0; $i < 30; $i++) { + $content = generateKnownContent(2000); + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + if (!$sock) break; + $request = "PUT /t15_protect/filler$i HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: 2000\r\n\r\n$content"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 3); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); +} + +// Wait for GET to complete - should get full content despite LRU pressure +$getResult = waitAsyncGet($getInfo); +$passed = verifyContent($getResult, $protectedContent, 'GET result during LRU eviction'); +testResult('Active reader completes with full content during LRU eviction', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// Verify evicted entries return 404 +// ============================================================ +testHeader('LRU: Evicted entries return 404'); + +// Try to GET k0 from the fill test - should be evicted (oldest) +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +if ($sock) { + $request = "GET /t15_fill/k0 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 3); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + fclose($sock); + + // k0 may or may not be evicted depending on exact sizing + $is404 = strpos($response, '404') !== false || strpos($response, 'Not Found') !== false; + $is200 = strpos($response, '200 OK') !== false; + $passed = $is404 || $is200; // Either is valid + testResult('Oldest entry state is consistent (404 or 200)', $passed); + $allPassed = $allPassed && $passed; +} + +echo "\n"; +if ($allPassed) { + echo "test_15_lru_eviction_consistency: ALL TESTS PASSED\n"; + exit(0); +} else { + echo "test_15_lru_eviction_consistency: SOME TESTS FAILED\n"; + exit(1); +} \ No newline at end of file diff --git a/tests/php/test_16_concurrent_readers.php b/tests/php/test_16_concurrent_readers.php new file mode 100644 index 0000000..3f7e99e --- /dev/null +++ b/tests/php/test_16_concurrent_readers.php @@ -0,0 +1,233 @@ + $pid, 'file' => $resultFile]; +} + +// Wait for all children +$allCorrect = true; +foreach ($children as $child) { + pcntl_waitpid($child['pid'], $status); + $result = @file_get_contents($child['file']); + @unlink($child['file']); + + if (!verifyContent($result, $content, 'concurrent GET')) { + $allCorrect = false; + } +} + +$passed = $allCorrect; +testResult('All 10 concurrent GETs return correct content', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// Mixed GET/PUT/GET on same key +// ============================================================ +testHeader('Concurrent readers: Mixed GET/PUT/GET'); + +$originalContent = generateKnownContent(SMALL_SIZE); +$newContent = generateKnownContent(SMALL_SIZE + 50); + +// PUT original +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "PUT /t16_mixed/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: " . strlen($originalContent) . "\r\n\r\n$originalContent"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +// Fork children doing mixed operations +$children = []; +$numChildren = 10; + +for ($i = 0; $i < $numChildren; $i++) { + $resultFile = tempnam(sys_get_temp_dir(), 'scache_mix_'); + + $pid = pcntl_fork(); + if ($pid == -1) { + echo "FAILED: Could not fork\n"; + exit(1); + } + + if ($pid == 0) { + $sock = @fsockopen($host, $port, $errno, $errstr, 5); + if (!$sock) { + file_put_contents($resultFile, "error:socket"); + exit(1); + } + + // Each child does: GET, then maybe PUT, then GET + $results = []; + + // GET 1 + $request = "GET /t16_mixed/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 3); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + $bodyStart = strpos($response, "\r\n\r\n"); + $results['get1'] = $bodyStart !== false ? substr($response, $bodyStart + 4) : null; + + // Some children do PUT + if ($i % 3 == 0) { + $request = "PUT /t16_mixed/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: " . strlen($newContent) . "\r\n\r\n$newContent"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 3); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + $results['put'] = strpos($response, '200 OK') !== false; + } + + // GET 2 + $request = "GET /t16_mixed/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; + fwrite($sock, $request); + $response = ''; + stream_set_timeout($sock, 3); + while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; + } + $bodyStart = strpos($response, "\r\n\r\n"); + $results['get2'] = $bodyStart !== false ? substr($response, $bodyStart + 4) : null; + + fclose($sock); + file_put_contents($resultFile, serialize($results)); + exit(0); + } + + $children[] = ['pid' => $pid, 'file' => $resultFile]; +} + +// Wait for all children and verify consistency +$allConsistent = true; +foreach ($children as $child) { + pcntl_waitpid($child['pid'], $status); + $data = @file_get_contents($child['file']); + @unlink($child['file']); + + if ($data === false || strpos($data, 'error:') === 0) { + $allConsistent = false; + continue; + } + + $results = @unserialize($data); + if (!is_array($results)) { + $allConsistent = false; + continue; + } + + // Each GET should return either original or new content (never partial/corrupt) + foreach (['get1', 'get2'] as $getKey) { + if (isset($results[$getKey]) && $results[$getKey] !== null) { + $isOriginal = verifyContent($results[$getKey], $originalContent); + $isNew = verifyContent($results[$getKey], $newContent); + if (!$isOriginal && !$isNew) { + echo " FAILED: GET returned neither original nor new content\n"; + $allConsistent = false; + } + } + } +} + +$passed = $allConsistent; +testResult('Mixed GET/PUT/GET returns consistent data (old or new, never corrupt)', $passed); +$allPassed = $allPassed && $passed; + +echo "\n"; +if ($allPassed) { + echo "test_16_concurrent_readers: ALL TESTS PASSED\n"; + exit(0); +} else { + echo "test_16_concurrent_readers: SOME TESTS FAILED\n"; + exit(1); +} \ No newline at end of file diff --git a/tests/php/test_17_table_operations_consistency.php b/tests/php/test_17_table_operations_consistency.php new file mode 100644 index 0000000..81b5cc3 --- /dev/null +++ b/tests/php/test_17_table_operations_consistency.php @@ -0,0 +1,295 @@ +4096 bytes) + * + * Verifies: GET during an in-progress PUT returns 404 (writing flag), + * and GET after completion returns correct content. + */ + +require_once __DIR__ . '/DataConsistencyHelper.php'; + +$host = $argv[1] ?? '127.0.0.1'; +$port = (int)($argv[2] ?? 8081); + +$allPassed = true; + +// ============================================================ +// A1 × B1: Small content (blockdb) - access while writing +// ============================================================ +testHeader('A1×B1: Access while writing (small/blockdb)'); + +$content = generateKnownContent(SMALL_SIZE); + +// Start async PUT with pause after headers +$info = startAsyncPutWithPause($host, $port, 't5_small', 'k1', $content); + +// While PUT is paused (writing=true), try GET - should get 404 +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +assertOrDie($sock !== false, "Could not connect: $errstr"); + +$request = "GET /t5_small/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$passed = strpos($response, '404') !== false || strpos($response, 'Not Found') !== false; +testResult('GET during write returns 404', $passed); +$allPassed = $allPassed && $passed; + +// Now signal PUT to continue and complete +signalAsyncPutContinue($info); +$putResponse = waitAsyncPut($info); +$passed = strpos($putResponse, '200 OK') !== false; +testResult('PUT completes successfully', $passed); +$allPassed = $allPassed && $passed; + +// GET after completion should return correct content +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "GET /t5_small/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$bodyStart = strpos($response, "\r\n\r\n"); +$body = $bodyStart !== false ? substr($response, $bodyStart + 4) : ''; +$passed = verifyContent($body, $content, 'GET after write'); +testResult('GET after write returns correct content', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// A1 × B2: Large content (file) - access while writing +// ============================================================ +testHeader('A1×B2: Access while writing (large/file)'); + +$content = generateKnownContent(LARGE_SIZE); + +$info = startAsyncPutWithPause($host, $port, 't5_large', 'k1', $content); + +// GET during write - should get 404 +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +assertOrDie($sock !== false, "Could not connect: $errstr"); + +$request = "GET /t5_large/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$passed = strpos($response, '404') !== false || strpos($response, 'Not Found') !== false; +testResult('GET during large write returns 404', $passed); +$allPassed = $allPassed && $passed; + +// Complete PUT +signalAsyncPutContinue($info); +$putResponse = waitAsyncPut($info); +$passed = strpos($putResponse, '200 OK') !== false; +testResult('Large PUT completes successfully', $passed); +$allPassed = $allPassed && $passed; + +// GET after completion +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "GET /t5_large/k1 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 5); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$bodyStart = strpos($response, "\r\n\r\n"); +$body = $bodyStart !== false ? substr($response, $bodyStart + 4) : ''; +$passed = verifyContent($body, $content, 'GET after large write'); +testResult('GET after large write returns correct content', $passed); +$allPassed = $allPassed && $passed; + +// ============================================================ +// Variant: GET different key in same table during write +// ============================================================ +testHeader('A1 Variant: GET different key during write'); + +// Pre-populate another key +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$otherContent = generateKnownContent(50); +$request = "PUT /t5_small/k2 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\nContent-Length: 50\r\n\r\n$otherContent"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +// Now start a write on k1 +$content3 = generateKnownContent(SMALL_SIZE); +$info = startAsyncPutWithPause($host, $port, 't5_small', 'k1', $content3); + +// GET k2 during write on k1 - should succeed normally +$sock = @fsockopen($host, $port, $errno, $errstr, 5); +$request = "GET /t5_small/k2 HTTP/1.1\r\nHost: $host:$port\r\nConnection: Keep-Alive\r\n\r\n"; +fwrite($sock, $request); +$response = ''; +stream_set_timeout($sock, 3); +while (!feof($sock)) { + $data = @fread($sock, 4096); + if ($data === false || $data === '') break; + $response .= $data; +} +fclose($sock); + +$bodyStart = strpos($response, "\r\n\r\n"); +$body = $bodyStart !== false ? substr($response, $bodyStart + 4) : ''; +$passed = verifyContent($body, $otherContent, 'GET different key'); +testResult('GET different key during write succeeds', $passed); +$allPassed = $allPassed && $passed; + +// Cleanup +signalAsyncPutContinue($info); +waitAsyncPut($info); + +echo "\n"; +if ($allPassed) { + echo "test_5_access_while_writing: ALL TESTS PASSED\n"; + exit(0); +} else { + echo "test_5_access_while_writing: SOME TESTS FAILED\n"; + exit(1); +} \ No newline at end of file diff --git a/tests/php/test_6_access_while_replacing.php b/tests/php/test_6_access_while_replacing.php new file mode 100644 index 0000000..39235a6 --- /dev/null +++ b/tests/php/test_6_access_while_replacing.php @@ -0,0 +1,307 @@ +