From 2422342f8c9e576bf601bca5caf8c4ba40bd3954 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 28 Aug 2020 16:16:41 -0400 Subject: [PATCH 01/54] ast.h: Add `extern` for `ast_expr_tombstone`. This is a backport of #239 to the fastly private mirror for testing. --- src/libre/ast.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libre/ast.h b/src/libre/ast.h index 7f9e82c4c..dc5edabf6 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -184,7 +184,7 @@ struct ast { struct ast_expr *expr; }; -struct ast_expr *ast_expr_tombstone; +extern struct ast_expr *ast_expr_tombstone; struct ast * ast_new(void); From 796c6abd6100686d1d884b795ab6f7a662c35416 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 30 Aug 2024 14:43:47 -0400 Subject: [PATCH 02/54] gen: No end states means nothing to do. (todo: cherry pick) --- src/libfsm/gen.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 26a77be1c..e0ac429dd 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -147,6 +147,10 @@ fsm_generate_matches(struct fsm *fsm, size_t max_length, return 0; } + if (!fsm_has(fsm, fsm_isend)) { + return 1; /* no end state -> nothing to do */ + } + INIT_TIMERS(); TIME(&pre); int res = gen_init_outer(fsm, max_length, cb, opaque, false, 0); From 4a32885726915980244ee89d957147cc97d63815 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 27 Aug 2024 14:08:39 -0400 Subject: [PATCH 03/54] experimental: Add eager outputs, similar to endids but eagerly matched. When combining several unanchored regexes it becomes VERY expensive to handle combinations of matches via the end state -- essentially, the whole reachable DFA gets separate matching and non-matching copies for each pattern, leading to a DFA whose size is proportional to the number of *possible combinations* of matches. With eager outputs, we can set a flag for matching as we reach the end of the original pattern (before looping back and possibly also matching other patterns), which keeps the state count from blowing up in fsm_determinise. To see how much difference this makes, the test tests/eager_output/run7 combines 26 different patterns. It should finish very quickly (~50 msec, just now). Try running it with `env FORCE_ENDIDS=N` for N increasing from 4 to 26. Around 10-11 it will start taking several seconds, and memory usage will roughly double with each step. This PR adds `fsm_union_repeated_pattern_group`, a variant of `fsm_union_array` that combines a set of DFAs into a single NFA, but correctly handles a mix of anchored and unanchored ends without the state count blowing up. It currently needs flags passed in for each fsm indicating whether the start and/or end are anchored, and there is a hacky special case that removes self-edges from states with eager outputs and instead connects them to a single overall unanchored end loop. I haven't yet figured out how to handle this properly in the general case, but it works for this specific use case, provided all the DFAs are combined at once. (Combining multiple DFAs each produced by determinising fsm_union_repeated_pattern_group's result probably won't work correctly.) I have tried detecting and ignoring those edges in fsm_determinise, after epsilon removal, but so far either it still causes the graph size to blow up or subtly breaks something else. This is still experimental, and the code generation for `-lc` here is quite hacky -- it expects the caller to define a `FSM_SET_EAGER_OUTPUT` acro, since the code generation interface doesn't define where the match info will go yet. A later PR will add a new code generation mode with better support for eager outputs, and I plan to eventually integrate this better with rx, AMBIG_MULTIPLE, and so on. (This squashes down a couple false starts.) --- Makefile | 1 + fuzz/target.c | 524 +++++++++++++++++- include/fsm/bool.h | 10 + include/fsm/fsm.h | 46 ++ include/fsm/print.h | 3 + include/re/re.h | 15 + src/libfsm/Makefile | 1 + src/libfsm/clone.c | 38 ++ src/libfsm/consolidate.c | 46 ++ src/libfsm/determinise.c | 131 ++++- src/libfsm/determinise_internal.h | 10 +- src/libfsm/eager_output.c | 403 ++++++++++++++ src/libfsm/eager_output.h | 46 ++ src/libfsm/epsilons.c | 147 ++++- src/libfsm/exec.c | 55 +- src/libfsm/fsm.c | 11 + src/libfsm/internal.h | 5 + src/libfsm/libfsm.syms | 11 + src/libfsm/merge.c | 42 ++ src/libfsm/minimise.c | 79 ++- src/libfsm/print/c.c | 13 + src/libfsm/print/ir.c | 35 ++ src/libfsm/print/ir.h | 5 + src/libfsm/state.c | 8 + src/libfsm/union.c | 233 ++++++++ src/libre/libre.syms | 1 + src/libre/re.c | 34 ++ tests/eager_output/Makefile | 22 + tests/eager_output/eager_output1.c | 12 + tests/eager_output/eager_output2.c | 17 + tests/eager_output/eager_output3.c | 16 + tests/eager_output/eager_output4.c | 13 + tests/eager_output/eager_output5.c | 14 + tests/eager_output/eager_output6.c | 34 ++ tests/eager_output/eager_output7.c | 103 ++++ tests/eager_output/eager_output_at_start.c | 12 + tests/eager_output/eager_output_fr1.c | 13 + tests/eager_output/eager_output_fr2.c | 13 + tests/eager_output/eager_output_fr3.c | 13 + .../eager_output_mixed_anchored_unanchored.c | 46 ++ tests/eager_output/utils.c | 278 ++++++++++ tests/eager_output/utils.h | 64 +++ 42 files changed, 2587 insertions(+), 36 deletions(-) create mode 100644 src/libfsm/eager_output.c create mode 100644 src/libfsm/eager_output.h create mode 100644 tests/eager_output/Makefile create mode 100644 tests/eager_output/eager_output1.c create mode 100644 tests/eager_output/eager_output2.c create mode 100644 tests/eager_output/eager_output3.c create mode 100644 tests/eager_output/eager_output4.c create mode 100644 tests/eager_output/eager_output5.c create mode 100644 tests/eager_output/eager_output6.c create mode 100644 tests/eager_output/eager_output7.c create mode 100644 tests/eager_output/eager_output_at_start.c create mode 100644 tests/eager_output/eager_output_fr1.c create mode 100644 tests/eager_output/eager_output_fr2.c create mode 100644 tests/eager_output/eager_output_fr3.c create mode 100644 tests/eager_output/eager_output_mixed_anchored_unanchored.c create mode 100644 tests/eager_output/utils.c create mode 100644 tests/eager_output/utils.h diff --git a/Makefile b/Makefile index 514f80bba..b356c4f0f 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,7 @@ SUBDIR += tests/equals SUBDIR += tests/subtract SUBDIR += tests/detect_required SUBDIR += tests/determinise +SUBDIR += tests/eager_output SUBDIR += tests/endids SUBDIR += tests/epsilons SUBDIR += tests/fsm diff --git a/fuzz/target.c b/fuzz/target.c index 543891bb9..d56a9bf82 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -26,10 +26,21 @@ /* 10 seconds */ #define TIMEOUT_USEC (10ULL * 1000 * 1000) +static bool verbosity_checked = false; +static bool verbose = false; + +#define LOG(...) \ + do { \ + if (verbose) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while (0) \ + enum run_mode { MODE_DEFAULT, MODE_SHUFFLE_MINIMISE, MODE_ALL_PRINT_FUNCTIONS, + MODE_EAGER_OUTPUT, }; @@ -344,6 +355,508 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const return EXIT_SUCCESS; } +#define MAX_PATTERNS 4 +struct eager_output_cb_info { + size_t used; + fsm_output_id_t ids[MAX_PATTERNS]; +}; + +static void +reset_eager_output_info(struct eager_output_cb_info *info) +{ + info->used = 0; +} + +struct feo_env { + bool ok; + size_t pattern_count; + size_t fsm_count; + size_t max_match_count; + size_t max_steps; + + char *patterns[MAX_PATTERNS]; + struct fsm *fsms[MAX_PATTERNS]; + struct fsm *combined; + + /* which pattern is being used for generation, (size_t)-1 for combined */ + size_t current_pattern; + + struct eager_output_cb_info outputs; + struct eager_output_cb_info outputs_combined; +}; + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque) +{ + struct eager_output_cb_info *info = (struct eager_output_cb_info *)opaque; + + for (size_t i = 0; i < info->used; i++) { + if (info->ids[i] == id) { + return; /* already present */ + } + } + + assert(info->used < MAX_PATTERNS); + info->ids[info->used++] = id; +} + +static enum fsm_generate_matches_cb_res +gen_combined_check_individual_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +static enum fsm_generate_matches_cb_res +gen_individual_check_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +#define DEF_MAX_STEPS 100000 +#define DEF_MAX_MATCH_COUNT 1000 + +/* This isn't part of the public interface, per se. */ +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +static int +fuzz_eager_output(const uint8_t *data, size_t size) +{ + struct feo_env env = { + .ok = true, + .pattern_count = 0, + .max_steps = DEF_MAX_STEPS, + .max_match_count = DEF_MAX_MATCH_COUNT, + }; + + { + const char *steps = getenv("STEPS"); + const char *matches = getenv("MATCHES"); + if (steps != NULL) { + env.max_steps = strtoul(steps, NULL, 10); + assert(env.max_steps > 0); + } + if (matches != NULL) { + env.max_match_count = strtoul(matches, NULL, 10); + assert(env.max_match_count > 0); + } + } + + int ret = 0; + + size_t max_pattern_length = 0; + + /* chop data into a series of patterns */ + { + size_t prev = 0; + size_t offset = 0; + + /* Patterns with lots of '.' can take a while to determinise. + * That slows down fuzzer coverage, but isn't interesting here. */ + size_t dots = 0; + + while (offset < size && env.pattern_count < MAX_PATTERNS) { +#define MAX_DOTS 4 + if (data[offset] == '.') { dots++; } + + if (data[offset] == '\0' || data[offset] == '\n' || offset == size - 1) { + size_t len = offset - prev; + + if (dots > MAX_DOTS) { + /* ignored */ + prev = offset; + } else if (len > 0) { + char *pattern = malloc(len + 1); + assert(pattern != NULL); + + memcpy(pattern, &data[prev], len); + if (len > 0 && pattern[len] == '\n') { + len--; /* drop trailing newline */ + } + pattern[len] = '\0'; + bool keep = true; + + if (len > 0) { + for (size_t i = 0; i < len - 1; i++) { + if (pattern[i] == '\\' && pattern[i + 1] == 'x') { + /* ignore unhandled parser errors from "\x", see #386 */ + keep = false; + } + } + } + + if (keep) { + env.patterns[env.pattern_count++] = pattern; + + if (len > max_pattern_length) { + max_pattern_length = len; + } + } else { + free(pattern); + } + prev = offset; + dots = 0; + } + } + + offset++; + } + } + + struct re_anchoring_info anchorage[MAX_PATTERNS] = {0}; + + /* for each pattern, attempt to compile to a DFA */ + for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { + const char *p = env.patterns[p_i]; + + if (!re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL, &anchorage[p_i])) { + continue; /* unsupported regex */ + } + + p = env.patterns[p_i]; + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); + + LOG("%s: pattern %zd: '%s' => %p\n", __func__, p_i, env.patterns[p_i], (void *)fsm); + + if (fsm == NULL) { + continue; /* invalid regex */ + } + + const fsm_output_id_t endid = (fsm_output_id_t)p_i; + ret = fsm_seteageroutputonends(fsm, endid); + assert(ret == 1); + + if (verbose) { + fprintf(stderr, "==== pattern %zd, pre det\n", p_i); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(fsm); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(fsm, i)); + } + } + + ret = fsm_determinise(fsm); + assert(ret == 1); + + ret = fsm_minimise(fsm); + assert(ret == 1); + + fsm_state_t start; + if (!fsm_getstart(fsm, &start)) { + fsm_free(fsm); + continue; + } + + if (verbose) { + fprintf(stderr, "==== pattern %zd, post det\n", p_i); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(fsm); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(fsm, i)); + } + } + + fsm_eager_output_set_cb(fsm, append_eager_output_cb, &env.outputs); + env.fsms[env.fsm_count++] = fsm; + } + + /* don't bother checking combined behavior unless there's multiple DFAs */ + if (env.fsm_count < 2) { goto cleanup; } + + /* copy and combine fsms into one DFA */ + { + size_t used = 0; + struct fsm_union_entry entries[MAX_PATTERNS] = {0}; + + for (size_t i = 0; i < env.fsm_count; i++) { + /* there can be gaps, fsms[] lines up with patterns[] */ + if (env.fsms[i] == NULL) { continue; } + + fsm_state_t start; + if (!fsm_getstart(env.fsms[i], &start)) { + assert(!"hit"); + } + + struct fsm *cp = fsm_clone(env.fsms[i]); + assert(cp != NULL); + + if (verbose) { + fprintf(stderr, "==== cp %zd\n", i); + fsm_dump(stderr, cp); + fsm_eager_output_dump(stderr, cp); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(cp); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(cp, i)); + } + } + + entries[used].fsm = cp; + entries[used].anchored_start = anchorage[i].start; + entries[used].anchored_end = anchorage[i].end; + used++; + } + + if (used == 0) { + goto cleanup; /* nothing to do */ + } + + /* consumes entries[] */ + struct fsm *fsm = fsm_union_repeated_pattern_group(used, entries, NULL); + assert(fsm != NULL); + + if (verbose) { + fprintf(stderr, "==== combined (pre-det)\n"); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + if (!fsm_determinise(fsm)) { + assert(!"failed to determinise"); + } + + if (!fsm_minimise(fsm)) { + assert(!"failed to minimise"); + } + + LOG("%s: combined state_count %d\n", __func__, fsm_countstates(fsm)); + env.combined = fsm; + /* fsm_eager_output_set_cb(fsm, append_eager_output_cb, &env.outputs_combined); */ + + if (verbose) { + fprintf(stderr, "==== combined\n"); + fsm_dump(stderr, env.combined); + fsm_eager_output_dump(stderr, env.combined); + fprintf(stderr, "====\n"); + } + + } + + /* Use fsm_generate_matches to check for matches that got lost + * and false positives introduced while combining the DFAs. + * Use the combined DFA to generate matches, check that the + * match behavior agrees with the individual DFA copies. */ + env.current_pattern = (size_t)-1; + if (!fsm_generate_matches(env.combined, max_pattern_length, gen_combined_check_individual_cb, &env)) { + goto cleanup; + } + + if (!env.ok) { goto cleanup; } + + /* Likewise, use every individual DFA to generate matches and */ + /* check behavior against the combined DFA. */ + for (size_t i = 0; i < env.pattern_count; i++) { + env.current_pattern = i; + if (!fsm_generate_matches(env.combined, max_pattern_length, gen_individual_check_combined_cb, &env)) { + goto cleanup; + } + } + + ret = env.ok ? EXIT_SUCCESS : EXIT_FAILURE; +cleanup: + for (size_t i = 0; i < MAX_PATTERNS; i++) { + if (env.patterns[i] != NULL) { + free(env.patterns[i]); + env.patterns[i] = NULL; + } + if (env.fsms[i] != NULL) { + fsm_free(env.fsms[i]); + } + } + if (env.combined != NULL) { + fsm_free(env.combined); + } + + return ret; +} + +static int +cmp_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static bool +match_input_get_eager_outputs(struct fsm *fsm, const char *input, size_t input_length, + struct eager_output_cb_info *dst) +{ + (void)input_length; + fsm_state_t end; + + reset_eager_output_info(dst); + + fsm_eager_output_set_cb(fsm, append_eager_output_cb, dst); + const int ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + if (ret == 0) { + return false; /* no match */ + } else { + assert(ret == 1); /* match */ + } + + /* sort the IDs, to make comparison cheaper */ + qsort(dst->ids, dst->used, sizeof(dst->ids[0]), cmp_output_id); + return true; /* match */ +} + +/* For a given matching input generated by the combined DFA, check that + * only the expected individual source DFAs match. */ +static enum fsm_generate_matches_cb_res +gen_combined_check_individual_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + (void)fsm; + (void)depth; + (void)end_state; + + struct feo_env *env = opaque; + assert(env->current_pattern == (size_t)-1); + + if (match_count > env->max_match_count) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + + /* execute, to set eager outputs */ + if (!match_input_get_eager_outputs(env->combined, input, input_length, &env->outputs_combined)) { + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + size_t individual_outputs_used = 0; + fsm_output_id_t individual_outputs[MAX_PATTERNS]; + + for (size_t i = 0; i < env->pattern_count; i++) { + struct fsm *fsm = env->fsms[i]; + if (fsm == NULL) { continue; } + + if (!match_input_get_eager_outputs(fsm, input, input_length, &env->outputs)) { + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (env->outputs.used > 0) { + assert(env->outputs.used == 1); + individual_outputs[individual_outputs_used++] = env->outputs.ids[0]; + } + } + + bool match = true; + if (env->outputs_combined.used != individual_outputs_used) { + match = false; + } + + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + const fsm_output_id_t cur = env->outputs_combined.ids[cmb_i]; + assert(env->fsms[cmb_i] != NULL); + bool found = false; + for (size_t i = 0; i < individual_outputs_used; i++) { + if (individual_outputs[i] == cur) { + found = true; + break; + } + } + if (!found) { + match = false; + break; + } + } + + if (!match) { + fprintf(stderr, "%s: combined <-> individual mismatch for input '%s'(%zd)!\n", __func__, input, input_length); + + fprintf(stderr, "-- combined: %zu IDs:", env->outputs_combined.used); + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + fprintf(stderr, " %d", env->outputs_combined.ids[cmb_i]); + } + fprintf(stderr, "\n"); + fprintf(stderr, "-- individiual: %zu IDs:", individual_outputs_used); + for (size_t i = 0; i < individual_outputs_used; i++) { + fprintf(stderr, " %d", individual_outputs[i]); + } + fprintf(stderr, "\n"); + goto fail; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; + +fail: + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; +} + +/* For a given matching input generated by one of the source DFAs, check that + * the combined DFA also matches, and that the only other source DFAs that match + * are ones that should according to the combined DFA. */ +static enum fsm_generate_matches_cb_res +gen_individual_check_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + (void)fsm; + (void)depth; + (void)end_state; + + struct feo_env *env = opaque; + assert(env->current_pattern < env->pattern_count); + if (match_count > env->max_match_count) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + + struct fsm *cur_fsm = env->fsms[env->current_pattern]; + if (cur_fsm == NULL) { return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; } + + /* execute, to set eager outputs */ + if (!match_input_get_eager_outputs(cur_fsm, input, input_length, &env->outputs)) { + goto fail; + } + if (!match_input_get_eager_outputs(env->combined, input, input_length, &env->outputs_combined)) { + goto fail; + } + + assert(env->outputs.used == 1); + + bool found = false; + for (size_t i = 0; i < env->outputs_combined.used; i++) { + if (env->outputs_combined.ids[i] == env->outputs.ids[0]) { + found = true; + break; + } + } + + if (!found) { + fprintf(stderr, "%s: combined <-> individual mismatch for input '%s'(%zd)!\n", __func__, input, input_length); + + fprintf(stderr, "-- combined: %zu IDs:", env->outputs_combined.used); + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + fprintf(stderr, " %d", env->outputs_combined.ids[cmb_i]); + } + fprintf(stderr, "\n"); + fprintf(stderr, "-- pattern %zd: %zu IDs:", env->current_pattern, env->outputs.used); + for (size_t i = 0; i < env->outputs.used; i++) { + fprintf(stderr, " %d", env->outputs.ids[i]); + } + fprintf(stderr, "\n"); + goto fail; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; + +fail: + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; +} +#undef MAX_PATTERNS + #define MAX_FUZZER_DATA (64 * 1024) static uint8_t data_buf[MAX_FUZZER_DATA + 1]; @@ -358,6 +871,7 @@ get_run_mode(void) switch (mode[0]) { case 'm': return MODE_SHUFFLE_MINIMISE; case 'p': return MODE_ALL_PRINT_FUNCTIONS; + case 'E': return MODE_EAGER_OUTPUT; case 'd': default: return MODE_DEFAULT; @@ -373,6 +887,11 @@ harness_fuzzer_target(const uint8_t *data, size_t size) return EXIT_SUCCESS; } + if (!verbosity_checked) { + verbosity_checked = true; + verbose = getenv("VERBOSE") != NULL; + } + /* Ensure that input is '\0'-terminated. */ if (size > MAX_FUZZER_DATA) { size = MAX_FUZZER_DATA; @@ -392,6 +911,9 @@ harness_fuzzer_target(const uint8_t *data, size_t size) case MODE_SHUFFLE_MINIMISE: return shuffle_minimise(pattern); + case MODE_EAGER_OUTPUT: + return fuzz_eager_output(data, size); + case MODE_ALL_PRINT_FUNCTIONS: { if (dev_null == NULL) { @@ -403,7 +925,7 @@ harness_fuzzer_target(const uint8_t *data, size_t size) const bool det = b0 & 0x1; const bool min = b0 & 0x2; const enum fsm_io io_mode = (b0 >> 2) % 3; - + const char *shifted_pattern = (const char *)&data_buf[1]; int res = fuzz_all_print_functions(dev_null, shifted_pattern, det, min, io_mode); return res; diff --git a/include/fsm/bool.h b/include/fsm/bool.h index d92518297..4d9f1889a 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -52,6 +52,16 @@ struct fsm * fsm_union_array(size_t fsm_count, struct fsm **fsms, struct fsm_combined_base_pair *bases); +struct fsm_union_entry { + struct fsm *fsm; + bool anchored_start; + bool anchored_end; +}; + +struct fsm * +fsm_union_repeated_pattern_group(size_t entry_count, + struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases); + struct fsm * fsm_intersect(struct fsm *a, struct fsm *b); diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 877d5c1bf..701efe70b 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -7,6 +7,7 @@ #ifndef FSM_H #define FSM_H +#include #include struct fsm; @@ -27,6 +28,9 @@ typedef unsigned int fsm_state_t; * original FSM(s) matched when executing a combined FSM. */ typedef unsigned int fsm_end_id_t; +/* Eager output ID. */ +typedef unsigned int fsm_output_id_t; + #define FSM_END_ID_MAX UINT_MAX /* @@ -266,6 +270,39 @@ fsm_mapendids(struct fsm * fsm, fsm_endid_remap_fun remap, void *opaque); void fsm_increndids(struct fsm * fsm, int delta); +/* Associate an eagerly matched numeric ID with the end states in an fsm. + * + * This is similar to fsm_setendid, but has different performance + * trade-offs. In particular, it can become extremely expensive to + * combine multiple DFAs with endids on their end states when they + * representing regexes with unanchored ends, because the FSM has to + * explicitly represent all the possible combinations of matches by + * copying the entire path to every reachable end state. Eager endids + * are associated with the edge leaving the main pattern match. + * + * Returns 1 on success, 0 on error. + * */ +int +fsm_seteagerendid(struct fsm *fsm, fsm_end_id_t id); + +/* Set an eager output ID to emit every time the state is entered. + * This turns the automata into a Moore machine. */ +int +fsm_seteageroutput(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id); + +/* Set an eager output ID on all current end states. */ +int +fsm_seteageroutputonends(struct fsm *fsm, fsm_output_id_t id); + +/* HACK */ +typedef void +fsm_eager_output_cb(fsm_output_id_t id, void *opaque); +void +fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque); + +void +fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque); + /* * Find the state (if there is just one), or add epsilon edges from all states, * for which the given predicate is true. @@ -436,6 +473,15 @@ fsm_shortest(const struct fsm *fsm, fsm_state_t start, fsm_state_t goal, unsigned (*cost)(fsm_state_t from, fsm_state_t to, char c)); +/* HACK */ +typedef void +fsm_eager_endid_cb(fsm_end_id_t id, void *opaque); +void +fsm_eager_endid_set_cb(struct fsm *fsm, fsm_eager_endid_cb *cb, void *opaque); + +void +fsm_eager_endid_get_cb(const struct fsm *fsm, fsm_eager_endid_cb **cb, void **opaque); + /* * Execute an FSM reading input from the user-specified callback fsm_getc(). * fsm_getc() is passed the opaque pointer given, and is expected to return diff --git a/include/fsm/print.h b/include/fsm/print.h index 9f7264e81..10244129b 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -45,6 +45,9 @@ enum fsm_print_lang { struct fsm_state_metadata { const fsm_end_id_t *end_ids; size_t end_id_count; + + const fsm_output_id_t *eager_output_ids; + size_t eager_output_count; }; /* diff --git a/include/re/re.h b/include/re/re.h index 20408e98a..a3e1f7e0c 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -136,6 +136,21 @@ re_comp(enum re_dialect dialect, const struct fsm_alloc *alloc, enum re_flags flags, struct re_err *err); +struct re_anchoring_info { + int start; + int end; + /* FIXME: this could also check for AST_FLAG_NULLABLE, AST_FLAG_UNSATISFIABLE, + * AST_FLAG_ALWAYS_CONSUMES, AST_FLAG_CAN_CONSUME */ +}; + +/* Parse and analyze the regex enough to determine whether it is + * anchored at the start and/or end. Returns 0 if the regex is not + * supported, otherwise returns 1 and writes anchoring flags into *info. */ +int +re_is_anchored(enum re_dialect dialect, re_getchar_fun *f, void *opaque, + enum re_flags flags, struct re_err *err, + struct re_anchoring_info *info); + /* * Return a human-readable string describing a given error code. The string * returned has static storage, and must not be freed. diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 5e2ed57e3..c7782f0ff 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -8,6 +8,7 @@ SRC += src/libfsm/consolidate.c SRC += src/libfsm/clone.c SRC += src/libfsm/closure.c SRC += src/libfsm/detect_required.c +SRC += src/libfsm/eager_output.c SRC += src/libfsm/edge.c SRC += src/libfsm/empty.c SRC += src/libfsm/end.c diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 9fd236a4d..2161599ae 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -19,6 +19,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define LOG_CLONE_ENDIDS 0 @@ -28,6 +29,9 @@ copy_capture_actions(struct fsm *dst, const struct fsm *src); static int copy_end_ids(struct fsm *dst, const struct fsm *src); +static int +copy_eager_output_ids(struct fsm *dst, const struct fsm *src); + struct fsm * fsm_clone(const struct fsm *fsm) { @@ -80,6 +84,12 @@ fsm_clone(const struct fsm *fsm) fsm_free(new); return NULL; } + + /* does not copy callback */ + if (!copy_eager_output_ids(new, fsm)) { + fsm_free(new); + return NULL; + } } return new; @@ -159,3 +169,31 @@ copy_end_ids(struct fsm *dst, const struct fsm *src) return env.ok; } + +struct copy_eager_output_ids_env { + bool ok; + struct fsm *dst; +}; + +static int +copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct copy_eager_output_ids_env *env = opaque; + if (!fsm_seteageroutput(env->dst, state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +copy_eager_output_ids(struct fsm *dst, const struct fsm *src) +{ + struct copy_eager_output_ids_env env; + env.dst = dst; + env.ok = true; + + fsm_eager_output_iter_all(src, copy_eager_output_ids_cb, &env); + return env.ok; +} diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 236a4f6f5..b7a8905b2 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -25,6 +25,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define LOG_MAPPING 0 #define LOG_CONSOLIDATE_CAPTURES 0 @@ -53,6 +54,10 @@ static int consolidate_end_ids(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count); +static int +consolidate_eager_output_ids(struct fsm *dst, const struct fsm *src, + const fsm_state_t *mapping, size_t mapping_count); + static fsm_state_t mapping_cb(fsm_state_t id, const void *opaque) { @@ -154,6 +159,10 @@ fsm_consolidate(const struct fsm *src, } } + if (!consolidate_eager_output_ids(dst, src, mapping, mapping_count)) { + goto cleanup; + } + f_free(src->alloc, seen); return dst; @@ -270,3 +279,40 @@ consolidate_end_ids(struct fsm *dst, const struct fsm *src, return ret; } + +struct consolidate_eager_output_ids_env { + bool ok; + struct fsm *dst; + const fsm_state_t *mapping; + size_t mapping_count; +}; + +static int +consolidate_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct consolidate_eager_output_ids_env *env = opaque; + assert(state < env->mapping_count); + const fsm_state_t dst_state = env->mapping[state]; + + if (!fsm_seteageroutput(env->dst, dst_state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +consolidate_eager_output_ids(struct fsm *dst, const struct fsm *src, + const fsm_state_t *mapping, size_t mapping_count) +{ + struct consolidate_eager_output_ids_env env = { + .ok = true, + .dst = dst, + .mapping = mapping, + .mapping_count = mapping_count, + }; + fsm_eager_output_iter_all(src, consolidate_eager_output_ids_cb, &env); + return env.ok; +} + diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 42992b6bc..8978ce06c 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -6,6 +6,9 @@ #include "determinise_internal.h" +#include +#include + static void dump_labels(FILE *f, const uint64_t labels[4]) { @@ -253,6 +256,10 @@ fsm_determinise(struct fsm *nfa) goto cleanup; } + if (!remap_eager_outputs(&map, issp, dfa, nfa)) { + goto cleanup; + } + fsm_move(nfa, dfa); } @@ -334,6 +341,22 @@ add_reverse_mapping(const struct fsm_alloc *alloc, return 1; } +static void +free_reverse_mappings(const struct fsm_alloc *alloc, size_t map_count, struct reverse_mapping *rmaps) +{ + if (rmaps == NULL) { return; } + + for (size_t map_i = 0; map_i < map_count; map_i++) { + struct reverse_mapping *rmap = &rmaps[map_i]; + for (size_t i = 0; i < rmap->count; i++) { + f_free(alloc, rmap[i].list); + rmap->count = 0; + rmap[i].list = NULL; + } + } + f_free(alloc, rmaps); +} + static int det_copy_capture_actions_cb(fsm_state_t state, enum capture_action_type type, unsigned capture_id, fsm_state_t to, @@ -405,7 +428,7 @@ hash_iss(interned_state_set_id iss) } static struct mapping * -map_first(struct map *map, struct map_iter *iter) +map_first(const struct map *map, struct map_iter *iter) { iter->m = map; iter->i = 0; @@ -641,22 +664,14 @@ stack_pop(struct mappingstack *stack) return item; } -static int -remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, - struct fsm *dst_dfa, struct fsm *src_nfa) +static struct reverse_mapping * +build_reverse_mappings(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa) { + struct reverse_mapping *reverse_mappings = NULL; struct map_iter it; struct state_iter si; struct mapping *m; - struct reverse_mapping *reverse_mappings; - fsm_state_t state; - const size_t capture_count = fsm_countcaptures(src_nfa); - size_t i, j; - int res = 0; - - if (capture_count == 0) { - return 1; - } /* This is not 1 to 1 -- if state X is now represented by multiple * states Y in the DFA, and state X has action(s) when transitioning @@ -667,9 +682,7 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, * checking reachability from every X, but the actual path * handling later will also check reachability. */ reverse_mappings = f_calloc(dst_dfa->alloc, src_nfa->statecount, sizeof(reverse_mappings[0])); - if (reverse_mappings == NULL) { - return 0; - } + if (reverse_mappings == NULL) { goto cleanup; } /* build reverse mappings table: for every NFA state X, if X is part * of the new DFA state Y, then add Y to a list for X */ @@ -679,6 +692,7 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, assert(m->dfastate < dst_dfa->statecount); ss = interned_state_set_get_state_set(issp, iss_id); + fsm_state_t state; for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { if (!add_reverse_mapping(dst_dfa->alloc, reverse_mappings, @@ -688,33 +702,47 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, } } -#if LOG_DETERMINISE_CAPTURES +#if LOG_BUILD_REVERSE_MAPPING fprintf(stderr, "#### reverse mapping for %zu states\n", src_nfa->statecount); - for (i = 0; i < src_nfa->statecount; i++) { + for (size_t i = 0; i < src_nfa->statecount; i++) { struct reverse_mapping *rm = &reverse_mappings[i]; fprintf(stderr, "%lu:", i); - for (j = 0; j < rm->count; j++) { + for (size_t j = 0; j < rm->count; j++) { fprintf(stderr, " %u", rm->list[j]); } fprintf(stderr, "\n"); } -#else - (void)j; #endif + return reverse_mappings; + +cleanup: + free_reverse_mappings(dst_dfa->alloc, src_nfa->statecount, reverse_mappings); + return NULL; +} + +static int +remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, struct fsm *src_nfa) +{ + const size_t capture_count = fsm_countcaptures(src_nfa); + int res = 0; + + if (capture_count == 0) { + return 1; + } + + struct reverse_mapping *reverse_mappings = build_reverse_mappings(map, issp, dst_dfa, src_nfa); + if (reverse_mappings == NULL) { goto cleanup; } + if (!det_copy_capture_actions(reverse_mappings, dst_dfa, src_nfa)) { goto cleanup; } res = 1; -cleanup: - for (i = 0; i < src_nfa->statecount; i++) { - if (reverse_mappings[i].list != NULL) { - f_free(dst_dfa->alloc, reverse_mappings[i].list); - } - } - f_free(dst_dfa->alloc, reverse_mappings); +cleanup: + free_reverse_mappings(dst_dfa->alloc, src_nfa->statecount, reverse_mappings); return res; } @@ -2528,3 +2556,50 @@ analyze_closures__grow_outputs(struct analyze_closures_env *env) env->output_ceil = nceil; return 1; } + +struct remap_eager_output_env { + bool ok; + struct fsm *dst; + fsm_state_t dst_state; +}; + +static int +remap_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct remap_eager_output_env *env = opaque; + if (!fsm_seteageroutput(env->dst, env->dst_state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +remap_eager_outputs(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa) +{ + /* For each DFA state, get the set of NFA states corresponding to it from the + * map and issp, then copy every eager output ID over. */ + struct map_iter iter; + for (struct mapping *b = map_first(map, &iter); b != NULL; b = map_next(&iter)) { + struct state_set *ss = interned_state_set_get_state_set(issp, b->iss); + assert(ss != NULL); + + struct state_iter it; + fsm_state_t s; + state_set_reset(ss, &it); + while (state_set_next(&it, &s)) { + struct remap_eager_output_env env = { + .ok = true, + .dst = dst_dfa, + .dst_state = b->dfastate, + }; + fsm_eager_output_iter_state(src_nfa, s, remap_eager_output_cb, &env); + if (!env.ok) { return 0; } + } + } + + return 1; +} diff --git a/src/libfsm/determinise_internal.h b/src/libfsm/determinise_internal.h index cfd4ea663..2e925d28c 100644 --- a/src/libfsm/determinise_internal.h +++ b/src/libfsm/determinise_internal.h @@ -23,6 +23,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #include @@ -35,6 +36,7 @@ #define LOG_AC 0 #define LOG_GROUPING 0 #define LOG_ANALYSIS_STATS 0 +#define LOG_BUILD_REVERSE_MAPPING 0 #if LOG_DETERMINISE_CAPTURES || LOG_INPUT #include @@ -72,7 +74,7 @@ struct map { }; struct map_iter { - struct map *m; + const struct map *m; size_t i; }; @@ -304,7 +306,7 @@ static void map_free(struct map *map); static struct mapping * -map_first(struct map *map, struct map_iter *iter); +map_first(const struct map *map, struct map_iter *iter); static struct mapping * map_next(struct map_iter *iter); @@ -325,6 +327,10 @@ static int remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, struct fsm *dst_dfa, struct fsm *src_nfa); +static int +remap_eager_outputs(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa); + static struct mappingstack * stack_init(const struct fsm_alloc *alloc); diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c new file mode 100644 index 000000000..e37a8a4bf --- /dev/null +++ b/src/libfsm/eager_output.c @@ -0,0 +1,403 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include + +#include "internal.h" + +#include +#include + +#include +#include +#include + +#include "eager_output.h" + +#define LOG_LEVEL 0 + +/* must be a power of 2 */ +#define DEF_BUCKET_COUNT 4 +#define DEF_ENTRY_CEIL 2 + +struct eager_output_info { + fsm_eager_output_cb *cb; + void *opaque; + + struct eager_output_htab { + size_t bucket_count; + size_t buckets_used; + /* empty if entry is NULL, otherwise keyed by state */ + struct eager_output_bucket { + fsm_state_t state; + struct eager_output_entry { + unsigned used; + unsigned ceil; + fsm_end_id_t ids[]; + } *entry; + } *buckets; + } htab; +}; + +void +fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque) +{ +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_set_cb %p\n", (void *)fsm); +#endif + assert(fsm != NULL); + assert(fsm->eager_output_info != NULL); + fsm->eager_output_info->cb = cb; + fsm->eager_output_info->opaque = opaque; +} + +void +fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque) +{ + *cb = fsm->eager_output_info->cb; + *opaque = fsm->eager_output_info->opaque; +} + +int +fsm_eager_output_init(struct fsm *fsm) +{ + struct eager_output_info *ei = f_calloc(fsm->alloc, 1, sizeof(*ei)); + + if (ei == NULL) { return 0; } + + struct eager_output_bucket *buckets = f_calloc(fsm->alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + f_free(fsm->alloc, ei); + return 0; + } + +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_init %p\n", (void *)fsm); +#endif + + ei->htab.buckets = buckets; + ei->htab.bucket_count = DEF_BUCKET_COUNT; + + fsm->eager_output_info = ei; + return 1; +} + +void +fsm_eager_output_free(struct fsm *fsm) +{ + if (fsm == NULL || fsm->eager_output_info == NULL) { return; } + + for (size_t i = 0; i < fsm->eager_output_info->htab.bucket_count; i++) { + struct eager_output_bucket *b = &fsm->eager_output_info->htab.buckets[i]; + if (b->entry == NULL) { continue; } + f_free(fsm->alloc, b->entry); + } + f_free(fsm->alloc, fsm->eager_output_info->htab.buckets); + + f_free(fsm->alloc, fsm->eager_output_info); +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_free %p\n", (void *)fsm); +#endif + fsm->eager_output_info = NULL; +} + +int +fsm_seteageroutputonends(struct fsm *fsm, fsm_output_id_t id) +{ + assert(fsm != NULL); + const size_t count = fsm_countstates(fsm); + for (size_t i = 0; i < count; i++) { + if (fsm_isend(fsm, i)) { + if (!fsm_seteageroutput(fsm, i, id)) { return 0; } + } + } + return 1; +} + +static bool +grow_htab(const struct fsm_alloc *alloc, struct eager_output_htab *htab) +{ + const size_t nbucket_count = 2*htab->bucket_count; + assert(nbucket_count != 0); + + struct eager_output_bucket *nbuckets = f_calloc(alloc, nbucket_count, + sizeof(nbuckets[0])); + if (nbuckets == NULL) { return false; } + + const uint64_t nmask = nbucket_count - 1; + assert((nmask & nbucket_count) == 0); /* power of 2 */ + + for (size_t ob_i = 0; ob_i < htab->bucket_count; ob_i++) { + struct eager_output_bucket *ob = &htab->buckets[ob_i]; + if (ob->entry == NULL) { continue; } + + const uint64_t hash = hash_id(ob->state); + for (size_t probes = 0; probes < nbucket_count; probes++) { + const size_t nb_i = (hash + probes) & nmask; + struct eager_output_bucket *nb = &nbuckets[nb_i]; + if (nb->entry == NULL) { + nb->state = ob->state; + nb->entry = ob->entry; + break; + } else { + assert(nb->state != ob->state); + } + } + } + + f_free(alloc, htab->buckets); + htab->bucket_count = nbucket_count; + htab->buckets = nbuckets; + return true; +} + +int +fsm_seteageroutput(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) +{ + assert(fsm != NULL); + + struct eager_output_info *info = fsm->eager_output_info; + assert(info->htab.bucket_count > 0); + + if (info->htab.buckets_used >= info->htab.bucket_count/2) { + if (!grow_htab(fsm->alloc, &info->htab)) { return 0; } + } + + const uint64_t hash = hash_id(state); + const uint64_t mask = info->htab.bucket_count - 1; + assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ + + /* fprintf(stderr, "%s: bucket_count %zd\n", __func__, info->htab.bucket_count); */ + for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { + const size_t b_i = (hash + probes) & mask; + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + /* fprintf(stderr, "%s: state %d -> b_i %zd, state %d, entry %p\n", */ + /* __func__, state, b_i, b->state, (void *)b->entry); */ + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty */ + /* add */ + const size_t alloc_sz = sizeof(*e) + + DEF_ENTRY_CEIL * sizeof(e->ids[0]); + e = f_calloc(fsm->alloc, 1, alloc_sz); + if (e == NULL) { + return 0; + } + e->ceil = DEF_ENTRY_CEIL; + b->state = state; + b->entry = e; + info->htab.buckets_used++; + /* fprintf(stderr, "%s: buckets_used %zd\n", __func__, info->htab.buckets_used); */ + /* fprintf(stderr, "%s: saved new entry in bucket %zd\n", __func__, b_i); */ + } else if (b->state != state) { /* collision */ + continue; + } + + if (e->used == e->ceil) { + const size_t nceil = 2 * e->ceil; + const size_t nsize = sizeof(*e) + + nceil * sizeof(e->ids[0]); + struct eager_output_entry *nentry = f_realloc(fsm->alloc, e, nsize); + if (nentry == NULL) { return 0; } + nentry->ceil = nceil; + b->entry = nentry; + e = b->entry; + } + + /* ignore duplicates */ + for (size_t i = 0; i < e->used; i++) { + if (e->ids[i] == id) { return 1; } + } + + e->ids[e->used++] = id; + /* fprintf(stderr, "%s: e->ids_used %u\n", __func__, e->used); */ + fsm->states[state].has_eager_outputs = 1; + return 1; + } + + return 1; +} + +bool +fsm_eager_output_has_eager_output(const struct fsm *fsm) +{ + assert(fsm->eager_output_info != NULL); + const struct eager_output_htab *htab = &fsm->eager_output_info->htab; + + for (size_t b_i = 0; b_i < htab->bucket_count; b_i++) { + struct eager_output_bucket *b = &htab->buckets[b_i]; + if (b->entry == NULL) { continue; } + if (b->entry->used > 0) { return 1; } + } + return 0; +} + +bool +fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state) +{ + assert(state < fsm->statecount); + return fsm->states[state].has_eager_outputs; +} + +void +fsm_eager_output_iter_state(const struct fsm *fsm, + fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque) +{ + assert(fsm != NULL); + assert(cb != NULL); + + const uint64_t hash = hash_id(state); + + struct eager_output_info *info = fsm->eager_output_info; + const uint64_t mask = info->htab.bucket_count - 1; + assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ + + for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { + const size_t b_i = (hash + probes) & mask; + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + /* fprintf(stderr, "%s: state %d -> b_i %zd, state %d, entry %p\n", */ + /* __func__, state, b_i, b->state, (void *)b->entry); */ + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty */ + return; + } else if (b->state != state) { /* collision */ + continue; + } + + assert(e->used == 0 || fsm->states[state].has_eager_outputs); + + for (size_t i = 0; i < e->used; i++) { + if (!cb(state, e->ids[i], opaque)) { return; } + } + } +} + +void +fsm_eager_output_iter_all(const struct fsm *fsm, + fsm_eager_output_iter_cb *cb, void *opaque) +{ + assert(fsm != NULL); + assert(cb != NULL); + assert(fsm->eager_output_info != NULL); + + struct eager_output_info *info = fsm->eager_output_info; + + /* fprintf(stderr, "%s: bucket_count %zd\n", __func__, info->htab.bucket_count); */ + for (size_t b_i = 0; b_i < info->htab.bucket_count; b_i++) { + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + struct eager_output_entry *e = b->entry; + /* fprintf(stderr, "%s: b_i %zd, state %d, entry %p\n", */ + /* __func__, b_i, b->state, (void *)b->entry); */ + if (e == NULL) { /* empty */ + continue; + } + assert(e->used == 0 || fsm->states[b->state].has_eager_outputs); + + for (size_t i = 0; i < e->used; i++) { + if (!cb(b->state, e->ids[i], opaque)) { return; } + } + } +} + +struct dump_env { + FILE *f; + size_t count; +}; + +static int +dump_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) + +{ + struct dump_env *env = opaque; + fprintf(env->f, "-- %d: id %d\n", state, id); + env->count++; + return 1; +} + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm) +{ + struct dump_env env = { .f = f }; + fprintf(f, "%s:\n", __func__); + fsm_eager_output_iter_all(fsm, dump_cb, (void *)&env); + fprintf(f, "== %zu total\n", env.count); +} + +static int +inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + (void)id; + size_t *count = opaque; + (*count)++; + return 1; +} + +bool +fsm_eager_output_has_any(const struct fsm *fsm, + fsm_state_t state, size_t *count) +{ + size_t c = 0; + fsm_eager_output_iter_state(fsm, state, &inc_cb, &c); + if (count != NULL) { *count = c; } + return c > 0; +} + +int +fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count) +{ + /* Don't reallocate unless something has actually changed. */ + bool changes = false; + for (size_t i = 0; i < mapping_count; i++) { + if (mapping[i] != i) { + changes = true; + break; + } + } + + /* nothing to do */ + if (!changes) { return 1; } + + struct eager_output_info *eoi = fsm->eager_output_info; + + struct eager_output_bucket *nbuckets = f_calloc(fsm->alloc, + eoi->htab.bucket_count, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + + const uint64_t mask = eoi->htab.bucket_count - 1; + assert((eoi->htab.bucket_count & mask) == 0); + + for (size_t ob_i = 0; ob_i < eoi->htab.bucket_count; ob_i++) { + const struct eager_output_bucket *ob = &eoi->htab.buckets[ob_i]; + if (ob->entry == NULL) { continue; } + + assert(ob->state < mapping_count); + const fsm_state_t nstate = mapping[ob->state]; + if (nstate == FSM_STATE_REMAP_NO_STATE) { continue; } + + const uint64_t hash = hash_id(nstate); + + bool placed = false; + for (size_t probes = 0; probes < eoi->htab.bucket_count; probes++) { + const size_t nb_i = (hash + probes) & mask; + struct eager_output_bucket *nb = &nbuckets[nb_i]; + if (nb->entry == NULL) { + nb->state = nstate; + nb->entry = ob->entry; + placed = true; + break; + } + } + assert(placed); + } + + f_free(fsm->alloc, eoi->htab.buckets); + eoi->htab.buckets = nbuckets; + return 1; +} diff --git a/src/libfsm/eager_output.h b/src/libfsm/eager_output.h new file mode 100644 index 000000000..1b48ba4c4 --- /dev/null +++ b/src/libfsm/eager_output.h @@ -0,0 +1,46 @@ +#ifndef EAGER_OUTPUT_H +#define EAGER_OUTPUT_H + +#include +#include +#include + +struct eager_output_info; + +int +fsm_eager_output_init(struct fsm *fsm); + +void +fsm_eager_output_free(struct fsm *fsm); + +bool +fsm_eager_output_has_eager_output(const struct fsm *fsm); + +bool +fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state); + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +/* Callback for fsm_eager_output_iter_*. + * The return value indicates whether iteration should continue. + * The results may not be sorted in any particular order. */ +typedef int +fsm_eager_output_iter_cb(fsm_state_t state, fsm_output_id_t id, void *opaque); + +void +fsm_eager_output_iter_state(const struct fsm *fsm, + fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque); + +void +fsm_eager_output_iter_all(const struct fsm *fsm, + fsm_eager_output_iter_cb *cb, void *opaque); + +bool +fsm_eager_output_has_any(const struct fsm *fsm, + fsm_state_t state, size_t *count); + +int +fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); + +#endif diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 9394a2d9b..adfcdec2a 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -9,24 +9,42 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define DUMP_EPSILON_CLOSURES 0 #define DEF_PENDING_CAPTURE_ACTIONS_CEIL 2 #define LOG_RM_EPSILONS_CAPTURES 0 #define DEF_CARRY_ENDIDS_COUNT 2 +#define LOG_LEVEL 0 + +#if LOG_LEVEL > 0 +static bool log_it; +#define LOG(LVL, ...) \ + do { \ + if (log_it && LVL <= LOG_LEVEL) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while (0) +#else +#define LOG(_LVL, ...) +#endif + struct remap_env { #ifndef NDEBUG char tag; @@ -57,6 +75,49 @@ static int carry_endids(struct fsm *fsm, struct state_set *states, fsm_state_t s); +static void +mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label); + +struct eager_output_buf { +#define DEF_EAGER_OUTPUT_BUF_CEIL 8 + bool ok; + const struct fsm_alloc *alloc; + size_t ceil; + size_t used; + fsm_output_id_t *ids; +}; + +static bool +append_eager_output_id(struct eager_output_buf *buf, fsm_output_id_t id) +{ + if (buf->used == buf->ceil) { + const size_t nceil = buf->ceil == 0 ? DEF_EAGER_OUTPUT_BUF_CEIL : 2*buf->ceil; + fsm_output_id_t *nids = f_realloc(buf->alloc, buf->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + buf->ok = false; + return false; + } + buf->ids = nids; + buf->ceil = nceil; + } + + for (size_t i = 0; i < buf->used; i++) { + /* avoid duplicates */ + if (buf->ids[i] == id) { return true; } + } + + buf->ids[buf->used++] = id; + return true; +} + +static int +collect_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct eager_output_buf *buf = opaque; + return append_eager_output_id(buf, id) ? 1 : 0; +} + int fsm_remove_epsilons(struct fsm *nfa) { @@ -64,9 +125,20 @@ fsm_remove_epsilons(struct fsm *nfa) int res = 0; struct state_set **eclosures = NULL; fsm_state_t s; + struct eager_output_buf eager_output_buf = { + .ok = true, + .alloc = nfa->alloc, + }; + uint64_t *reachable_by_label = NULL; + + LOG(2, "%s: starting\n", __func__); INIT_TIMERS(); +#if LOG_LEVEL > 0 + log_it = getenv("LOG") != NULL; +#endif + assert(nfa != NULL); TIME(&pre); @@ -94,6 +166,17 @@ fsm_remove_epsilons(struct fsm *nfa) } #endif + const size_t state_words = u64bitset_words(state_count); + reachable_by_label = f_calloc(nfa->alloc, state_words, sizeof(reachable_by_label[0])); + if (reachable_by_label == NULL) { goto cleanup; } + + mark_states_reachable_by_label(nfa, reachable_by_label); + + fsm_state_t start; + if (!fsm_getstart(nfa, &start)) { + goto cleanup; /* no start state */ + } + for (s = 0; s < state_count; s++) { struct state_iter si; fsm_state_t es_id; @@ -101,6 +184,12 @@ fsm_remove_epsilons(struct fsm *nfa) struct edge_group_iter egi; struct edge_group_iter_info info; + /* If the state isn't reachable by a label and isn't the start state, + * skip processing -- it will soon become garbage. */ + if (!u64bitset_get(reachable_by_label, s) && s != start) { + continue; + } + /* Process the epsilon closure. */ state_set_reset(eclosures[s], &si); while (state_set_next(&si, &es_id)) { @@ -129,6 +218,16 @@ fsm_remove_epsilons(struct fsm *nfa) } } + /* Collect every eager output ID from any state + * in the current state's epsilon closure to the + * current state. These will be added at the end. */ + { + if (fsm_eager_output_has_any(nfa, es_id, NULL)) { + fsm_eager_output_iter_state(nfa, es_id, collect_eager_output_ids_cb, &eager_output_buf); + if (!eager_output_buf.ok) { goto cleanup; } + } + } + /* For every state in this state's transitive * epsilon closure, add all of their sets of * labeled edges. */ @@ -144,6 +243,13 @@ fsm_remove_epsilons(struct fsm *nfa) } } } + + for (size_t i = 0; i < eager_output_buf.used; i++) { + if (!fsm_seteageroutput(nfa, s, eager_output_buf.ids[i])) { + goto cleanup; + } + } + eager_output_buf.used = 0; /* clear */ } /* Remove the epsilon-edge state sets from everything. @@ -170,13 +276,53 @@ fsm_remove_epsilons(struct fsm *nfa) res = 1; cleanup: + LOG(2, "%s: finishing\n", __func__); if (eclosures != NULL) { closure_free(nfa, eclosures, state_count); } + f_free(nfa->alloc, reachable_by_label); + f_free(nfa->alloc, eager_output_buf.ids); return res; } +/* For every state, mark every state reached by a labeled edge as + * reachable. This doesn't check that the FROM state is reachable from + * the start state (trim will do that soon enough), it's just used to + * check which states will become unreachable once epsilon edges are + * removed. We don't need to add eager endids for them, because they + * will soon be disconnected from the epsilon-free NFA. */ +static void +mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label) +{ + fsm_state_t start; + if (!fsm_getstart(nfa, &start)) { + return; /* nothing reachable */ + } + u64bitset_set(reachable_by_label, start); + + const fsm_state_t state_count = fsm_countstates(nfa); + + for (size_t s_i = 0; s_i < state_count; s_i++) { + struct edge_group_iter egi; + struct edge_group_iter_info info; + + struct fsm_state *s = &nfa->states[s_i]; + + /* Clear the visited flag, it will be used to avoid cycles. */ +#if 1 + assert(s->visited == 0); /* stale */ +#endif + s->visited = 0; + + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + LOG(1, "%s: reachable: %d\n", __func__, info.to); + u64bitset_set(reachable_by_label, info.to); + } + } +} + static int remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) { @@ -425,4 +571,3 @@ carry_endids(struct fsm *fsm, struct state_set *states, return env.ok; } - diff --git a/src/libfsm/exec.c b/src/libfsm/exec.c index 9f7b21802..077494b8f 100644 --- a/src/libfsm/exec.c +++ b/src/libfsm/exec.c @@ -20,9 +20,12 @@ #include "internal.h" #include "capture.h" +#include "eager_output.h" #define LOG_EXEC 0 +#define LOG_EAGER 0 + static int transition(const struct fsm *fsm, fsm_state_t state, int c, size_t offset, struct fsm_capture *captures, @@ -43,6 +46,44 @@ transition(const struct fsm *fsm, fsm_state_t state, int c, return 1; } +struct check_eager_outputs_for_state_env { + const struct fsm *fsm; + fsm_eager_output_cb *cb; + void *opaque; +}; + +static int +match_eager_outputs_for_state_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +{ + /* HACK update the types here once it's working */ + (void)state; + struct check_eager_outputs_for_state_env *env = opaque; +#if LOG_EAGER + fprintf(stderr, "%s: state %d, id %d\n", __func__, state, id); +#endif + env->cb(id, env->opaque); + return 1; +} + +static int +match_eager_outputs_for_state(const struct fsm *fsm, fsm_state_t state) +{ + /* HACK update the types here once it's working */ + fsm_eager_output_cb *cb = NULL; + void *opaque = NULL; + fsm_eager_output_get_cb(fsm, &cb, &opaque); + if (cb == NULL) { return 1; } /* nothing to do */ + + struct check_eager_outputs_for_state_env env = { + .fsm = fsm, + .cb = cb, + .opaque = opaque, + }; + fsm_eager_output_iter_state(fsm, + state, match_eager_outputs_for_state_cb, &env); + return 1; +} + int fsm_exec(const struct fsm *fsm, int (*fsm_getc)(void *opaque), void *opaque, @@ -73,6 +114,7 @@ fsm_exec(const struct fsm *fsm, errno = EINVAL; return -1; } + const fsm_state_t start = state; for (i = 0; i < capture_count; i++) { captures[i].pos[0] = FSM_CAPTURE_NO_POS; @@ -83,6 +125,12 @@ fsm_exec(const struct fsm *fsm, fprintf(stderr, "fsm_exec: starting at %d\n", state); #endif + if (fsm->states[start].has_eager_outputs) { + if (!match_eager_outputs_for_state(fsm, start)) { + return 0; + } + } + while (c = fsm_getc(opaque), c != EOF) { if (!transition(fsm, state, c, offset, captures, &state)) { #if LOG_EXEC @@ -91,6 +139,12 @@ fsm_exec(const struct fsm *fsm, return 0; } + if (fsm->states[state].has_eager_outputs) { + if (!match_eager_outputs_for_state(fsm, state)) { + return 0; + } + } + #if LOG_EXEC fprintf(stderr, "fsm_exec: @ %zu, input '%c', new state %u\n", offset, c, state); @@ -113,4 +167,3 @@ fsm_exec(const struct fsm *fsm, *end = state; return 1; } - diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index ba2d2db26..c442c8262 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -21,6 +21,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" /* guess for default state allocation */ #define FSM_DEFAULT_STATEALLOC 128 @@ -39,6 +40,7 @@ free_contents(struct fsm *fsm) fsm_capture_free(fsm); fsm_endid_free(fsm); + fsm_eager_output_free(fsm); f_free(fsm->alloc, fsm->states); } @@ -92,6 +94,14 @@ fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc) return NULL; } + if (!fsm_eager_output_init(new)) { + f_free(new->alloc, new->states); + f_free(new->alloc, new); + fsm_capture_free(new); + fsm_endid_free(new); + return NULL; + } + return new; } @@ -133,6 +143,7 @@ fsm_move(struct fsm *dst, struct fsm *src) dst->capture_info = src->capture_info; dst->endid_info = src->endid_info; + dst->eager_output_info = src->eager_output_info; f_free(src->alloc, src); } diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index f84bbef0f..46997c82a 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -60,6 +60,10 @@ struct fsm_state { /* meaningful within one particular transformation only */ unsigned int visited:1; + + /* If 0, then this state has no need for checking + * the fsm->eager_output_info struct. */ + unsigned int has_eager_outputs:1; }; struct fsm { @@ -75,6 +79,7 @@ struct fsm { struct fsm_capture_info *capture_info; struct endid_info *endid_info; + struct eager_output_info *eager_output_info; }; struct fsm * diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 34be09e77..75c20eb64 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -2,6 +2,7 @@ fsm_complement fsm_union fsm_union_array +fsm_union_repeated_pattern_group fsm_intersect fsm_intersect_charset @@ -72,6 +73,8 @@ fsm_removestate fsm_shuffle fsm_vacuum +fsm_new_statealloc + fsm_addedge_any fsm_addedge_epsilon fsm_addedge_literal @@ -95,6 +98,14 @@ fsm_setendid fsm_mapendids fsm_increndids +fsm_endid_dump + +fsm_seteageroutput +fsm_seteageroutputonends +# short term hack +fsm_eager_output_set_cb +fsm_eager_output_dump + fsm_countedges fsm_countstates diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index 8c972c145..ccc1568ff 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -22,6 +22,7 @@ #include "capture.h" #include "internal.h" #include "endids.h" +#include "eager_output.h" #define LOG_MERGE_ENDIDS 0 @@ -39,6 +40,9 @@ copy_capture_actions(struct fsm *dst, struct fsm *src); static int copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); +static int +copy_eager_output_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); + static struct fsm * merge(struct fsm *dst, struct fsm *src, fsm_state_t *base_dst, fsm_state_t *base_src, @@ -113,6 +117,11 @@ merge(struct fsm *dst, struct fsm *src, return NULL; } + if (!copy_eager_output_ids(dst, src, *base_src)) { + /* non-recoverable -- destructive operation */ + return NULL; + } + f_free(src->alloc, src->states); src->states = NULL; src->statealloc = 0; @@ -194,6 +203,39 @@ copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) return fsm_endid_iter_bulk(src, copy_end_ids_cb, &env); } +struct copy_eager_output_ids_env { + bool ok; + struct fsm *dst; + struct fsm *src; + fsm_state_t base_src; +}; + +static int +copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct copy_eager_output_ids_env *env = opaque; + if (!fsm_seteageroutput(env->dst, state + env->base_src, id)) { + env->ok = false; + return 0; + } + + return 1; + +} + +static int +copy_eager_output_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) +{ + struct copy_eager_output_ids_env env = { + .ok = true, + .dst = dst, + .src = src, + .base_src = base_src, + }; + fsm_eager_output_iter_all(src, copy_eager_output_ids_cb, &env); + return env.ok; +} + struct fsm * fsm_mergeab(struct fsm *a, struct fsm *b, fsm_state_t *base_b) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index a8d53c57e..86f00b46f 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -25,6 +25,8 @@ #include "internal.h" #include "capture.h" +#include "eager_output.h" +#include "endids.h" #define LOG_MAPPINGS 0 #define LOG_STEPS 0 @@ -54,12 +56,21 @@ struct end_metadata { unsigned count; fsm_end_id_t *ids; } end; + + struct end_metadata_eager_outputs { + unsigned count; + fsm_output_id_t *ids; + } eager_outputs; }; static int collect_end_ids(const struct fsm *fsm, fsm_state_t s, struct end_metadata_end *e); +static int +collect_eager_output_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_eager_outputs *e); + int fsm_minimise(struct fsm *fsm) { @@ -122,6 +133,10 @@ fsm_minimise(struct fsm *fsm) /* Minimisation should never add states. */ assert(minimised_states <= orig_states); + for (size_t i = 0; i < fsm->statecount; i++) { + assert(mapping[i] < fsm->statecount); + } + /* Use the mapping to consolidate the current states * into a new DFA, combining states that could not be * proven distinguishable. */ @@ -693,6 +708,9 @@ same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) if (a->end.count != b->end.count) { return 0; } + if (a->eager_outputs.count != b->eager_outputs.count) { + return 0; + } /* compare -- these must be sorted */ @@ -702,6 +720,12 @@ same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) } } + for (size_t i = 0; i < a->eager_outputs.count; i++) { + if (a->eager_outputs.ids[i] != b->eager_outputs.ids[i]) { + return 0; + } + } + return 1; } @@ -750,14 +774,21 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) #endif while (s != NO_ID) { struct end_metadata *e = &end_md[s]; - if (!fsm_isend(fsm, s)) { - break; /* this EC has non-end states, skip */ + const bool is_end = fsm_isend(fsm, s); + const bool has_eager_outputs = fsm_eager_output_state_has_eager_output(fsm, s); + + if (!is_end && !has_eager_outputs) { + break; /* skip */ } if (!collect_end_ids(fsm, s, &e->end)) { goto cleanup; } + if (!collect_eager_output_ids(fsm, s, &e->eager_outputs)) { + goto cleanup; + } + s = env->jump[s]; } } @@ -789,6 +820,10 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) incremental_hash_of_ids(&hash, s_md->end.ids[eid_i]); } + for (size_t eo_i = 0; eo_i < s_md->eager_outputs.count; eo_i++) { + incremental_hash_of_ids(&hash, s_md->eager_outputs.ids[eo_i]); + } + for (size_t b_i = 0; b_i < bucket_count; b_i++) { fsm_state_t *b = &htab[(b_i + hash) & mask]; const fsm_state_t other = *b; @@ -932,6 +967,9 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) if (e->end.ids != NULL) { f_free(fsm->alloc, e->end.ids); } + if (e->eager_outputs.ids != NULL) { + f_free(fsm->alloc, e->eager_outputs.ids); + } } f_free(fsm->alloc, end_md); } @@ -959,7 +997,7 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, #if LOG_ECS fprintf(stderr, "%d:", s); - for (size_t i = 0; i < written; i++) { + for (size_t i = 0; i < e->count; i++) { fprintf(stderr, " %u", e->ids[i]); } fprintf(stderr, "\n"); @@ -968,6 +1006,41 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, return 1; } +static int +collect_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct end_metadata_eager_outputs *e = opaque; + e->ids[e->count++] = id; + return 1; +} + +static int cmp_eager_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static int +collect_eager_output_ids(const struct fsm *fsm, fsm_state_t state, + struct end_metadata_eager_outputs *e) +{ + size_t count = 0; + if (!fsm_eager_output_has_any(fsm, state, &count)) { + return 1; /* nothing to do */ + } + + e->ids = f_malloc(fsm->alloc, count * sizeof(e->ids[0])); + if (e->ids == NULL) { return 0; } + + fsm_eager_output_iter_state(fsm, state, collect_cb, e); + + /* sort, to normalize set */ + qsort(e->ids, e->count, sizeof(e->ids[0]), cmp_eager_output_id); + return 1; +} + #if EXPENSIVE_CHECKS static void check_done_ec_offset(const struct min_env *env) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 22b03963e..cc3927dc6 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -222,6 +222,14 @@ print_case(FILE *f, const struct ir *ir, assert(f != NULL); assert(cs != NULL); + if (cs->eager_outputs != NULL && opt->fragment) { + /* If .fragment is set and the state has eager outputs, then emit a call to a + * macro (the caller is expected to define). This is a temporary interface. */ + for (size_t i = 0; i < cs->eager_outputs->count; i++) { + fprintf(f, "\t\t\tFSM_SET_EAGER_OUTPUT(%u);\n", cs->eager_outputs->ids[i]); + } + } + switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); @@ -377,6 +385,11 @@ print_endstates(FILE *f, const struct fsm_state_metadata state_metadata = { .end_ids = ir->states[i].endids.ids, .end_id_count = ir->states[i].endids.count, + + .eager_output_count = (ir->states[i].eager_outputs == NULL + ? 0 : ir->states[i].eager_outputs->count), + .eager_output_ids = (ir->states[i].eager_outputs == NULL + ? NULL : ir->states[i].eager_outputs->ids), }; if (-1 == print_hook_accept(f, opt, hooks, diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 457716dcc..81d5890e0 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -26,6 +26,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/eager_output.h" #include "ir.h" @@ -505,6 +506,23 @@ make_example(const struct fsm *fsm, fsm_state_t s, char **example) return 0; } +static int +append_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct ir_state_eager_output *outputs = opaque; + (void)state; + outputs->ids[outputs->count++] = id; + return 1; +} + +static int +cmp_fsm_output_id_t(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + struct ir * make_ir(const struct fsm *fsm, const struct fsm_options *opt) { @@ -544,6 +562,8 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) ir->states[i].endids.ids = NULL; ir->states[i].endids.count = 0; + ir->states[i].eager_outputs = NULL; + if (fsm_isend(fsm, i)) { fsm_end_id_t *ids; size_t count; @@ -567,6 +587,20 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) ir->states[i].endids.count = count; } + size_t count; + if (fsm_eager_output_has_any(fsm, i, &count)) { + struct ir_state_eager_output *outputs = f_malloc(fsm->alloc, + sizeof(*outputs) + count * sizeof(outputs->ids[0])); + if (outputs == NULL) { + goto error; + } + outputs->count = 0; + fsm_eager_output_iter_state(fsm, i, append_eager_output_cb, outputs); + assert(outputs->count == count); + qsort(outputs->ids, outputs->count, sizeof(outputs->ids[0]), cmp_fsm_output_id_t); + ir->states[i].eager_outputs = outputs; + } + if (make_state(fsm, i, &ir->states[i]) == -1) { goto error; } @@ -630,6 +664,7 @@ free_ir(const struct fsm *fsm, struct ir *ir) for (i = 0; i < ir->n; i++) { f_free(fsm->alloc, (void *) ir->states[i].example); f_free(fsm->alloc, (void *) ir->states[i].endids.ids); + f_free(fsm->alloc, (void *) ir->states[i].eager_outputs); switch (ir->states[i].strategy) { case IR_TABLE: diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index b375ba850..7678d3f35 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -59,6 +59,11 @@ struct ir_state { size_t count; } endids; + struct ir_state_eager_output { + size_t count; + fsm_output_id_t ids[]; + } *eager_outputs; /* NULL -> 0 */ + unsigned int isend:1; enum ir_strategy strategy; diff --git a/src/libfsm/state.c b/src/libfsm/state.c index c845cbe46..d96c33653 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -19,6 +19,7 @@ #include "internal.h" #include "endids.h" +#include "eager_output.h" int fsm_addstate(struct fsm *fsm, fsm_state_t *state) @@ -44,6 +45,7 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) for (i = fsm->statealloc; i < n; i++) { tmp[i].has_capture_actions = 0; + tmp[i].has_eager_outputs = 0; } fsm->statealloc = n; @@ -87,6 +89,8 @@ fsm_addstate_bulk(struct fsm *fsm, size_t n) new->visited = 0; new->epsilons = NULL; new->edges = NULL; + + new->has_eager_outputs = 0; } fsm->statecount += n; @@ -259,6 +263,10 @@ fsm_compact_states(struct fsm *fsm, if (!fsm_endid_compact(fsm, mapping, orig_statecount)) { return 0; } + if (!fsm_eager_output_compact(fsm, mapping, orig_statecount)) { + return 0; + } + assert(dst == kept); assert(kept == fsm->statecount); diff --git a/src/libfsm/union.c b/src/libfsm/union.c index a3b4b230c..0b18cd30c 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -15,9 +15,14 @@ #include #include #include +#include +#include #include "internal.h" +#include +#include "eager_output.h" + #define LOG_UNION_ARRAY 0 struct fsm * @@ -151,3 +156,231 @@ fsm_union_array(size_t fsm_count, return res; } + +#define LOG_UNION_REPEATED_PATTERN_GROUP 0 + +/* Combine an array of FSMs into a single FSM in one pass, with an extra loop + * so that more than one pattern with eager outputs can match. */ +struct fsm * +fsm_union_repeated_pattern_group(size_t entry_count, + struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases) +{ + const struct fsm_alloc *alloc = entries[0].fsm->alloc; + const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; + + if (entry_count == 1) { + return entries[0].fsm; + } + + size_t est_total_states = 0; + for (size_t i = 0; i < entry_count; i++) { + assert(entries[i].fsm); + if (entries[i].fsm->alloc != alloc) { + errno = EINVAL; + return NULL; + } + const size_t count = fsm_countstates(entries[i].fsm); + est_total_states += count; + } + + est_total_states += 5; /* new start and end, new unanchored start and end loops */ + + struct fsm *res = fsm_new_statealloc(alloc, est_total_states); + if (res == NULL) { return NULL; } + + /* collected end states */ + struct ends_buf { + size_t ceil; + size_t used; + fsm_state_t *states; + } ends = { .ceil = 0 }; + + /* The new overall start state, which will have an epsilon edge to... */ + fsm_state_t global_start; + if (!fsm_addstate(res, &global_start)) { goto fail; } + + /* states linking to the starts of unanchored and anchored subgraphs, respectively. */ + fsm_state_t global_start_loop, global_start_anchored; + if (!fsm_addstate(res, &global_start_loop)) { goto fail; } + if (!fsm_addstate(res, &global_start_anchored)) { goto fail; } + + /* The unanchored end loop state, and an end state with no outgoing edges. */ + fsm_state_t global_end_loop, global_end; + if (!fsm_addstate(res, &global_end)) { goto fail; } + if (!fsm_addstate(res, &global_end_loop)) { goto fail; } + + /* link the start to the start loop and anchored start, and the start loop to itself */ + if (log) { + fprintf(stderr, "link_before: global_start %d -> global_start_loop %d and global_start_anchored %d\n", + global_start, global_start_loop, global_start_anchored); + } + if (!fsm_addedge_epsilon(res, global_start, global_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_start_anchored)) { goto fail; } + if (!fsm_addedge_any(res, global_start_loop, global_start_loop)) { goto fail; } + + /* link the end loop and end */ + if (log) { + fprintf(stderr, "link_before: global_end_loop %d -> global_end %d (and -> self)\n", global_end_loop, global_end); + } + if (!fsm_addedge_epsilon(res, global_end_loop, global_end)) { goto fail; } + if (!fsm_addedge_any(res, global_end_loop, global_end_loop)) { goto fail; } + + if (bases != NULL) { + memset(bases, 0x00, entry_count * sizeof(bases[0])); + } + + for (size_t fsm_i = 0; fsm_i < entry_count; fsm_i++) { + ends.used = 0; /* reset */ + + struct fsm *fsm = entries[fsm_i].fsm; + entries[fsm_i].fsm = NULL; /* transfer ownership */ + + const size_t state_count = fsm_countstates(fsm); + + fsm_state_t fsm_start; + if (!fsm_getstart(fsm, &fsm_start)) { + fsm_free(fsm); /* no start, just discard */ + continue; + } + + for (fsm_state_t s_i = 0; s_i < state_count; s_i++) { + if (fsm_isend(fsm, s_i)) { + if (ends.used == ends.ceil) { /* grow? */ + size_t nceil = (ends.ceil == 0 ? 4 : 2*ends.ceil); + fsm_state_t *nstates = f_realloc(alloc, + ends.states, nceil * sizeof(nstates[0])); + if (nstates == NULL) { goto fail; } + ends.ceil = nceil; + ends.states = nstates; + } + ends.states[ends.used++] = s_i; + } + } + + if (ends.used == 0) { + fsm_free(fsm); /* no ends, just discard */ + continue; + } + + /* When combining these, remove self-edges from any states on the FSMs to be + * combined that also have eager output IDs. We are about to add an epsilon edge + * from each to a shared state that won't have eager output IDs. + * + * Eager output matching should be idempotent, so carrying it to other reachable + * state is redundant, and it leads to a combinatorial explosion that blows up the + * state count while determinising the combined FSM otherwise. + * + * For example, if /aaa/, /bbb/, and /ccc/ are combined into a DFA that repeats + * the sub-patterns (like `^.*(?:(aaa)|(bbb)|(ccc))+.*$`), the self-edge at each + * eager output state would combine with every reachable state from then on, + * leading to a copy of the whole reachable subgraph colored by every + * combination of eager output IDs: aaa, bbb, ccc, aaa+bbb, aaa+ccc, + * bbb+ccc, aaa+bbb+ccc. Instead of three relatively separate subgraphs + * that set the eager output at their last state, one for each pattern, + * it leads to 8 (2**3) subgraph clusters because it encodes _each + * distinct combination_ in the DFA. This becomes incredibly expensive + * as the combined pattern count increases; it's essentially what I'm + * trying to avoid by adding eager output support in the first place. + * + * FIXME: instead of actively removing these, filter in fsm_determinise? */ + if (fsm_eager_output_has_eager_output(fsm)) { + /* for any state that has eager outputs and a self edge, + * remove the self edge before further linkage */ + for (fsm_state_t s = 0; s < fsm->statecount; s++) { + if (!fsm_eager_output_has_any(fsm, s, NULL)) { continue; } + struct edge_set *edges = fsm->states[s].edges; + struct edge_set *new = edge_set_new(); + + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (info.to != s) { + if (!edge_set_add_bulk(&new, fsm->alloc, + info.symbols, info.to)) { + goto fail; + } + } + } + edge_set_free(fsm->alloc, edges); + fsm->states[s].edges = new; + } + } + + /* call fsm_merge; we really don't care which is which */ + struct fsm_combine_info combine_info; + struct fsm *merged = fsm_merge(res, fsm, &combine_info); + if (merged == NULL) { goto fail; } + + /* update offsets if res had its state IDs shifted forward */ + global_start += combine_info.base_a; + global_start_loop += combine_info.base_a; + global_start_anchored += combine_info.base_a;; + global_end += combine_info.base_a; + global_end_loop += combine_info.base_a; + + /* also update offsets for the FSM's states */ + fsm_start += combine_info.base_b; + for (size_t i = 0; i < ends.used; i++) { + ends.states[i] += combine_info.base_b; + } + + if (bases != NULL) { + bases[fsm_i].state = combine_info.base_b; + bases[fsm_i].capture = combine_info.capture_base_b; + } + + if (log) { + fprintf(stderr, "%s: fsm[%zd].start: %d\n", __func__, fsm_i, fsm_start); + for (size_t i = 0; i < ends.used; i++) { + fprintf(stderr, "%s: fsm[%zd].ends[%zd]: %d\n", __func__, fsm_i, i, ends.states[i]); + } + } + + /* link to the FSM's start state */ + const fsm_state_t start_src = entries[fsm_i].anchored_start ? global_start_anchored : global_start_loop; + if (!fsm_addedge_epsilon(merged, start_src, fsm_start)) { goto fail; } + if (log) { + fprintf(stderr, "%s: linking %s %d to fsm[%zd]'s start %d (anchored? %d)\n", + __func__, + entries[fsm_i].anchored_start ? "global_start_anchored" : "global_start_loop", + start_src, fsm_i, fsm_start, entries[fsm_i].anchored_start); + } + + /* link from the FSM's ends */ + const fsm_state_t end_dst = entries[fsm_i].anchored_end ? global_end : global_end_loop; + for (size_t i = 0; i < ends.used; i++) { + if (log) { + fprintf(stderr, "%s: linking fsm[%zd]'s end[%zd] %d (anchored? %d) to %s %d\n", + __func__, fsm_i, i, ends.states[i], entries[fsm_i].anchored_end, + entries[fsm_i].anchored_end ? "global_end" : "global_end_loop", + end_dst); + } + if (!fsm_addedge_epsilon(merged, ends.states[i], end_dst)) { goto fail; } + } + + res = merged; + } + + /* Link from the global_end_loop to the global_start_loop, so patterns with an + * unanchored start can follow other patterns with an unanchored end. */ + if (log) { + fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", + __func__, global_start, global_start_loop, global_start_anchored, global_end_loop, global_end); + fprintf(stderr, "%s: linking global_end_loop %d to global_start_loop %d\n", + __func__, global_end_loop, global_start_loop); + fprintf(stderr, "%s: setting global_start %d and end %d\n", __func__, global_start, global_end); + } + if (!fsm_addedge_epsilon(res, global_end_loop, global_start_loop)) { goto fail; } + + /* This needs to be set after merging, because that clears the start state. */ + fsm_setstart(res, global_start); + fsm_setend(res, global_end, 1); + + f_free(alloc, ends.states); + return res; + +fail: + f_free(alloc, ends.states); + return NULL; +} diff --git a/src/libre/libre.syms b/src/libre/libre.syms index a4f1a223b..9d381cb0f 100644 --- a/src/libre/libre.syms +++ b/src/libre/libre.syms @@ -3,6 +3,7 @@ re_is_literal re_flags re_strerror re_perror +re_is_anchored ast_print ast_print_dot diff --git a/src/libre/re.c b/src/libre/re.c index 15af848b5..c19183dcc 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -335,3 +335,37 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, return -1; } +/* FIXME: placeholder interface */ +int +re_is_anchored(enum re_dialect dialect, re_getchar_fun *getc, void *opaque, + enum re_flags flags, struct re_err *err, + struct re_anchoring_info *info) +{ + /* FIXME: copy/pasted from above, factor out common */ + + struct ast *ast; + const struct dialect *m; + int unsatisfiable; + + assert(getc != NULL); + assert(info != NULL); + + m = re_dialect(dialect); + if (m == NULL) { + if (err != NULL) { err->e = RE_EBADDIALECT; } + return 0; + } + + flags |= m->flags; + + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); + if (ast == NULL) { + return 0; + } + + info->start = (ast->expr->flags & AST_FLAG_ANCHORED_START) != 0; + info->end = (ast->expr->flags & AST_FLAG_ANCHORED_END) != 0; + + ast_free(ast); + return 1; +} diff --git a/tests/eager_output/Makefile b/tests/eager_output/Makefile new file mode 100644 index 000000000..a650bf802 --- /dev/null +++ b/tests/eager_output/Makefile @@ -0,0 +1,22 @@ +.include "../../share/mk/top.mk" + +TEST.tests/eager_output != ls -1 tests/eager_output/eager_output*.c +TEST_SRCDIR.tests/eager_output = tests/eager_output +TEST_OUTDIR.tests/eager_output = ${BUILD}/tests/eager_output + +.for n in ${TEST.tests/eager_output:T:R:C/^eager_output//} +INCDIR.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c += src/adt +.endfor + +SRC += ${TEST_SRCDIR.tests/eager_output}/utils.c + +.for n in ${TEST.tests/eager_output:T:R:C/^eager_output//} +test:: ${TEST_OUTDIR.tests/eager_output}/res${n} +SRC += ${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c +CFLAGS.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c += -UNDEBUG + +${TEST_OUTDIR.tests/eager_output}/run${n}: ${TEST_OUTDIR.tests/eager_output}/eager_output${n}.o ${TEST_OUTDIR.tests/eager_output}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c} -o ${TEST_OUTDIR.tests/eager_output}/run${n} ${TEST_OUTDIR.tests/eager_output}/eager_output${n}.o ${TEST_OUTDIR.tests/eager_output}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a +${TEST_OUTDIR.tests/eager_output}/res${n}: ${TEST_OUTDIR.tests/eager_output}/run${n} + ( ${TEST_OUTDIR.tests/eager_output}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/eager_output}/res${n} +.endfor diff --git a/tests/eager_output/eager_output1.c b/tests/eager_output/eager_output1.c new file mode 100644 index 000000000..f20ef77b7 --- /dev/null +++ b/tests/eager_output/eager_output1.c @@ -0,0 +1,12 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "abc" }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output2.c b/tests/eager_output/eager_output2.c new file mode 100644 index 000000000..cdac204e2 --- /dev/null +++ b/tests/eager_output/eager_output2.c @@ -0,0 +1,17 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab(c|d|e)" }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "abd", .expected_ids = { 1 } }, + { .input = "abe", .expected_ids = { 1 } }, + { .input = "Xabe", .expected_ids = { 1 } }, + { .input = "abeX", .expected_ids = { 1 } }, + { .input = "XabeX", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output3.c b/tests/eager_output/eager_output3.c new file mode 100644 index 000000000..c11bc58a4 --- /dev/null +++ b/tests/eager_output/eager_output3.c @@ -0,0 +1,16 @@ +#include "utils.h" + +/* test that eager endids are correctly propagated through fsm_determinise() and fsm_minimise() */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab(c|d|e)?" }, + .inputs = { + { .input = "ab", .expected_ids = { 1 } }, + { .input = "abc", .expected_ids = { 1 } }, + { .input = "abd", .expected_ids = { 1 } }, + { .input = "abe", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output4.c b/tests/eager_output/eager_output4.c new file mode 100644 index 000000000..47cd32029 --- /dev/null +++ b/tests/eager_output/eager_output4.c @@ -0,0 +1,13 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "abcde$" }, + .inputs = { + { .input = "abcde", .expected_ids = { 1 } }, + { .input = "Xabcde", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output5.c b/tests/eager_output/eager_output5.c new file mode 100644 index 000000000..4551c68b1 --- /dev/null +++ b/tests/eager_output/eager_output5.c @@ -0,0 +1,14 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "^abc$", "^ab*c$" }, + .inputs = { + { .input = "ac", .expected_ids = { 2 } }, + { .input = "abc", .expected_ids = { 1, 2 } }, + { .input = "abbc", .expected_ids = { 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output6.c b/tests/eager_output/eager_output6.c new file mode 100644 index 000000000..5431d0981 --- /dev/null +++ b/tests/eager_output/eager_output6.c @@ -0,0 +1,34 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "apple", + "banana", + "carrot", + "durian", + "eggplant", + "fig", + "grapefruit", + "hazelnut", + "iceberg lettuce", + "jicama", + }, + .inputs = { + { .input = "apple", .expected_ids = { 1 } }, + { .input = "banana", .expected_ids = { 2 } }, + { .input = "carrot", .expected_ids = { 3 } }, + { .input = "durian", .expected_ids = { 4 } }, + { .input = "eggplant", .expected_ids = { 5 } }, + { .input = "fig", .expected_ids = { 6 } }, + { .input = "grapefruit", .expected_ids = { 7 } }, + { .input = "hazelnut", .expected_ids = { 8 } }, + { .input = "iceberg lettuce", .expected_ids = { 9 } }, + { .input = "jicama", .expected_ids = { 10 } }, + { .input = "apple banana carrot", .expected_ids = { 1, 2, 3 } }, + }, + }; + + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output7.c b/tests/eager_output/eager_output7.c new file mode 100644 index 000000000..3d123878b --- /dev/null +++ b/tests/eager_output/eager_output7.c @@ -0,0 +1,103 @@ +#include "utils.h" + +int main(void) +{ + /* Run this test with env FORCE_ENDIDS=N ... to see how much more + * expensive it is to combine the first N patterns using endids, + * rather than eager_outputs. It becomes VERY slow for >= 9 or so. + * (Note that the checks probably will not pass for N < 4, because + * it will start skipping appear in the early test inputs.) */ + bool force_endids = false; + size_t force_endid_count = 0; + { + const char *str = getenv("FORCE_ENDIDS"); + if (str != NULL) { + force_endid_count = atoi(str); + if (force_endid_count == 0) { + force_endid_count = 26; + } + force_endids = true; + } + } + + struct eager_output_test test = { + .patterns = { + [0] = "apple", + [1] = "banana", + [2] = "carrot", + [3] = "durian", + [4] = "eggplant", + [5] = "fig", + [6] = "grapefruit", + [7] = "hazelnut", + [8] = "iceberg lettuce", + [9] = "jicama", + [10] = "kiwano", + [11] = "lemon", + [12] = "mango", + [13] = "nectarine", + [14] = "orange", + [15] = "plum", + [16] = "quince", + [17] = "radish", + [18] = "strawberry", + [19] = "turnip", + [20] = "ube", + [21] = "vanilla", + [22] = "watermelon", + [23] = "xigua watermelon", + [24] = "yam", + [25] = "zucchini", + }, + .inputs = { + /* Note: expected IDs are shifted by 1, it's 0-terminated. */ + { .input = "apple", .expected_ids = { 1 } }, + { .input = "banana", .expected_ids = { 2 } }, + { .input = "carrot", .expected_ids = { 3 } }, + { .input = "apple banana", .expected_ids = { 1, 2 } }, + { .input = "carrot durian apple", .expected_ids = { 1, 3, 4 } }, + { .input = "carrot fig apple", .expected_ids = { 1, 3, 6 } }, + + /* leading characters and an incomplete trailing match */ + { .input = "mumble mumble fig hazelnut banana xigua watermelo", .expected_ids = { 2, 6, 8 } }, + + /* redundant matches */ + { .input = "ube ube ube ube ube", .expected_ids = { 21 } }, + + /* everything */ + { .input = + "apple banana carrot durian eggplant fig grapefruit " + "hazelnut iceberg lettuce jicamaa kiwano lemon mango " + "nectarine orange plum quince radish strawberry " + "turnip ube vanilla watermelon xigua watermelon yam zucchini", + .expected_ids = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + }, + }, + /* everything, only spaces appearing in patterns */ + { .input = + "applebananacarrotdurianeggplantfiggrapefruit" + "hazelnuticeberg lettucejicamaakiwanolemonmango" + "nectarineorangeplumquinceradishstrawberry" + "turnipubevanillawatermelonxigua watermelonyamzucchini", + .expected_ids = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + }, + }, + }, + }; + + /* truncate patterns to the first N */ + if (force_endids) { + assert(force_endid_count > 0 && force_endid_count <= 26); + test.patterns[force_endid_count] = NULL; + + /* truncate test inputs to just the first couple, since + * later inputs use later patterns */ + test.inputs[5].input = NULL; + } + + return run_test(&test, false, force_endids); +} diff --git a/tests/eager_output/eager_output_at_start.c b/tests/eager_output/eager_output_at_start.c new file mode 100644 index 000000000..407aa4e77 --- /dev/null +++ b/tests/eager_output/eager_output_at_start.c @@ -0,0 +1,12 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_fr1.c b/tests/eager_output/eager_output_fr1.c new file mode 100644 index 000000000..e8e5f3395 --- /dev/null +++ b/tests/eager_output/eager_output_fr1.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab", "" }, + .inputs = { + { .input = "ab", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_fr2.c b/tests/eager_output/eager_output_fr2.c new file mode 100644 index 000000000..404e98644 --- /dev/null +++ b/tests/eager_output/eager_output_fr2.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "", "" }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_fr3.c b/tests/eager_output/eager_output_fr3.c new file mode 100644 index 000000000..c7e4127a6 --- /dev/null +++ b/tests/eager_output/eager_output_fr3.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "^", "" }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_mixed_anchored_unanchored.c b/tests/eager_output/eager_output_mixed_anchored_unanchored.c new file mode 100644 index 000000000..a586f9840 --- /dev/null +++ b/tests/eager_output/eager_output_mixed_anchored_unanchored.c @@ -0,0 +1,46 @@ +#include "utils.h" + +int main(void) +{ + /* fprintf(stderr, "%s: skipping for now, this doesn't pass yet.\n", __FILE__); */ + /* return EXIT_SUCCESS; */ + + struct eager_output_test test = { + .patterns = { + "^abc$", + "def", + "^ghi", + "jkl$", + "mno", + }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "def", .expected_ids = { 2 } }, + { .input = "ghi", .expected_ids = { 3 } }, + { .input = "jkl", .expected_ids = { 4 } }, + { .input = "mno", .expected_ids = { 5 } }, + + { .input = "defmno", .expected_ids = { 2, 5 } }, + { .input = " def mno ", .expected_ids = { 2, 5 } }, + + /* Matching a start-anchored pattern followed by + * unanchored ones should just work. */ + { .input = "ghi def", .expected_ids = { 2, 3 } }, + + /* An unanchored pattern before a start-anchored pattern + * should only match the unanchored pattern. */ + { .input = "def ghi", .expected_ids = { 2 } }, + + /* Matching an unanchored pattern before an + * end-anchored one is fine. */ + { .input = "mno jkl", .expected_ids = { 4, 5 } }, + + /* This should match "mno" with the "jkl" prefix + * ignored by the unanchored start, which does + * not count as a match for "jkl$". */ + { .input = "jkl mno", .expected_ids = { 5 } }, + }, + }; + + return run_test(&test, false, false); +} diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c new file mode 100644 index 000000000..4bee8d848 --- /dev/null +++ b/tests/eager_output/utils.c @@ -0,0 +1,278 @@ +#include "utils.h" + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +void +fsm_endid_dump(FILE *f, const struct fsm *fsm); + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque) +{ + struct cb_info *info = (struct cb_info *)opaque; + assert(info->used < MAX_IDS); + + for (size_t i = 0; i < info->used; i++) { + if (info->ids[i] == id) { + return; /* already present */ + } + } + + info->ids[info->used++] = id; +} + +int +cmp_output(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +struct fsm_options print_options = { + .consolidate_edges = 1, + .comments = 0, + .group_edges = 1, +}; + +void +dump(const struct fsm *fsm) +{ + fsm_print(stderr, fsm, + &print_options, NULL, FSM_PRINT_DOT); +} + +int +run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids) +{ + struct fsm_union_entry entries[MAX_PATTERNS] = {0}; + + allow_extra_outputs = false; + + size_t fsms_used = 0; + int ret = 0; + + int log = 0; + { + const char *logstr = getenv("LOG"); + if (logstr != NULL) { + if (logstr[0] == 'y') { /* make "y" or "yes" non-zero */ + logstr = "1"; + } + log = atoi(logstr); + } + } + + for (size_t i = 0; i < MAX_PATTERNS; i++) { + const char *p = test->patterns[i]; + if (test->patterns[i] == NULL) { break; } + const size_t len = strlen(p); + struct fsm_union_entry *e = &entries[fsms_used]; + + /* For sake of these patterns, they are anchored if the first/last + * character is '^' and '$', respectively. This is too simplistic + * for the general case, though. */ + if (len > 0) { + if (p[0] == '^') { e->anchored_start = true; } + if (p[len - 1] == '$') { e->anchored_end = true; } + /* fprintf(stderr, "%s: p[%zd]: '%s', start %d, end %d\n", */ + /* __func__, fsms_used, p, e->anchored_start, e->anchored_end); */ + } + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); + assert(fsm != NULL); + + /* Zero is used to terminate expected_ids, so don't use it here. */ + const fsm_output_id_t output_id = (fsm_output_id_t) (i + 1); + const fsm_end_id_t end_id = (fsm_end_id_t) (i + 1); + + /* Set either an end ID or an eager output ID, depending on + * whether the fsm is anchored at the end or not. */ + if (e->anchored_end || force_endids) { + ret = fsm_setendid(fsm, end_id); + } else { + ret = fsm_seteageroutputonends(fsm, output_id); + } + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== source DFA %zd (pre det+min)\n", i); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + ret = fsm_determinise(fsm); + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== source DFA %zd (post det)\n", i); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + ret = fsm_minimise(fsm); + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== source DFA %zd (post det+min)\n", i); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + e->fsm = fsm; + fsms_used++; + } + + /* If there's only one pattern this just returns fsms[0]. */ + struct fsm *fsm = fsm_union_repeated_pattern_group(fsms_used, entries, NULL); + assert(fsm != NULL); + + if (log) { + fprintf(stderr, "==== combined (pre det+min)\n"); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + if (log) { + fprintf(stderr, "=== determinising combined... NFA has %u states\n", fsm_countstates(fsm)); + } + ret = fsm_determinise(fsm); + assert(ret == 1); + if (log) { + fprintf(stderr, "=== determinising combined...done, DFA has %u states\n", fsm_countstates(fsm)); + } + + if (log) { + fprintf(stderr, "==== combined (post det)\n"); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + ret = fsm_minimise(fsm); + if (log) { + fprintf(stderr, "=== minimised combined...done, DFA has %u states\n", fsm_countstates(fsm)); + } + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== combined (post det+min)\n"); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + struct cb_info outputs = { 0 }; + fsm_eager_output_set_cb(fsm, append_eager_output_cb, &outputs); + + for (size_t i_i = 0; i_i < MAX_INPUTS; i_i++) { + outputs.used = 0; + const char *input = test->inputs[i_i].input; + if (input == NULL) { break; } + + size_t expected_id_count = 0; + for (size_t id_i = 0; id_i < MAX_ENDIDS; id_i++) { + const fsm_output_id_t id = test->inputs[i_i].expected_ids[id_i]; + if (id == 0) { break; } + expected_id_count++; + + /* must be ascending */ + if (id_i > 0) { + assert(id > test->inputs[i_i].expected_ids[id_i - 1]); + } + } + + if (log) { + fprintf(stderr, "%s: input %zd: \"%s\", expecting %zd ids:", + __func__, i_i, input, expected_id_count); + for (size_t i = 0; i < expected_id_count; i++) { + fprintf(stderr, " %d", test->inputs[i_i].expected_ids[i]); + } + } + + if (test->inputs[i_i].expect_fail) { + expected_id_count = 0; + } + + fsm_state_t end; /* only set on match */ + ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + + if (ret == 1) { +#define ENDID_BUF_SIZE 32 + fsm_end_id_t endid_buf[ENDID_BUF_SIZE] = {0}; + const size_t endid_count = fsm_endid_count(fsm, end); + /* fprintf(stderr, "%s: endid_count %zd for state %d\n", __func__, endid_count, end); */ + assert(endid_count < ENDID_BUF_SIZE); + if (!fsm_endid_get(fsm, end, /*ENDID_BUF_SIZE*/ endid_count, endid_buf)) { + assert(!"fsm_endid_get failed"); + } + + /* Copy endid outputs into outputs.ids[], since for testing + * purposes we don't care about the difference between eager + * output and endids here -- the values don't overlap. */ + assert(outputs.used + endid_count <= MAX_IDS); + for (size_t endid_i = 0; endid_i < endid_count; endid_i++) { + if (log) { + fprintf(stderr, "-- adding endid %zd: %d\n", endid_i, endid_buf[endid_i]); + } + outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + } + } + + if (ret == 0) { + /* if it didn't match, ignore the eager output IDs. this should + * eventually happen internal to fsm_exec or codegen. */ + outputs.used = 0; + } + + /* NEXT match IDs, sort outputs[] buffer first */ + qsort(outputs.ids, outputs.used, sizeof(outputs.ids[0]), cmp_output); + + if (log) { + fprintf(stderr, "-- got %zd:", outputs.used); + for (size_t i = 0; i < outputs.used; i++) { + fprintf(stderr, " %d", outputs.ids[i]); + } + fprintf(stderr, "\n"); + } + + if (expected_id_count == 0) { + assert(ret == 0 || outputs.used == 0); /* no match */ + continue; + } else { + assert(ret == 1); + } + + if (!allow_extra_outputs) { + assert(outputs.used == expected_id_count); + } else { + assert(outputs.used >= expected_id_count); + } + + size_t floor = 0; + for (size_t exp_i = 0; exp_i < outputs.used; exp_i++) { + bool found = false; + for (size_t got_i = floor; got_i < outputs.used; got_i++) { + if (outputs.ids[got_i] == test->inputs[i_i].expected_ids[exp_i]) { + floor = got_i + 1; + found = true; + break; + } + } + assert(found); + } + } + + fsm_free(fsm); + + return EXIT_SUCCESS;; +} diff --git a/tests/eager_output/utils.h b/tests/eager_output/utils.h new file mode 100644 index 000000000..672c01977 --- /dev/null +++ b/tests/eager_output/utils.h @@ -0,0 +1,64 @@ +#ifndef UTILS_H +#define UTILS_H + +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#define MAX_IDS 32 + +#include + +#include + +#define MAX_PATTERNS 150 +#define MAX_INPUTS 64 +#define MAX_ENDIDS 32 + +struct eager_output_test { + const char *patterns[MAX_PATTERNS]; + + struct { + const char *input; + bool expect_fail; + /* Terminated by 0. pattern[i] => id of i+1. Must be sorted. */ + fsm_output_id_t expected_ids[MAX_ENDIDS]; + } inputs[MAX_INPUTS]; +}; + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque); + +int +cmp_output(const void *pa, const void *pb); + +int +run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids); + +struct cb_info { + size_t used; + fsm_end_id_t ids[MAX_IDS]; +}; + +void +dump(const struct fsm *fsm); + +void +append_eager_output_cb(fsm_end_id_t id, void *opaque); + +#endif From 9fcbdf5222e13f8cb1311fe8317d8dfa6646c0a7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 13:32:38 -0400 Subject: [PATCH 04/54] Ensure .has_eager_outputs is zeroed on new states. (msan) --- src/libfsm/state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/state.c b/src/libfsm/state.c index d96c33653..8f1146038 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -65,6 +65,7 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) new->visited = 0; new->epsilons = NULL; new->edges = NULL; + new->has_eager_outputs = 0; } fsm->statecount++; From 1baee53e62c8a92a4babb9ef61cb93aeb6991e1d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 14:15:33 -0400 Subject: [PATCH 05/54] eager_output interface cleanup: Replace _any with _count and _get. fsm_eager_output_count and fsm_eager_output_get aligns better with the endid interface, and now fsm_eager_output_get ensures the buffer contents are sorted, --- include/fsm/fsm.h | 10 ++++++ src/libfsm/eager_output.c | 69 +++++++++++++++++++++++++++------------ src/libfsm/eager_output.h | 4 --- src/libfsm/epsilons.c | 3 +- src/libfsm/libfsm.syms | 2 ++ src/libfsm/minimise.c | 4 +-- src/libfsm/print/ir.c | 30 ++++------------- src/libfsm/union.c | 3 +- 8 files changed, 73 insertions(+), 52 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 701efe70b..9baf929d0 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -303,6 +303,16 @@ fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque); void fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque); +/* Get the number of eager output IDs associated with a state. */ +size_t +fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state); + +/* Get eager output associated with a state. It's expected that buf[] has + * sufficient space -- call fsm_eager_output_count first to get the count. + * The contents of buf will be sorted and unique. */ +void +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, fsm_output_id_t *buf); + /* * Find the state (if there is just one), or add epsilon edges from all states, * for which the given predicate is true. diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c index e37a8a4bf..e00e96cd1 100644 --- a/src/libfsm/eager_output.c +++ b/src/libfsm/eager_output.c @@ -276,6 +276,55 @@ fsm_eager_output_iter_state(const struct fsm *fsm, } } +static int +inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + (void)id; + size_t *count = opaque; + (*count)++; + return 1; +} + +/* Get the number of eager output IDs associated with a state. */ +size_t +fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state) +{ + size_t res = 0; + fsm_eager_output_iter_state(fsm, state, inc_cb, (void *)&res); + return res; +} + +struct get_env { + size_t count; + fsm_output_id_t *buf; +}; + +static int +append_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct get_env *env = opaque; + (void)state; + env->buf[env->count++] = id; + return 1; +} + +static int +cmp_fsm_output_id_t(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +void +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, fsm_output_id_t *buf) +{ + struct get_env env = { .buf = buf }; + fsm_eager_output_iter_state(fsm, state, append_cb, &env); + qsort(buf, env.count, sizeof(buf[0]), cmp_fsm_output_id_t); +} + void fsm_eager_output_iter_all(const struct fsm *fsm, fsm_eager_output_iter_cb *cb, void *opaque) @@ -327,26 +376,6 @@ fsm_eager_output_dump(FILE *f, const struct fsm *fsm) fprintf(f, "== %zu total\n", env.count); } -static int -inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) -{ - (void)state; - (void)id; - size_t *count = opaque; - (*count)++; - return 1; -} - -bool -fsm_eager_output_has_any(const struct fsm *fsm, - fsm_state_t state, size_t *count) -{ - size_t c = 0; - fsm_eager_output_iter_state(fsm, state, &inc_cb, &c); - if (count != NULL) { *count = c; } - return c > 0; -} - int fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count) { diff --git a/src/libfsm/eager_output.h b/src/libfsm/eager_output.h index 1b48ba4c4..6093adc9e 100644 --- a/src/libfsm/eager_output.h +++ b/src/libfsm/eager_output.h @@ -36,10 +36,6 @@ void fsm_eager_output_iter_all(const struct fsm *fsm, fsm_eager_output_iter_cb *cb, void *opaque); -bool -fsm_eager_output_has_any(const struct fsm *fsm, - fsm_state_t state, size_t *count); - int fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index adfcdec2a..8041c29d3 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -222,7 +222,8 @@ fsm_remove_epsilons(struct fsm *nfa) * in the current state's epsilon closure to the * current state. These will be added at the end. */ { - if (fsm_eager_output_has_any(nfa, es_id, NULL)) { + const size_t count = fsm_eager_output_count(nfa, es_id); + if (count > 0) { fsm_eager_output_iter_state(nfa, es_id, collect_eager_output_ids_cb, &eager_output_buf); if (!eager_output_buf.ok) { goto cleanup; } } diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 75c20eb64..f109bbf3e 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -102,9 +102,11 @@ fsm_endid_dump fsm_seteageroutput fsm_seteageroutputonends +fsm_eager_output_count # short term hack fsm_eager_output_set_cb fsm_eager_output_dump +fsm_eager_output_get fsm_countedges fsm_countstates diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 86f00b46f..a2ff1b818 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -1026,8 +1026,8 @@ static int collect_eager_output_ids(const struct fsm *fsm, fsm_state_t state, struct end_metadata_eager_outputs *e) { - size_t count = 0; - if (!fsm_eager_output_has_any(fsm, state, &count)) { + size_t count = fsm_eager_output_count(fsm, state); + if (count == 0) { return 1; /* nothing to do */ } diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 81d5890e0..a18dadbbc 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -506,23 +506,6 @@ make_example(const struct fsm *fsm, fsm_state_t s, char **example) return 0; } -static int -append_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) -{ - struct ir_state_eager_output *outputs = opaque; - (void)state; - outputs->ids[outputs->count++] = id; - return 1; -} - -static int -cmp_fsm_output_id_t(const void *pa, const void *pb) -{ - const fsm_output_id_t a = *(fsm_output_id_t *)pa; - const fsm_output_id_t b = *(fsm_output_id_t *)pb; - return a < b ? -1 : a > b ? 1 : 0; -} - struct ir * make_ir(const struct fsm *fsm, const struct fsm_options *opt) { @@ -587,17 +570,16 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) ir->states[i].endids.count = count; } - size_t count; - if (fsm_eager_output_has_any(fsm, i, &count)) { + const size_t eager_output_count = fsm_eager_output_count(fsm, i); + if (eager_output_count > 0) { struct ir_state_eager_output *outputs = f_malloc(fsm->alloc, - sizeof(*outputs) + count * sizeof(outputs->ids[0])); + sizeof(*outputs) + eager_output_count * sizeof(outputs->ids[0])); if (outputs == NULL) { goto error; } - outputs->count = 0; - fsm_eager_output_iter_state(fsm, i, append_eager_output_cb, outputs); - assert(outputs->count == count); - qsort(outputs->ids, outputs->count, sizeof(outputs->ids[0]), cmp_fsm_output_id_t); + fsm_eager_output_get(fsm, i, outputs->ids); + outputs->count = eager_output_count; + ir->states[i].eager_outputs = outputs; } diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 0b18cd30c..126181992 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -287,7 +287,8 @@ fsm_union_repeated_pattern_group(size_t entry_count, /* for any state that has eager outputs and a self edge, * remove the self edge before further linkage */ for (fsm_state_t s = 0; s < fsm->statecount; s++) { - if (!fsm_eager_output_has_any(fsm, s, NULL)) { continue; } + const size_t eager_output_count = fsm_eager_output_count(fsm, s); + if (eager_output_count == 0) { continue; } struct edge_set *edges = fsm->states[s].edges; struct edge_set *new = edge_set_new(); From 8fd728e829b1fc936157d8013b428625e93fc769 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 14:16:59 -0400 Subject: [PATCH 06/54] minimise_test_oracle.c: mismatched eager outputs also prevent merging. Do the same check with eager outputs as it does with endids -- any two states with distinct sets for either should end up in different equivalence classes. --- src/libfsm/minimise_test_oracle.c | 107 ++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index ec6d0d83c..20d4633a1 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -109,13 +109,19 @@ fsm_minimise_test_oracle(const struct fsm *fsm) fsm_state_t *tmp_map = NULL; fsm_state_t *mapping = NULL; - /* endid_group_assignments[X] = Y: state X is in endid group Y - * endid_group_leaders[X] = Y: see end state Y for endid group X */ - unsigned *endid_group_assignments = NULL; - size_t endid_group_count = 1; /* group 0 is the empty set */ - unsigned *endid_group_leaders = NULL; + /* End metadata grouping: The fixpoint algorithm here isn't aware + * of endids or eager outputs associated with particular states, + * so do a pass grouping them by matching end metadata. + * + * end_md_group_assignments[X] = Y: state X is in end_md group Y + * end_md_group_leaders[X] = Y: see end state Y for end_md group X */ + unsigned *end_md_group_assignments = NULL; + size_t end_md_group_count = 1; /* group 0 is the empty set */ + unsigned *end_md_group_leaders = NULL; fsm_end_id_t *ids_a = NULL; fsm_end_id_t *ids_b = NULL; + fsm_output_id_t *eo_ids_a = NULL; + fsm_output_id_t *eo_ids_b = NULL; table = calloc(row_words * table_states, sizeof(table[0])); if (table == NULL) { goto cleanup; } @@ -126,11 +132,11 @@ fsm_minimise_test_oracle(const struct fsm *fsm) mapping = malloc(state_count * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; } - endid_group_assignments = calloc(state_count, sizeof(tmp_map[0])); - if (endid_group_assignments == NULL) { goto cleanup; } + end_md_group_assignments = calloc(state_count, sizeof(tmp_map[0])); + if (end_md_group_assignments == NULL) { goto cleanup; } - endid_group_leaders = calloc(state_count, sizeof(tmp_map[0])); - if (endid_group_leaders == NULL) { goto cleanup; } + end_md_group_leaders = calloc(state_count, sizeof(tmp_map[0])); + if (end_md_group_leaders == NULL) { goto cleanup; } /* macros for NxN bit table */ #define POS(X,Y) ((X*table_states) + Y) @@ -139,6 +145,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) #define CHECK(X,Y) u64bitset_get(table, POS(X,Y)) size_t max_endid_count = 0; + size_t max_eager_output_count = 0; /* Mark all pairs of states where one is final and one is not. * This includes the dead state. */ @@ -150,6 +157,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) } } + /* count eager outputs, not just on end states */ + const size_t eo_count = fsm_eager_output_count(fsm, i); + if (eo_count > max_eager_output_count) { + max_eager_output_count = eo_count; + } + for (size_t j = 0; j < i; j++) { const bool end_i = i == dead_state ? false : fsm_isend(fsm, i); @@ -171,50 +184,70 @@ fsm_minimise_test_oracle(const struct fsm *fsm) ids_b = malloc(max_endid_count * sizeof(ids_b[0])); if (ids_b == NULL) { goto cleanup; } - /* For every end state, check if it has endids. If not, assign it - * to endid group 0 (none). Otherwise, check if its endids match - * any of the other end states. If so, assign it to the same endid + eo_ids_a = malloc(max_eager_output_count * sizeof(eo_ids_a[0])); + if (eo_ids_a == NULL) { goto cleanup; } + eo_ids_b = malloc(max_eager_output_count * sizeof(eo_ids_b[0])); + if (eo_ids_b == NULL) { goto cleanup; } + + /* For every end state, check if it has endids or eager outputs. + * If not, assign it to group 0 (none). Otherwise, check if its IDs match + * any of the other end states. If so, assign it to the same * group, otherwise assign a new one and mark it as the leader. */ for (size_t i = 0; i < state_count; i++) { if (!fsm_isend(fsm, i)) { - endid_group_assignments[i] = 0; /* none */ + end_md_group_assignments[i] = 0; /* none */ continue; } - size_t count_a = fsm_endid_count(fsm, i); - assert(count_a <= max_endid_count); - if (count_a == 0) { + const size_t endid_count_a = fsm_endid_count(fsm, i); + assert(endid_count_a <= max_endid_count); + + const size_t eager_output_count_a = fsm_eager_output_count(fsm, i); + assert(eager_output_count_a <= max_eager_output_count); + + if (endid_count_a == 0 && eager_output_count_a == 0) { continue; } - int eres = fsm_endid_get(fsm, i, count_a, ids_a); + int eres = fsm_endid_get(fsm, i, endid_count_a, ids_a); assert(eres == 1); + fsm_eager_output_get(fsm, i, eo_ids_a); + bool found = false; /* note: skipping eg 0 here since that's the empty set */ - for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { - size_t count_b = fsm_endid_count(fsm, endid_group_leaders[eg_i]); - if (count_b != count_a) { + for (size_t eg_i = 1; eg_i < end_md_group_count; eg_i++) { + size_t endid_count_b = fsm_endid_count(fsm, end_md_group_leaders[eg_i]); + if (endid_count_b != endid_count_a) { continue; } - assert(count_b > 0); - assert(count_b <= max_endid_count); - eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], - count_b, ids_b); + const size_t eager_output_count_b = fsm_eager_output_count(fsm, end_md_group_leaders[eg_i]); + assert(eager_output_count_b <= max_eager_output_count); + if (eager_output_count_b != eager_output_count_a) { + continue; + } + + assert(endid_count_b > 0 || eager_output_count_b > 0); + assert(endid_count_b <= max_endid_count); + eres = fsm_endid_get(fsm, end_md_group_leaders[eg_i], + endid_count_b, ids_b); assert(eres == 1); - if (0 == memcmp(ids_a, ids_b, count_a * sizeof(ids_a[0]))) { + fsm_eager_output_get(fsm, end_md_group_leaders[eg_i], eo_ids_b); + + if ((0 == memcmp(ids_a, ids_b, endid_count_a * sizeof(ids_a[0]))) && + (0 == memcmp(eo_ids_a, eo_ids_b, eager_output_count_a * sizeof(eo_ids_a[0])))) { found = true; - endid_group_assignments[i] = eg_i; + end_md_group_assignments[i] = eg_i; break; } } if (!found) { - endid_group_assignments[i] = endid_group_count; - endid_group_leaders[endid_group_count] = i; - endid_group_count++; + end_md_group_assignments[i] = end_md_group_count; + end_md_group_leaders[end_md_group_count] = i; + end_md_group_count++; } } @@ -222,10 +255,10 @@ fsm_minimise_test_oracle(const struct fsm *fsm) * group must be distinguishable. */ for (size_t i = 0; i < state_count; i++) { if (fsm_isend(fsm, i)) { - const unsigned i_group = endid_group_assignments[i]; + const unsigned i_group = end_md_group_assignments[i]; for (size_t j = 0; j < i; j++) { if (fsm_isend(fsm, j)) { - const unsigned j_group = endid_group_assignments[j]; + const unsigned j_group = end_md_group_assignments[j]; if (i_group != j_group) { MARK(i, j); } @@ -359,10 +392,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) free(table); free(tmp_map); free(mapping); - free(endid_group_assignments); - free(endid_group_leaders); + free(end_md_group_assignments); + free(end_md_group_leaders); free(ids_a); free(ids_b); + free(eo_ids_a); + free(eo_ids_b); return res; @@ -370,10 +405,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) if (table != NULL) { free(table); } if (tmp_map != NULL) { free(tmp_map); } if (mapping != NULL) { free(mapping); } - if (endid_group_assignments != NULL) { free(endid_group_assignments); } - if (endid_group_leaders != NULL) { free(endid_group_leaders); } + if (end_md_group_assignments != NULL) { free(end_md_group_assignments); } + if (end_md_group_leaders != NULL) { free(end_md_group_leaders); } if (ids_a != NULL) { free(ids_a); } if (ids_b != NULL) { free(ids_b); } if (res != NULL) { fsm_free(res); } + free(eo_ids_a); + free(eo_ids_b); return NULL; } From 98e9cb490c431fcff7980e1117c67eacbae22742 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 27 Sep 2024 18:21:58 -0400 Subject: [PATCH 07/54] Experimental initial support for CDATA output mode. This is an alternative for -lc and -lvmc that avoids very expensive compilation when the resulting C output is quite large. For this mode, most of the output is C data literals (a couple structs tables), followed by a very small (~50 loc) interpreter for the data. This is much faster to compile -- for a data set I'm working with now, it's 30 seconds to build compared to several hours and/or gcc exhausting memory. Generating output with comments enabled will include inline comments about the format, along with per-state comments showing labels, endids, and eager outputs. It will only generate code for endids and eager outputs if the DFA has them. This is experimental. I expect the interfaces will change a bit in the near future, and I am still working on performance tuning. There is some code to detect and reuse repeated runs of IDs in the output tables, but there is a bug leading to them not being terminated properly (possibly causing false positives), so it's currently disabled. To see a good example of the format, with comments, run: build/bin//re -rpcre -lcdata -u '^abc' --- include/fsm/print.h | 2 + include/fsm/walk.h | 6 +- src/fsm/main.c | 2 +- src/libfsm/gen.c | 65 +- src/libfsm/print.c | 2 + src/libfsm/print.h | 1 + src/libfsm/print/Makefile | 1 + src/libfsm/print/cdata.c | 1234 +++++++++++++++++++++++++++++++++++++ src/re/main.c | 3 +- tests/gen/gen1.c | 1 + tests/gen/gen2.c | 2 +- tests/gen/gen3.c | 2 +- 12 files changed, 1311 insertions(+), 10 deletions(-) create mode 100644 src/libfsm/print/cdata.c diff --git a/include/fsm/print.h b/include/fsm/print.h index 10244129b..3df5db304 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -37,6 +37,8 @@ enum fsm_print_lang { FSM_PRINT_VMC, /* ISO C90 code, VM style */ FSM_PRINT_VMDOT, /* Graphviz Dot format, showing VM opcodes */ + FSM_PRINT_CDATA, /* C data tables and small interpreter */ + FSM_PRINT_VMOPS_C, /* VM opcodes as a datastructure */ FSM_PRINT_VMOPS_H, FSM_PRINT_VMOPS_MAIN diff --git a/include/fsm/walk.h b/include/fsm/walk.h index e1ab5f29e..ea5a223e0 100644 --- a/include/fsm/walk.h +++ b/include/fsm/walk.h @@ -90,6 +90,10 @@ fsm_walk_edges(const struct fsm *fsm, void *opaque, * functionally equivalent cases makes testing dramatically faster, * but exploring every edge could be added later. * + * If seed is zero then it will generate the first label in the label + * set, otherwise a label from the set will be chosen using rand() + * (favoring printable characters). + * * Note: fsm is non-const because it calls fsm_trim on the FSM * internally. This records the shortest distance from each state to an * end state, which is used to prune branches that would not produce @@ -114,7 +118,7 @@ fsm_generate_matches_cb(const struct fsm *fsm, const char *input, size_t input_length, fsm_state_t end_state, void *opaque); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, +fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, fsm_generate_matches_cb *cb, void *opaque); /* Callback provided for the most basic use case for diff --git a/src/fsm/main.c b/src/fsm/main.c index da65791dd..f9d1bb3b0 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -770,7 +770,7 @@ main(int argc, char *argv[]) } if (generate_bounds > 0) { - r = fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt); + r = fsm_generate_matches(fsm, generate_bounds, 0, fsm_generate_cb_printf_escaped, &opt); } fsm_free(fsm); diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index e0ac429dd..9f78e67db 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -77,6 +77,7 @@ struct gen_ctx { fsm_generate_matches_cb *cb; bool done; + bool randomized; size_t buf_ceil; size_t buf_used; @@ -139,7 +140,7 @@ static bool grow_stack(struct gen_ctx *ctx); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, +fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, fsm_generate_matches_cb *cb, void *opaque) { if (max_length == 0) { @@ -153,7 +154,7 @@ fsm_generate_matches(struct fsm *fsm, size_t max_length, INIT_TIMERS(); TIME(&pre); - int res = gen_init_outer(fsm, max_length, cb, opaque, false, 0); + int res = gen_init_outer(fsm, max_length, cb, opaque, seed != 0, seed); TIME(&post); DIFF_MSEC("fsm_generate_matches", pre, post, NULL); @@ -212,8 +213,9 @@ gen_init_outer(struct fsm *fsm, size_t max_length, assert(fsm_all(fsm, fsm_isdfa)); /* DFA-only */ - assert(!randomized); /* not yet supported */ - (void)seed; + if (randomized) { + srand(seed); + } #if LOG_GEN > 1 fprintf(stderr, "%s: %u states\n", __func__, fsm_countstates(fsm)); @@ -228,6 +230,7 @@ gen_init_outer(struct fsm *fsm, size_t max_length, .max_length = max_length, .cb = cb, .opaque = opaque, + .randomized = randomized, }; if (!gen_init(&ctx, fsm)) { @@ -528,6 +531,55 @@ first_symbol(const uint64_t *symbols) return 0; } +static unsigned char +random_symbol(const uint64_t *symbols) +{ + bool has_zero = false; + unsigned i = 0; + + /* printable and non-printable character choices */ + size_t choice_count = 0; + unsigned char choices[256]; + size_t np_choice_count = 0; + unsigned char np_choices[256]; + + while (i < 256) { + const uint64_t w = symbols[i/64]; + if ((i & 63) == 0 && w == 0) { + i += 64; + continue; + } + if (w & (1ULL << (i & 63))) { + if (i == 0) { + has_zero = true; + } else if (isprint(i)) { + choices[choice_count++] = (unsigned char)i; + } else { + np_choices[np_choice_count++] = (unsigned char)i; + } + } + i++; + } + + if (choice_count > 0) { + const size_t c = rand() % choice_count; + return choices[c]; + } + + if (np_choice_count > 0) { + const size_t c = rand() % np_choice_count; + return np_choices[c]; + } + + /* Prefer anything besides 0x00 if present, since that will truncate the string. */ + if (has_zero) { + return 0; + } + + assert(!"empty set"); + return 0; +} + #if DUMP_EDGES static void dump_edges(fsm_state_t state, struct edge_set *edges) @@ -542,6 +594,7 @@ dump_edges(fsm_state_t state, struct edge_set *edges) size_t i = 0; while (edge_set_group_iter_next(&ei, &eg)) { const unsigned char symbol = first_symbol(eg.symbols); + const unsigned char symbol = random_symbol(eg.symbols); fprintf(stderr, "%s: %d -- %zu/%zu -- 0x%02x (%c) -> %d\n", __func__, state, i, count, symbol, isprint(symbol) ? symbol : '.', eg.to); @@ -589,7 +642,9 @@ sfs_step_edges(struct gen_ctx *ctx, struct gen_stack_frame *sf) struct edge_group_iter_info eg; if (iter_next_transition(ctx, sf, &eg)) { - const unsigned char symbol = first_symbol(eg.symbols); + const unsigned char symbol = ctx->randomized + ? random_symbol(eg.symbols) + : first_symbol(eg.symbols); const fsm_state_t state = eg.to; LOG(2, "sfs_step_edges: got edge 0x%x ('%c')\n", diff --git a/src/libfsm/print.c b/src/libfsm/print.c index e1a425cc0..533b264f6 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -327,6 +327,8 @@ fsm_print(FILE *f, const struct fsm *fsm, case FSM_PRINT_VMC: print_vm = fsm_print_vmc; break; case FSM_PRINT_VMDOT: print_vm = fsm_print_vmdot; break; + case FSM_PRINT_CDATA: print_ir = fsm_print_cdata; break; + case FSM_PRINT_VMOPS_C: print_vm = fsm_print_vmops_c; break; case FSM_PRINT_VMOPS_H: print_vm = fsm_print_vmops_h; break; case FSM_PRINT_VMOPS_MAIN: print_vm = fsm_print_vmops_main; break; diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 0a8e2e0c1..86c2fdfce 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -88,6 +88,7 @@ vm_print_f fsm_print_llvm; vm_print_f fsm_print_rust; vm_print_f fsm_print_sh; vm_print_f fsm_print_vmc; +ir_print_f fsm_print_cdata; vm_print_f fsm_print_vmdot; vm_print_f fsm_print_vmops_c; diff --git a/src/libfsm/print/Makefile b/src/libfsm/print/Makefile index c0015c899..51f43d3f7 100644 --- a/src/libfsm/print/Makefile +++ b/src/libfsm/print/Makefile @@ -11,6 +11,7 @@ SRC += src/libfsm/print/irdot.c SRC += src/libfsm/print/irjson.c SRC += src/libfsm/print/json.c SRC += src/libfsm/print/llvm.c +SRC += src/libfsm/print/cdata.c SRC += src/libfsm/print/rust.c SRC += src/libfsm/print/sh.c SRC += src/libfsm/print/vmasm.c diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c new file mode 100644 index 000000000..863eff5f5 --- /dev/null +++ b/src/libfsm/print/cdata.c @@ -0,0 +1,1234 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include "libfsm/internal.h" +#include "libfsm/print.h" + +#include "ir.h" + +/* Print mode that generates C data literals for the DFA, plus a small interepreter. + * This mostly exists to sidestep very expensive compilation for large data sets + * using the other C modes. -sv */ + + +/* Whether to check the table buffers for previous instances of + * the same sets of dst_states, endids, and eager_outputs. This + * should always be an improvement, making the generated code + * smaller and improve locality. */ +#define REUSE_ALL_SETS 0 + +/* If reusing sets, use a simplistic and inefficient (but easily checked) + * implementation. If EXPENSIVE_CHECKS is set this will still be used, to check + * the result of the more efficient approaches. */ +#define REUSE_NAIVE 1 + +/* Disabled for now. Linear scan for this is too slow, whereas + * it's fine for the other two, and reuse makes a much bigger + * difference for those. */ +#define REUSE_DST_TABLE_SETS (REUSE_ALL_SETS || 0) + +/* Disabled for now, this can lead to false positives due to + * unterminated runs. Needs more testing. */ +#define REUSE_ENDID_SETS (REUSE_ALL_SETS || 0) +#define REUSE_EAGER_OUTPUT_SETS (REUSE_ALL_SETS || 0) + +/* Log size information to stderr. */ +#define LOG_SIZES 0 + +/* Log stats for set reuse to stderr. */ +#define LOG_REUSE 0 + +/* How large a numeric ID type do the particular tables need? */ +enum id_type { + U8, + U16, + U32, + U64, + UNSIGNED, /* always use 'unsigned' */ +}; + +static const char* +id_type_str(enum id_type t) +{ + switch (t) { + case U8: return "uint8_t"; + case U16: return "uint16_t"; + case U32: return "uint32_t"; + case U64: return "uint64_t"; + case UNSIGNED: return "unsigned"; + default: + assert(!"match fail"); + return ""; + } +} + +static enum id_type +size_needed(size_t max_value) +{ + if (max_value < (1ULL << 8)) { return U8; } + if (max_value < (1ULL << 16)) { return U16; } + if (max_value < (1ULL << 32)) { return U32; } + return U64; +} + +#define STATE_OFFSET_NONE ((size_t)-1) + +/* Configuration. Figure out whether we really need a uint32_t for edge offsets + * or can get by with a uint16_t, etc. and make the output more dense. */ +struct cdata_config { + fsm_state_t start; + size_t state_count; + + /* numeric type for state IDs, based on the higest state # */ + enum id_type t_state_id; + + /* numeric type for indexes into the dst_state table, based on + * the number of label groups */ + enum id_type t_dst_state_offset; + + /* numeric types for endid and eager_output table offsets */ + enum id_type t_endid_offset; + enum id_type t_eager_output_offset; + + /* numeric types for endid and eager_output table values */ + enum id_type t_endid_value; + enum id_type t_eager_output_value; + + struct dst_buf { + size_t ceil; + size_t used; + uint32_t *buf; + } dst_buf; + + struct endid_buf { + size_t ceil; + size_t used; + unsigned *buf; + } endid_buf; + size_t max_endid; + + struct eager_output_buf { + size_t ceil; + size_t used; + uint64_t *buf; + } eager_output_buf; + size_t max_eager_output_id; + + struct state_info { + uint64_t labels[256/64]; + uint64_t label_group_starts[256/64]; + uint8_t rank_sums[4]; + + /* Offsets into the other data tables, or STATE_OFFSET_NONE, + * which will be converted to the table length and treated + * as NONE by a runtime range check. */ + size_t default_dst; + size_t dst; + size_t endid; + size_t eager_output; + } *state_info; + +#if LOG_REUSE + struct reuse_stats { + size_t miss; + size_t hit; + } reuse_stats_dst; + struct reuse_stats reuse_stats_endid; + struct reuse_stats reuse_stats_eager_output; +#endif +}; + +static bool +generate_struct_definition(FILE *f, const struct cdata_config *config, bool comments, const char *prefix) +{ + const bool has_endids = config->endid_buf.used > 0; + const bool has_eager_outputs = config->eager_output_buf.used > 0; + + fprintf(f, + "\ttypedef %s %s_cdata_state;\n", + id_type_str(config->t_state_id), prefix); + + /* TODO: move .end and .endid_offset into a separate table, they aren't accessed + * until the end of input, so this should slightly reduce cache misses. + * Do this once there's baseline timing info. */ + fprintf(f, + "\tstruct %s_cdata_dfa {\n" + "\t\t%s_cdata_state start;\n" + "\t\tstruct %s_cdata_state {\n" + "\t\t\tbool end; /* is this an end state? */\n" + "\n", prefix, prefix, prefix); + + if (comments) { + fprintf(f, + "\t\t\t/* To find the destination state for label character C,\n" + "\t\t\t * check if the bit C is set in .labels[]. If so, find the\n" + "\t\t\t * the 1 bit at or preceding C in .label_group_starts[],\n" + "\t\t\t * which represents the start of the Nth label group, the\n" + "\t\t\t * group label group that contains C. The dst state will be in\n" + "\t\t\t * .dst_table[.dst_table_offset + N]. This offset N is called\n" + "\t\t\t * the rank, and .rank_sums has precomputed sums for each\n" + "\t\t\t * word preceding .label_group_starts[C/64]. If .labels[]\n" + "\t\t\t * isn't set for C, the destination is .default_dst, or the\n" + "\t\t\t * state count (%zu) for no match. */\n" + "\n", config->state_count); + } + fprintf(f, + "\t\t\t%s_cdata_state default_dst; /* or %zu for NONE */\n" + "\t\t\tuint64_t labels[256/4]; /* which labels have non-default edges */\n" + "\t\t\tuint64_t label_group_starts[256/4]; /* start of each label group */\n" + "\t\t\tuint8_t rank_sums[4]; /* rank at end of label_group_starts[n] */\n" + "\n", prefix, config->state_count); + + if (comments) { + fprintf(f, "\t\t\t/* Offsets into values in other tables */\n"); + } + fprintf(f, "\t\t\t%s dst_table_offset;\n", id_type_str(config->t_dst_state_offset)); + + if (has_endids) { + fprintf(f, "\t\t\t%s endid_offset;\n", id_type_str(config->t_endid_offset)); + } + if (has_eager_outputs) { + fprintf(f, "\t\t\t%s eager_output_offset;\n", id_type_str(config->t_eager_output_offset)); + } + + fprintf(f, + "\t\t} states[%zd];\n" + "\n", config->state_count); + + if (comments) { + fprintf(f, + "\t\t/* Destination states for each edge group in each state,\n" + "\t\t * starting from .states[state_id].dst_state_offset. */\n"); + } + fprintf(f, + "\t\t%s_cdata_state dst_table[%zd];\n", + prefix, config->dst_buf.used); + + if (has_endids) { + if (comments) { + fprintf(f, + "\n" + "\t\t/* Ascending runs of endids, refered to\n" + "\t\t * by .states[state_id].endid_offset,\n" + "\t\t * terminated by non-increasing value. */\n"); + } + fprintf(f, "\t\t%s endid_table[%zd + 1];\n", + id_type_str(config->t_endid_value), config->endid_buf.used); + } + + if (has_eager_outputs > 0) { + if (comments) { + fprintf(f, + "\n" + "\t\t/* Ascending runs of eager_outputs, refered to\n" + "\t\t * by .states[state_id].eager_output_offset,\n" + "\t\t * terminated by non-increasing value. */\n"); + } + fprintf(f, "\t\t%s eager_output_table[%zd + 1];\n", + id_type_str(config->t_eager_output_value), config->eager_output_buf.used); + } + + fprintf(f, + "\t};\n"); + return true; +} + +static bool +generate_data(FILE *f, const struct cdata_config *config, + bool comments, const char *prefix, const struct ir *ir) +{ + fprintf(f, + "\tstatic struct %s_cdata_dfa %s_dfa_data = {\n" + "\t\t.start = %u,\n" + "\t\t.states = {\n", prefix, prefix, config->start); + + for (size_t s_i = 0; s_i < ir->n; s_i++) { + const struct ir_state *s = &ir->states[s_i]; + const struct state_info *si = &config->state_info[s_i]; + + const bool is_end = s->isend; + const bool has_endids = si->endid != STATE_OFFSET_NONE; + const bool has_eager_outputs = si->eager_output != STATE_OFFSET_NONE; + + fprintf(f, "\t\t\t[%zd] = {%s\n", s_i, s_i == config->start ? " /* start */" : ""); + + if (comments) { + fprintf(f, "\t\t\t\t// "); + for (size_t i = 0; i < 256; i++) { + if (si->labels[i/64] & ((uint64_t)1 << (i & 63))) { + char c = (char)i; + fprintf(f, "%c", isprint(c) ? c : '.'); + } + } + fprintf(f, "\n"); + } + fprintf(f, "\t\t\t\t.labels = { 0x%lx, 0x%lx, 0x%lx, 0x%lx },\n", + si->labels[0], si->labels[1], si->labels[2], si->labels[3]); + + size_t dst_count = 0; + if (comments) { + fprintf(f, "\t\t\t\t// "); + for (size_t i = 0; i < 256; i++) { + if (si->label_group_starts[i/64] & ((uint64_t)1 << (i & 63))) { + char c = (char)i; + fprintf(f, "%c", isprint(c) ? c : '.'); + dst_count++; + } + } + fprintf(f, "\n"); + } + fprintf(f, "\t\t\t\t.label_group_starts = { 0x%lx, 0x%lx, 0x%lx, 0x%lx },\n", + si->label_group_starts[0], si->label_group_starts[1], si->label_group_starts[2], si->label_group_starts[3]); + + /* rank_sums[0] is always 0, but allows us to avoid a subtraction in the inner loop, + * and the space would be wasted otherwise anyway due to alignment. */ + fprintf(f, "\t\t\t\t.rank_sums = { %u, %u, %u, %u },\n", + si->rank_sums[0], si->rank_sums[1], si->rank_sums[2], si->rank_sums[3]); + + const size_t state_NONE = config->state_count; + const size_t dst_table_NONE = config->dst_buf.used; + const size_t endid_NONE = config->endid_buf.used; + const size_t eager_output_NONE = config->eager_output_buf.used; + + if (si->default_dst == STATE_OFFSET_NONE) { + fprintf(f, "\t\t\t\t.default_dst = %zu, /* NONE */\n", state_NONE); + } else { + if (comments) { + fprintf(f, "\t\t\t\t//"); + for (size_t i = 0; i < dst_count; i++) { + fprintf(f, " %u,", config->dst_buf.buf[si->dst + i]); + } + fprintf(f, "\n"); + } + fprintf(f, "\t\t\t\t.default_dst = %zu,\n", si->default_dst); + } + + fprintf(f, "\t\t\t\t.end = %d,\n", is_end); + + if (si->dst == STATE_OFFSET_NONE) { /* no non-default outgoing edges */ + fprintf(f, "\t\t\t\t.dst_table_offset = %zd, /* NONE */\n", dst_table_NONE); + } else { + fprintf(f, "\t\t\t\t.dst_table_offset = %zd,\n", si->dst); + } + + /* Only include these if any state uses endids/eager_outputs, and + * if this state doesn't then use the end of the array as NONE. */ + if (config->endid_buf.used > 0) { + if (has_endids) { + if (comments) { + fprintf(f, "\t\t\t\t/* endids:"); + for (size_t i = 0; i < s->endids.count; i++) { + if (i > 0 && (i & 15) == 0) { + fprintf(f, "\n\t\t\t\t *"); + } + fprintf(f, " %u", s->endids.ids[i]); + } + fprintf(f, " */\n"); + } + fprintf(f, "\t\t\t\t.endid_offset = %zd,\n", si->endid); + } else { + fprintf(f, "\t\t\t\t.endid_offset = %zd, /* NONE */\n", endid_NONE); + } + } + + if (config->eager_output_buf.used > 0) { + if (has_eager_outputs) { + if (comments) { + fprintf(f, "\t\t\t\t/* eager_outputs:"); + for (size_t i = 0; i < s->eager_outputs->count; i++) { + if (i > 0 && (i & 15) == 0) { + fprintf(f, "\n\t\t\t\t *"); + } + fprintf(f, " %u", s->eager_outputs->ids[i]); + } + fprintf(f, " */\n"); + } + fprintf(f, "\t\t\t\t.eager_output_offset = %zd,\n", si->eager_output); + } else { + fprintf(f, "\t\t\t\t.eager_output_offset = %zd, /* NONE */\n", eager_output_NONE); + } + } + + fprintf(f, "\t\t\t},\n"); + } + fprintf(f, "\t\t},\n"); + + fprintf(f, + "\t\t.dst_table = {"); + + for (size_t i = 0; i < config->dst_buf.used; i++) { + if ((i & 15) == 0) { fprintf(f, "\n\t\t\t"); } + fprintf(f, " %u,", config->dst_buf.buf[i]); + } + + /* edges */ + fprintf(f, "\n\t\t},\n"); + + if (config->endid_buf.used > 0) { + fprintf(f, "\t\t.endid_table = {"); + for (size_t i = 0; i < config->endid_buf.used; i++) { + if ((i & 15) == 0) { fprintf(f, "\n\t\t\t"); } + fprintf(f, " %u,", config->endid_buf.buf[i]); + } + fprintf(f, "\n\t\t\t 0 /* end */,\n"); + fprintf(f, "\n\t\t},\n"); + } + + if (config->eager_output_buf.used > 0) { + fprintf(f, "\t\t.eager_output_table = {"); + for (size_t i = 0; i < config->eager_output_buf.used; i++) { + if ((i & 15) == 0) { fprintf(f, "\n\t\t\t"); } + fprintf(f, " %lu,", config->eager_output_buf.buf[i]); + } + fprintf(f, "\n\t\t\t 0 /* end */,\n"); + fprintf(f, "\n\t\t},\n"); + } + + fprintf(f, "\t};\n"); + + return true; +} + +static void +generate_eager_output_check(FILE *f, const struct cdata_config *config, const char *prefix) +{ + /* If any states have eager outputs, check if the current state + * does, and if so, set their flags. This assumes eager_output_buf is large enough, + * and is a strong incentive to use sequentially assigned IDs. */ + if (config->eager_output_buf.used > 0) { + fprintf(f, + "\n" + "\t\tif (state->eager_output_offset < %s_EAGER_OUTPUT_TABLE_COUNT) {\n" + "\t\t\tif (debug_traces) {\n" + "\t\t\t\tfprintf(stderr, \"-- eager_output_offset %%u\\n\", state->eager_output_offset);\n" + "\t\t\t}\n" + "\t\t\t%s *eo_scan = &%s_dfa_data.eager_output_table[state->eager_output_offset];\n" + "\t\t\t%s cur, next;\n" + "\t\t\tdo {\n" + "\t\t\t\tcur = *eo_scan;\n" + "\t\t\t\tif (debug_traces) {\n" + "\t\t\t\t\tfprintf(stderr, \"%%s: setting eager_output flag %%u\\n\", __func__, cur);\n" + "\t\t\t\t}\n" + "\t\t\t\teager_output_buf[cur/64] |= (uint64_t)1 << (cur & 63);\n" + "\t\t\t\teo_scan++;\n" + "\t\t\t\tnext = *eo_scan;\n" + "\t\t\t} while (next > cur);\n" + "\t\t}\n", + prefix, + id_type_str(config->t_eager_output_value), + prefix, + id_type_str(config->t_eager_output_value)); + } +} + +static bool +generate_interpreter(FILE *f, const struct cdata_config *config, const struct fsm_options *opt, const char *prefix) +{ + const bool has_endids = config->endid_buf.used > 0; + const bool has_eager_outputs = config->eager_output_buf.used > 0; + + fprintf(f, "\tconst size_t %s_STATE_COUNT = %zd;\n", prefix, config->state_count); + + if (has_endids) { + fprintf(f, "\tconst size_t %s_ENDID_TABLE_COUNT = %zd;\n", prefix, config->endid_buf.used); + } + + if (has_eager_outputs) { + fprintf(f, "\tconst size_t %s_EAGER_OUTPUT_TABLE_COUNT = %zd;\n", prefix, config->eager_output_buf.used); + } + + /* start state */ + fprintf(f, "\tuint32_t cur_state = %s_dfa_data.start;\n", prefix); + fprintf(f, "\n"); + + /* Setting this to true will log out execution steps. */ + const bool debug_traces = false; + fprintf(f, + "\tconst bool debug_traces = %s;\n", debug_traces ? "true" : "false"); + + /* Loop over the input characters */ + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "\tint raw_c;\n"); + fprintf(f, "\twhile (raw_c = fsm_getc(getc_opaque), raw_c != EOF) {\n"); + fprintf(f, "\t\tconst uint8_t c = (uint8_t)raw_c;\n"); + break; + + case FSM_IO_STR: + fprintf(f, "\tconst char *p;\n"); + fprintf(f, "\tfor (p = s; *p != '\\0'; p++) {\n"); + fprintf(f, "\t\tconst uint8_t c = (uint8_t)*p;\n"); + break; + + case FSM_IO_PAIR: + fprintf(f, "\tconst char *p;\n"); + fprintf(f, "\tfor (p = b; p != e; p++) {\n"); + fprintf(f, "\t\tconst uint8_t c = (uint8_t)*p;\n"); + break; + } + + fprintf(f, + "\t\tconst struct %s_cdata_state *state = &%s_dfa_data.states[cur_state];\n", prefix, prefix); + + /* If the state being entered has eager_outputs, set their flags. */ + generate_eager_output_check(f, config, prefix); + + /* Function to count the bits set in a uint64_t. + * + * TODO It may be faster to use a small lookup table and add + * the next N least significant bits, halting on 0: + * + * size_t sum = 0; + * while (word != 0) { + * sum += lookup_table_8_bit_popcount[word & 0xff]; + * word >>= 8; + * } + * + * because usually many of the upper bits will be masked out. */ + const char *popcount = "__builtin_popcountl"; + + /* For each character of the input, check if it's in the set of + * outgoing labels. If so, find the label group that contains + * that label by counting the bits preceding that offset in + * label_group_starts[]. Each label group is represented by its + * starting character. Call that the Nth label group. Next, + * find the Nth state ID in the edge table, starting from the + * state's base offset in that table. + * + * The bit counts for each word of label_group_starts[] are + * cached in state->rank_sums, so it only needs to count the bits + * within state->label_group_starts[c/64] before the character mod + * 64. + * + * If the character isn't in the label set, then go to the + * default destination state. If default_dst is set to an + * out-of-bounds value it means there isn't one, so there's no + * match. */ + fprintf(f, + "\t\tconst size_t w_i = c/64;\n" + "\t\tconst size_t word_rem = c & 63;\n" + "\t\tconst uint64_t bit = (uint64_t)1 << word_rem;\n" + "\t\tif (state->labels[w_i] & bit) { /* if state has label */\n" + "\t\t\tif (debug_traces) {\n" + "\t\t\t\tfprintf(stderr, \"-- label '%%c' (0x%%02x) -> w_i %%zd, bit 0x%%016lx\\n\", isprint(c) ? c : 'c', c, w_i, bit);\n" + "\t\t\t}\n" + "\t\t\tconst uint64_t lgs_word = state->label_group_starts[w_i];\n" + "\t\t\tconst size_t back = (lgs_word & bit) ? 0 : 1; /* back to start of label group */\n" + "\t\t\tconst uint64_t mask = bit - 1;\n" + "\t\t\tconst uint64_t masked_word = lgs_word & mask;\n" + "\t\t\tconst size_t bit_rank_in_masked_word = %s(masked_word) - back;\n" + "\t\t\tconst size_t rank = state->rank_sums[w_i] + bit_rank_in_masked_word;\n" + "\t\t\tconst size_t dst_offset = state->dst_table_offset + rank;\n" + "\t\t\tcur_state = %s_dfa_data.dst_table[dst_offset];\n" + "\t\t\tif (debug_traces) {\n" + "\t\t\t\tfprintf(stderr, \"-- has label, rank %%zd -> dst_offset %%zu -> next_state %%u\\n\",\n" + "\t\t\t\t\trank, dst_offset, cur_state);\n" + "\t\t\t}\n" + "\t\t\tcontinue;\n" + "\t\t} else if (state->default_dst < %s_STATE_COUNT) {\n" + "\t\t\tcur_state = state->default_dst;\n" + "\t\t\tif (debug_traces) {\n" + "\t\t\t\tfprintf(stderr, \"-- doesn't have label -> default state %%u\\n\", state->default_dst);\n" + "\t\t\t}\n" + "\t\t} else {\n" + "\t\t\tif (debug_traces) {\n" + "\t\t\t\tfprintf(stderr, \"-- doesn't have label -> match fail\\n\");\n" + "\t\t\t}\n" + "\t\t\treturn 0; /* no match */\n" + "\t\t}\n" + "\t}\n", + popcount, prefix, prefix); + + /* At the end of the input, check if the current state is an end. + * If not, there's no match. */ + fprintf(f, + "\tconst struct %s_cdata_state *state = &%s_dfa_data.states[cur_state];\n" + "\tif (!state->end) { return 0; /* no match */ }\n", prefix, prefix); + + /* Set the passed-in reference to the endids, if any. */ + if (has_endids) { + /* If there are endids in the DFA, check if the current state's + * endid_offset is in range. (If not, the state has none.) + * Those endids appear in as a run of ascending values in + * the endid_table, starting from that offset, and are terminated + * by the first lower value. endid_table[] has an extra 0 appended + * as a terminator for the last set. */ + fprintf(f, + "\tif (state->endid_offset < %s_ENDID_TABLE_COUNT) {\n" + "\t\t%s *endid_scan = &%s_dfa_data.endid_table[state->endid_offset];\n" + "\t\tconst %s *endid_base = endid_scan;\n" + "\t\tsize_t endid_count = 0;\n" + "\t\tuint64_t cur, next;\n" + "\t\tdo {\n" + "\t\t\tcur = *endid_scan;\n" + "\t\t\tendid_scan++;\n" + "\t\t\tendid_count++;\n" + "\t\t\tnext = *endid_scan;\n" + "\t\t} while (next > cur);\n", + prefix, + id_type_str(config->t_endid_value), + prefix, + id_type_str(config->t_endid_value)); + + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, + "\t\t*id = endid_base;\n" + "\t\t(void)endid_count;\n"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, + "\t\t*%s = endid_base;\n" + "\t\t*%s = endid_count;\n" + "\t}\n", + /* TODO: rename these to endid_ids and endid_count? + * That will be an interface change. */ + "ids", "count"); + break; + + default: + assert(!"unreached"); + abort(); + } + } + + /* If the end state has eager_outputs, set their flags. */ + generate_eager_output_check(f, config, prefix); + + /* Got a match. */ + fprintf(f, "\treturn 1; /* match */\n"); + return true; +} + +static bool +append_endid(struct endid_buf *buf, uint64_t id) +{ + if (buf->used == buf->ceil) { + const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; + unsigned *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); + if (nbuf == NULL) { return false; } + buf->buf = nbuf; + buf->ceil = nceil; + } + + assert(buf->used < buf->ceil); + buf->buf[buf->used++] = id; + return true; +} + +static bool +save_state_endids(struct cdata_config *config, const struct ir_state_endids *endids, size_t *offset) +{ + if (endids->count == 0) { + assert(*offset == STATE_OFFSET_NONE); + return true; + } + + /* These must be in ascending order. */ + for (size_t i = 1; i < endids->count; i++) { + assert(endids->ids[i - 1] < endids->ids[i]); + } + +#if REUSE_ENDID_SETS + /* Intern the run of endids. They are often identical + * between states, so the earlier reference could be reused. + * This is particulary important since they're all stored as + * "unsigned" rather than reducing to the smallest numeric + * type that fits all values used. */ + +#if REUSE_NAIVE || EXPENSIVE_CHECKS + /* Search for a previous run of the same endids in the buffer via linear scan. + * This is simple but scales poorly. */ + size_t naive_offset = STATE_OFFSET_NONE; + for (size_t b_i = 0; b_i < config->endid_buf.used; b_i++) { + size_t e_i; + for (e_i = 0; e_i < endids->count; e_i++) { + if (b_i + e_i >= config->endid_buf.used) { + break; /* reached the end, not found */ + } + if (config->endid_buf.buf[b_i + e_i] != endids->ids[e_i]) { + break; /* mismatch */ + } + } + + /* If there's a potential match, check that it isn't followed by + * a value > the last, because it would falsely continue the run. */ + if (e_i == endids->count + && b_i + e_i + 1 < config->endid_buf.used + && config->endid_buf.buf[b_i + e_i + 1] <= endids->ids[e_i - 1]) { + naive_offset = b_i; + break; + } + } + + if (REUSE_NAIVE) { + *offset = naive_offset; + } +#endif /* REUSE_NAIVE || EXPENSIVE_CHECKS */ + + /* TODO: better impl? */ + +#if LOG_REUSE + if (*offset == STATE_OFFSET_NONE) { + config->reuse_stats_endid.miss++; + } else { + config->reuse_stats_endid.hit++; + } +#endif + + if (*offset != STATE_OFFSET_NONE) { + return true; + } +#endif /* REUSE_ENDID_SETS */ + + /* If the first endid for this state is later than the last + * endid in the buffer, append an extra terminator 0 for the + * last run of endids. Otherwise, the last state with endids + * will be falsely associated with this state's as well. */ + if (config->endid_buf.used > 0 + && endids->ids[0] > config->endid_buf.buf[config->endid_buf.used - 1]) { + if (!append_endid(&config->endid_buf, 0)) { + return false; + } + } + + const size_t base = config->endid_buf.used; + + for (size_t i = 0; i < endids->count; i++) { + if (endids->ids[i] > config->max_endid) { + config->max_endid = endids->ids[i]; + } + if (!append_endid(&config->endid_buf, endids->ids[i])) { + return false; + } + } + + assert(base != STATE_OFFSET_NONE); + *offset = base; + return true; +} + +static bool +append_eager_output(struct eager_output_buf *buf, uint64_t id) +{ + if (buf->used == buf->ceil) { + const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; + uint64_t *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); + assert(nbuf != NULL); + buf->buf = nbuf; + buf->ceil = nceil; + } + + assert(buf->used < buf->ceil); + buf->buf[buf->used++] = id; + return true; +} + +static bool +save_state_eager_outputs(struct cdata_config *config, const struct ir_state_eager_output *eager_outputs, size_t *offset) +{ + if (eager_outputs == NULL || eager_outputs->count == 0) { + assert(*offset == STATE_OFFSET_NONE); + return true; + } + + /* These must be in ascending order. */ + for (size_t i = 1; i < eager_outputs->count; i++) { + assert(eager_outputs->ids[i - 1] < eager_outputs->ids[i]); + } + +#if REUSE_EAGER_OUTPUT_SETS +#if REUSE_NAIVE || EXPENSIVE_CHECKS + /* Linear scan. See comments about reuse in save_state_endids. */ + size_t naive_offset = STATE_OFFSET_NONE; + for (size_t b_i = 0; b_i < config->eager_output_buf.used; b_i++) { + size_t e_i; + for (e_i = 0; e_i < eager_outputs->count; e_i++) { + if (b_i + e_i >= config->eager_output_buf.used) { + break; /* reached the end, not found */ + } + if (config->eager_output_buf.buf[b_i + e_i] != eager_outputs->ids[e_i]) { + break; /* mismatch */ + } + } + + /* If there's a potential match, check that it isn't followed by + * a value > the last, because it would falsely continue the run. */ + if (e_i == eager_outputs->count + && b_i + e_i + 1 < config->eager_output_buf.used + && config->eager_output_buf.buf[b_i + e_i + 1] <= eager_outputs->ids[e_i - 1]) { + naive_offset = b_i; + break; + } + } + + if (REUSE_NAIVE) { + *offset = naive_offset; + } +#endif /* REUSE_NAIVE || EXPENSIVE_CHECKS */ + + /* TODO: better impl */ + +#if LOG_REUSE + if (*offset == STATE_OFFSET_NONE) { + config->reuse_stats_eager_output.miss++; + } else { + config->reuse_stats_eager_output.hit++; + } +#endif + + if (*offset != STATE_OFFSET_NONE) { + return true; + } +#endif /* REUSE_EAGER_OUTPUT_SETS */ + + /* If necessary add a 0, as in save_state_endids above. */ + if (config->eager_output_buf.used > 0 + && eager_outputs->ids[0] > config->eager_output_buf.buf[config->eager_output_buf.used - 1]) { + if (!append_eager_output(&config->eager_output_buf, 0)) { + return false; + } + } + + const size_t base = config->eager_output_buf.used; + + for (size_t i = 0; i < eager_outputs->count; i++) { + if (eager_outputs->ids[i] > config->max_eager_output_id) { + config->max_eager_output_id = eager_outputs->ids[i]; + } + + if (!append_eager_output(&config->eager_output_buf, eager_outputs->ids[i])) { + return false; + } + } + + assert(base != STATE_OFFSET_NONE); + *offset = base; + return true; +} + +struct range_info { + uint8_t start; + uint8_t end; + uint32_t dst_state; +}; + +static int +cmp_outgoing(const void *pa, const void *pb) +{ + const struct range_info *a = (const struct range_info *)pa; + const struct range_info *b = (const struct range_info *)pb; + return a->start < b->start ? -1 : a->start > b->start ? 1 : 0; +} + +static bool +append_dst(struct dst_buf *buf, uint32_t dst) +{ + if (buf->used == buf->ceil) { + const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; + uint32_t *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); + assert(nbuf != NULL); + buf->buf = nbuf; + buf->ceil = nceil; + } + + assert(buf->used < buf->ceil); + buf->buf[buf->used++] = dst; + return true; +} + +static bool +save_state_edge_group_destinations(struct cdata_config *config, struct state_info *si, + size_t group_count, const struct ir_group *groups) +{ + /* Convert the group ranges to bitsets and an edge->destination state list. */ +#define DUMP_GROUP 0 + if (DUMP_GROUP) { + fprintf(stderr, "\n%s: dump_group [[\n", __func__); + for (size_t g_i = 0; g_i < group_count; g_i++) { + fprintf(stderr, "- group %zd:\n", g_i); + const struct ir_group *g = &groups[g_i]; + for (size_t r_i = 0; r_i < g->n; r_i++) { + const struct ir_range *r = &g->ranges[r_i]; + assert(r->start <= r->end); + fprintf(stderr, " - range[%zd]: '%c'(0x%02x) -- '%c'(0x%02x)\n", + r_i, + isprint(r->start) ? r->start : '.', r->start, + isprint(r->end) ? r->end : '.', r->end); + } + } + fprintf(stderr, "]]\n"); + } + + /* Because this is a DFA there should be at most 256 outgoing. */ + size_t outgoing_used = 0; + struct range_info outgoing[256]; + + /* Groups and ranges aren't necessarily ordered by character, so collect and sort first, + * otherwise the ranks can point to the wrong offset in edges[]. */ + for (size_t g_i = 0; g_i < group_count; g_i++) { + const struct ir_group *g = &groups[g_i]; + for (size_t r_i = 0; r_i < g->n; r_i++) { + const struct ir_range *r = &g->ranges[r_i]; + assert(r->start <= r->end); + outgoing[outgoing_used++] = (struct range_info){ + .start = r->start, + .end = r->end, + .dst_state = g->to, + }; + assert(outgoing_used <= 256); + } + } + + /* Sort them by .start. They should be unique and non-overlapping. */ + qsort(outgoing, outgoing_used, sizeof(outgoing[0]), cmp_outgoing); + for (size_t o_i = 1; o_i < outgoing_used; o_i++) { + assert(outgoing[o_i - 1].start < outgoing[o_i].start); + } + + for (size_t i = 0; i < 4; i++) { + si->labels[i] = 0; + si->label_group_starts[i] = 0; + } + + /* First pass: populate the bitsets. */ + for (size_t o_i = 0; o_i < outgoing_used; o_i++) { + const struct range_info *r = &outgoing[o_i]; + assert(!u64bitset_get(si->label_group_starts, r->start)); + u64bitset_set(si->label_group_starts, r->start); + + for (uint8_t c = r->start; c <= r->end; c++) { + assert(!u64bitset_get(si->labels, c)); + u64bitset_set(si->labels, c); + } + } + + /* Precompute label_group_starts[] rank sums so lookup only needs to + * compute rank for the label's word, not every word preceding it. */ + si->rank_sums[0] = 0; + uint8_t total = 0; + for (size_t i = 1; i < 4; i++) { + total += u64bitset_popcount(si->label_group_starts[i - 1]); + si->rank_sums[i] = total; + } + + struct dst_buf *dst_buf = &config->dst_buf; + + /* Second pass: search for an previous intance of the same run + * of destination states in dst_buf, reusing that if possible. */ +#if REUSE_DST_TABLE_SETS +#if REUSE_NAIVE || EXPENSIVE_CHECKS + /* Linear scan. See comments about reuse in save_state_endids. */ + size_t naive_offset = STATE_OFFSET_NONE; + for (size_t b_i = 0; b_i < dst_buf->used; b_i++) { + size_t o_i; + for (o_i = 0; o_i < outgoing_used; o_i++) { + if (b_i + o_i >= dst_buf->used) { + break; /* reached the end, not found */ + } + if (dst_buf->buf[b_i + o_i] != outgoing[o_i].dst_state) { + break; /* mismatch */ + } + } + if (o_i == outgoing_used) { /* got a match */ + naive_offset = b_i; + break; + } + } + + if (REUSE_NAIVE) { + si->dst = naive_offset; + } +#endif /* REUSE_NAIVE || EXPENSIVE_CHECKS */ + + /* TODO: Better implementation. This should use the same + * backward chaining search index approach that heatshrink uses, + * and since the DFA is trimmed all states should have at least + * one edge leading to them: it doesn't need to be sparse. + * Making the dst_state table smaller will improve locality and + * make it faster. */ + +#if LOG_REUSE + if (si->dst == STATE_OFFSET_NONE) { + config->reuse_stats_dst.miss++; + } else { + config->reuse_stats_dst.hit++; + } +#endif + + if (si->dst != STATE_OFFSET_NONE) { + return true; + } +#endif /* REUSE_DST_TABLE_SETS */ + + /* Otherwise, append the destination states to dst_buf. */ + const size_t base = dst_buf->used; + for (size_t o_i = 0; o_i < outgoing_used; o_i++) { + const struct range_info *r = &outgoing[o_i]; + if (!append_dst(dst_buf, r->dst_state)) { + return false; + } + } + + assert(base != STATE_OFFSET_NONE); + si->dst = base; + return true; +} + +static bool +populate_config_from_ir(struct cdata_config *config, const struct ir *ir) +{ + memset(config, 0x00, sizeof(*config)); + config->start = ir->start; + config->state_count = ir->n; + + config->state_info = malloc(ir->n * sizeof(config->state_info[0])); + assert(config->state_info != NULL); + + /* could just memset this to 0xff, but this is explicit */ + for (size_t s_i = 0; s_i < ir->n; s_i++) { + struct state_info *si = &config->state_info[s_i]; + si->dst = STATE_OFFSET_NONE; + si->default_dst = STATE_OFFSET_NONE; + si->endid = STATE_OFFSET_NONE; + si->eager_output = STATE_OFFSET_NONE; + } + + for (size_t s_i = 0; s_i < ir->n; s_i++) { + const struct ir_state *s = &ir->states[s_i]; + + if (!save_state_endids(config, &s->endids, &config->state_info[s_i].endid)) { + goto alloc_fail; + } + + if (!save_state_eager_outputs(config, s->eager_outputs, &config->state_info[s_i].eager_output)) { + goto alloc_fail; + } + + struct state_info *si = &config->state_info[s_i]; + + switch (s->strategy) { + case IR_NONE: + break; + case IR_SAME: /* all default */ + config->state_info[s_i].default_dst = s->u.same.to; + break; + case IR_COMPLETE: + if (!save_state_edge_group_destinations(config, + si, s->u.complete.n, s->u.complete.groups)) { + goto alloc_fail; + } + break; + case IR_PARTIAL: + if (!save_state_edge_group_destinations(config, + si, s->u.partial.n, s->u.partial.groups)) { + goto alloc_fail; + } + break; + case IR_DOMINANT: + if (!save_state_edge_group_destinations(config, + si, s->u.dominant.n, s->u.dominant.groups)) { + goto alloc_fail; + } + config->state_info[s_i].default_dst = s->u.dominant.mode; + break; + case IR_ERROR: + goto not_implemented; + case IR_TABLE: + goto not_implemented; + default: + goto not_implemented; + } + } + + /* Get the smallest numeric type that will fit all state IDs in + * the current DFA, reserving one extra to use as an out-of-band + * "NONE" value for fields like default_dst. These also the values + * in the dst_state table (the destination state for every edge group), + * so storing them more densely has a big impact on the overall data size. */ + config->t_state_id = size_needed(config->state_count + 1); + + /* Offset into the dst_state table. */ + config->t_dst_state_offset = size_needed(config->dst_buf.used); + + /* Both of these also ensure there's space for at least one out-of-band value + * to use as "NONE". */ + config->t_endid_offset = size_needed(config->endid_buf.used + 1); + config->t_eager_output_offset = size_needed(config->eager_output_buf.used + 1); + + /* The caller expects this to be unsigned, and the current interface just sets + * a pointer to the array of IDs. */ + config->t_endid_value = UNSIGNED; //size_needed(config->max_endid); + + config->t_eager_output_value = size_needed(config->max_eager_output_id); + return true; + +not_implemented: + assert(!"not implemented"); + return false; + +alloc_fail: + assert(!"alloc fail"); + return false; +} + +int +fsm_print_cdata(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir) +{ + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + assert(ir != NULL); + + (void)hooks; /* ignored, for now */ + + /* First pass, figure out totals and index sizes */ + struct cdata_config config; + populate_config_from_ir(&config, ir); + +#if LOG_SIZES + fprintf(stderr, "// config: dst_state_count %zu, start %d, dst_buf.used %zd, endid_buf.used %zd, eager_output_buf.used %zd\n", + config.state_count, + config.start, + config.dst_buf.used, + config.endid_buf.used, + config.eager_output_buf.used); +#endif + +#if LOG_REUSE + fprintf(stderr, "// reuse: dst: hit %zu (%g%%), miss %zu\n", + config.reuse_stats_dst.hit, (100.0 * config.reuse_stats_dst.hit) / (config.reuse_stats_dst.hit + config.reuse_stats_dst.miss), + config.reuse_stats_dst.miss); + if (config.reuse_stats_endid.hit + config.reuse_stats_endid.miss > 0) { + fprintf(stderr, "// reuse: endids: hit %zu (%g%%), miss %zu\n", + config.reuse_stats_endid.hit, (100.0 * config.reuse_stats_endid.hit) / (config.reuse_stats_endid.hit + config.reuse_stats_endid.miss), + config.reuse_stats_endid.miss); + } + if (config.reuse_stats_eager_output.hit + config.reuse_stats_eager_output.miss > 0) { + fprintf(stderr, "// reuse: eager_outputs: hit %zu (%g%%), miss %zu\n", + config.reuse_stats_eager_output.hit, + (100.0 * config.reuse_stats_eager_output.hit) / (config.reuse_stats_eager_output.hit + config.reuse_stats_eager_output.miss), + config.reuse_stats_eager_output.miss); + } +#endif + + const char *prefix; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (!opt->fragment) { /* generate function head */ + fprintf(f, "\n"); + + fprintf(f, "int\n%smain", prefix); + fprintf(f, "("); + + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "int (*fsm_getc)(void *getc_opaque), void *getc_opaque"); + break; + + case FSM_IO_STR: + fprintf(f, "const char *s"); + break; + + case FSM_IO_PAIR: + fprintf(f, "const char *b, const char *e"); + break; + } + + /* TODO: add an opt flag for eager_output codegen */ + if (config.eager_output_buf.used > 0) { + fprintf(f, ",\n"); + fprintf(f, "\tuint64_t *eager_output_buf"); + } + + /* + * unsigned rather than fsm_end_id_t here, so the generated code + * doesn't depend on fsm.h + */ + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned *id"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); + break; + + default: + assert(!"unreached"); + abort(); + } + + if (hooks->args != NULL) { + fprintf(f, ",\n"); + fprintf(f, "\t"); + + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { + return -1; + } + } + fprintf(f, ")\n"); + fprintf(f, "{\n"); + + if (!generate_struct_definition(f, &config, opt->comments, prefix)) { return -1; } + if (!generate_data(f, &config, opt->comments, prefix, ir)) { return -1; } + if (!generate_interpreter(f, &config, opt, prefix)) { return -1; } + + fprintf(f, "}\n"); + fprintf(f, "\n"); + } else { + /* caller sets up the function head */ + if (!generate_struct_definition(f, &config, opt->comments, prefix)) { return -1; } + if (!generate_data(f, &config, opt->comments, prefix, ir)) { return -1; } + if (!generate_interpreter(f, &config, opt, prefix)) { return -1; } + } + + free(config.dst_buf.buf); + free(config.endid_buf.buf); + free(config.eager_output_buf.buf); + free(config.state_info); + + return 0; +} diff --git a/src/re/main.c b/src/re/main.c index 62e51f78d..4bacdbe1a 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -124,6 +124,7 @@ lang_name(const char *name, enum fsm_print_lang *fsm_lang, enum ast_print_lang * { "rust", FSM_PRINT_RUST }, { "sh", FSM_PRINT_SH }, { "vmc", FSM_PRINT_VMC }, + { "cdata", FSM_PRINT_CDATA }, { "vmdot", FSM_PRINT_VMDOT }, { "vmops_c", FSM_PRINT_VMOPS_C }, @@ -1047,7 +1048,7 @@ main(int argc, char *argv[]) } if (generate_bounds > 0) { - if (!fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt)) { + if (!fsm_generate_matches(fsm, generate_bounds, 0, fsm_generate_cb_printf_escaped, &opt)) { exit(EXIT_FAILURE); } diff --git a/tests/gen/gen1.c b/tests/gen/gen1.c index 25b30b82b..b25ab2bbc 100644 --- a/tests/gen/gen1.c +++ b/tests/gen/gen1.c @@ -34,6 +34,7 @@ int main(void) { assert(fsm != NULL); if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1 /* for \0 */, + 0, gtest_matches_cb, &matches)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); diff --git a/tests/gen/gen2.c b/tests/gen/gen2.c index 02faa5e50..a475e5395 100644 --- a/tests/gen/gen2.c +++ b/tests/gen/gen2.c @@ -28,7 +28,7 @@ int main(void) { struct fsm *fsm = gtest_fsm_of_matches(&matches); assert(fsm != NULL); - if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1, gtest_matches_cb, &matches)) { + if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1, 0, gtest_matches_cb, &matches)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); } diff --git a/tests/gen/gen3.c b/tests/gen/gen3.c index f24217622..7aa0aebce 100644 --- a/tests/gen/gen3.c +++ b/tests/gen/gen3.c @@ -146,7 +146,7 @@ int main(void) { struct fsm *fsm = build(); assert(fsm != NULL); - if (!fsm_generate_matches(fsm, 11, matches_cb, NULL)) { + if (!fsm_generate_matches(fsm, 11, 0, matches_cb, NULL)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); } From 23e6142fd555f703801b2d16610f57a484f5bffc Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 15:48:13 -0400 Subject: [PATCH 08/54] fuzzer: Add seed argument for fsm_generate_matches (interface change). --- fuzz/target.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index d56a9bf82..5f8c51c2e 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -446,6 +446,8 @@ fuzz_eager_output(const uint8_t *data, size_t size) size_t max_pattern_length = 0; + const unsigned seed = size == 0 ? 0 : data[0]; + /* chop data into a series of patterns */ { size_t prev = 0; @@ -645,7 +647,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) * Use the combined DFA to generate matches, check that the * match behavior agrees with the individual DFA copies. */ env.current_pattern = (size_t)-1; - if (!fsm_generate_matches(env.combined, max_pattern_length, gen_combined_check_individual_cb, &env)) { + if (!fsm_generate_matches(env.combined, max_pattern_length, seed, gen_combined_check_individual_cb, &env)) { goto cleanup; } @@ -655,7 +657,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) /* check behavior against the combined DFA. */ for (size_t i = 0; i < env.pattern_count; i++) { env.current_pattern = i; - if (!fsm_generate_matches(env.combined, max_pattern_length, gen_individual_check_combined_cb, &env)) { + if (!fsm_generate_matches(env.combined, max_pattern_length, seed, gen_individual_check_combined_cb, &env)) { goto cleanup; } } From 4ea2588d5d310ac083422995538acbcfcaba8a75 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 11:38:57 -0400 Subject: [PATCH 09/54] Add `srand(seed)` to fuzzer harness. fsm_generate_matches is no longer seeding `rand()` directly. --- fuzz/target.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzz/target.c b/fuzz/target.c index 5f8c51c2e..e0c35c018 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -447,6 +447,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) size_t max_pattern_length = 0; const unsigned seed = size == 0 ? 0 : data[0]; + srand(seed); /* chop data into a series of patterns */ { From 25a91411d698594b66dbb73f17f26abd0e090266 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 11:48:55 -0400 Subject: [PATCH 10/54] cdata: Remove assertions on realloc's result. I confirmed that the callers actually check the return. These should probably use the alloc interface (with `f_realloc`), but the existing print callback typedefs don't seem to pass along the alloc handle anymore, since it was removed from fsm_options, so if that changes it will be in a different commit. --- src/libfsm/print/cdata.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 863eff5f5..b2aaa6748 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -635,7 +635,9 @@ append_endid(struct endid_buf *buf, uint64_t id) if (buf->used == buf->ceil) { const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; unsigned *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); - if (nbuf == NULL) { return false; } + if (nbuf == NULL) { + return false; + } buf->buf = nbuf; buf->ceil = nceil; } @@ -743,7 +745,9 @@ append_eager_output(struct eager_output_buf *buf, uint64_t id) if (buf->used == buf->ceil) { const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; uint64_t *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); - assert(nbuf != NULL); + if (nbuf == NULL) { + return false; + } buf->buf = nbuf; buf->ceil = nceil; } @@ -856,7 +860,9 @@ append_dst(struct dst_buf *buf, uint32_t dst) if (buf->used == buf->ceil) { const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; uint32_t *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); - assert(nbuf != NULL); + if (nbuf == NULL) { + return false; + } buf->buf = nbuf; buf->ceil = nceil; } From 3ea08983cc60c92722f00132634d589c18b406df Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 11:57:25 -0400 Subject: [PATCH 11/54] Add CDATA to rx. --- src/rx/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rx/main.c b/src/rx/main.c index 8fe40eb58..d3099a4d4 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -705,6 +705,7 @@ print_name(const char *name, enum fsm_ambig ambig) enum fsm_ambig mask; } a[] = { { "c", FSM_PRINT_VMC, AMBIG_ANY }, + { "cdata", FSM_PRINT_CDATA, AMBIG_ANY }, { "dot", FSM_PRINT_DOT, AMBIG_ANY }, { "fsm", FSM_PRINT_FSM, AMBIG_ANY }, { "rust", FSM_PRINT_RUST, AMBIG_ANY }, From 4b611924fd55e1d79e4e8f3367a5af53c0691fbb Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 12:48:18 -0400 Subject: [PATCH 12/54] cdata: Avoid "zero size arrays are an extension" warning. If the dst_table buffer is empty, add a 0. --- src/libfsm/print/cdata.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index b2aaa6748..0c75f8f8c 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -226,7 +226,7 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm } fprintf(f, "\t\t%s_cdata_state dst_table[%zd];\n", - prefix, config->dst_buf.used); + prefix, config->dst_buf.used == 0 ? 1 : config->dst_buf.used); if (has_endids) { if (comments) { @@ -384,6 +384,10 @@ generate_data(FILE *f, const struct cdata_config *config, if ((i & 15) == 0) { fprintf(f, "\n\t\t\t"); } fprintf(f, " %u,", config->dst_buf.buf[i]); } + if (config->dst_buf.used == 0) { + /* Avoid "error: use of GNU empty initializer extension". */ + fprintf(f, " 0, /* empty */"); + } /* edges */ fprintf(f, "\n\t\t},\n"); From 1d5c0a0926a01c19e16b0d7d49381d05acf4db65 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:23:19 -0400 Subject: [PATCH 13/54] cdata: Fix output type for single endid in AMBIG_EARLIEST. --- src/libfsm/print/cdata.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 0c75f8f8c..96fba468d 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -605,8 +605,9 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, - "\t\t*id = endid_base;\n" - "\t\t(void)endid_count;\n"); + "\t\t*id = *endid_base;\n" + "\t\t(void)endid_count;\n" + "\t}\n"); break; case AMBIG_MULTIPLE: @@ -1198,7 +1199,7 @@ fsm_print_cdata(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, ",\n"); - fprintf(f, "\tconst unsigned *id"); + fprintf(f, "\tunsigned *id"); break; case AMBIG_MULTIPLE: From 32121f33e5ce2844d735f4c7ea62dd6a31565c4a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:23:47 -0400 Subject: [PATCH 14/54] cdata: Fix rollover on (255,255). --- src/libfsm/print/cdata.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 96fba468d..c58d1947f 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -937,9 +937,11 @@ save_state_edge_group_destinations(struct cdata_config *config, struct state_inf assert(!u64bitset_get(si->label_group_starts, r->start)); u64bitset_set(si->label_group_starts, r->start); - for (uint8_t c = r->start; c <= r->end; c++) { - assert(!u64bitset_get(si->labels, c)); - u64bitset_set(si->labels, c); + for (uint16_t c = r->start; c <= r->end; c++) { + /* c is a uint16_t to avoid rollover for { .start=255,.end=255 }. */ + const uint8_t c8 = (uint8_t)c; + assert(!u64bitset_get(si->labels, c8)); + u64bitset_set(si->labels, c8); } } From 454409154305974c04a557c5714b0a0fadc584b8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:49:21 -0400 Subject: [PATCH 15/54] cdata: Treat '\' as non-printable, since it leads to line continuation. --- src/libfsm/print/cdata.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index c58d1947f..5529e18a9 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -281,6 +281,7 @@ generate_data(FILE *f, const struct cdata_config *config, for (size_t i = 0; i < 256; i++) { if (si->labels[i/64] & ((uint64_t)1 << (i & 63))) { char c = (char)i; + if (c == '\\') { c = '.'; } fprintf(f, "%c", isprint(c) ? c : '.'); } } @@ -295,6 +296,7 @@ generate_data(FILE *f, const struct cdata_config *config, for (size_t i = 0; i < 256; i++) { if (si->label_group_starts[i/64] & ((uint64_t)1 << (i & 63))) { char c = (char)i; + if (c == '\\') { c = '.'; } fprintf(f, "%c", isprint(c) ? c : '.'); dst_count++; } From bbcac922ba58c8697fedd59349b0c3a92fa0ccb0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:54:23 -0400 Subject: [PATCH 16/54] fuzz/target.c: re_is_anchor interface changes. --- fuzz/target.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index d56a9bf82..53b6003c1 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -503,13 +503,14 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - struct re_anchoring_info anchorage[MAX_PATTERNS] = {0}; + enum re_is_anchored_res anchorage[MAX_PATTERNS] = {0}; /* for each pattern, attempt to compile to a DFA */ for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { const char *p = env.patterns[p_i]; - if (!re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL, &anchorage[p_i])) { + enum re_is_anchored_res a = re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL); + if (a == RE_IS_ANCHORED_ERROR) { continue; /* unsupported regex */ } @@ -599,8 +600,8 @@ fuzz_eager_output(const uint8_t *data, size_t size) } entries[used].fsm = cp; - entries[used].anchored_start = anchorage[i].start; - entries[used].anchored_end = anchorage[i].end; + entries[used].anchored_start = anchorage[i] & RE_IS_ANCHORED_START; + entries[used].anchored_end = anchorage[i] & RE_IS_ANCHORED_END; used++; } From e133a746b9cb94272d7cf92bf0ff0b76bf749d02 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:55:21 -0400 Subject: [PATCH 17/54] fuzz/target: fsm_generate_matches has randomized flag. Use the first byte of input as a seed, when available. --- fuzz/target.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 53b6003c1..2e2dcf07c 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -422,6 +422,11 @@ fsm_eager_output_dump(FILE *f, const struct fsm *fsm); static int fuzz_eager_output(const uint8_t *data, size_t size) { + if (size > 0) { + unsigned seed = data[0]; + srand(seed); + } + struct feo_env env = { .ok = true, .pattern_count = 0, @@ -646,7 +651,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) * Use the combined DFA to generate matches, check that the * match behavior agrees with the individual DFA copies. */ env.current_pattern = (size_t)-1; - if (!fsm_generate_matches(env.combined, max_pattern_length, gen_combined_check_individual_cb, &env)) { + if (!fsm_generate_matches(env.combined, max_pattern_length, 1, gen_combined_check_individual_cb, &env)) { goto cleanup; } @@ -656,7 +661,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) /* check behavior against the combined DFA. */ for (size_t i = 0; i < env.pattern_count; i++) { env.current_pattern = i; - if (!fsm_generate_matches(env.combined, max_pattern_length, gen_individual_check_combined_cb, &env)) { + if (!fsm_generate_matches(env.combined, max_pattern_length, 1, gen_individual_check_combined_cb, &env)) { goto cleanup; } } From 5f087786896438ba0eca83ec6b8ca3dc8883ff20 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:59:55 -0400 Subject: [PATCH 18/54] cdata: Ensure config->state_info is zeroed. --- src/libfsm/print/cdata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 5529e18a9..fae96821d 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -1026,7 +1026,7 @@ populate_config_from_ir(struct cdata_config *config, const struct ir *ir) config->start = ir->start; config->state_count = ir->n; - config->state_info = malloc(ir->n * sizeof(config->state_info[0])); + config->state_info = calloc(ir->n, sizeof(config->state_info[0])); assert(config->state_info != NULL); /* could just memset this to 0xff, but this is explicit */ From 7a935b4e2c0def7c8826ed786951f80f3cfdd21b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 14:01:02 -0400 Subject: [PATCH 19/54] Add cdata to retest. (This found a couple issues.) --- src/retest/main.c | 3 +++ src/retest/runner.c | 27 +++++++++++++++++++++++++++ src/retest/runner.h | 1 + tests/retest/Makefile | 2 +- 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/retest/main.c b/src/retest/main.c index 53eba216c..6baa0ab7b 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -175,6 +175,7 @@ usage(void) fprintf(stderr, " vm interpret vm instructions (default)\n"); fprintf(stderr, " asm/goasm generate assembly and assemble\n"); fprintf(stderr, " c compile as per fsm_print_c()\n"); + fprintf(stderr, " cdata compile as per fsm_print_cdata()\n"); fprintf(stderr, " vmc compile as per fsm_print_vmc()\n"); fprintf(stderr, " vmops compile as per fsm_print_vmops_{c,h,main}()\n"); fprintf(stderr, " rust compile as per fsm_print_rust()\n"); @@ -1274,6 +1275,8 @@ main(int argc, char *argv[]) impl = IMPL_VMASM; } else if (strcmp(optarg, "c") == 0) { impl = IMPL_C; + } else if (strcmp(optarg, "cdata") == 0) { + impl = IMPL_CDATA; } else if (strcmp(optarg, "go") == 0) { impl = IMPL_GO; } else if (strcmp(optarg, "goasm") == 0) { diff --git a/src/retest/runner.c b/src/retest/runner.c index 49cd611af..797dbe68f 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -83,11 +83,22 @@ print(const struct fsm *fsm, fprintf(f, "#include \n\n"); } + /* the cdata codegen uses fixed-width numeric types, bool, size_t, + * fprintf, isprint, and C99 designated initializers */ + if (impl == IMPL_CDATA) { + fprintf(f, "#include \n\n"); + fprintf(f, "#include \n\n"); + fprintf(f, "#include \n\n"); + fprintf(f, "#include \n\n"); + fprintf(f, "#include \n\n"); + } + { int e; switch (impl) { case IMPL_C: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_C); break; + case IMPL_CDATA: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_CDATA); break; case IMPL_RUST: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_RUST); break; case IMPL_LLVM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_LLVM); break; case IMPL_VMC: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMC); break; @@ -157,6 +168,17 @@ compile(enum implementation impl, break; + case IMPL_CDATA: /* requires c99 for designated initializers */ + if (0 != systemf("%s %s -shared -fPIC %s -o %s", + cc ? cc : "gcc", cflags ? cflags : "-std=c99 -pedantic -Wall -Werror -O3", + tmp_src, tmp_so)) + { + return 0; + } + + break; + + case IMPL_RUST: if (0 != systemf("%s %s -C opt-level=3 --crate-type dylib %s -o %s", "rustc", "--edition 2021", @@ -309,6 +331,7 @@ runner_init_compiled(struct fsm *fsm, switch (impl) { case IMPL_VMOPS: case IMPL_C: + case IMPL_CDATA: case IMPL_VMC: tmp_src = tmp_src_c; break; case IMPL_RUST: tmp_src = tmp_src_rs; break; case IMPL_LLVM: tmp_src = tmp_src_ll; break; @@ -362,6 +385,7 @@ runner_init_compiled(struct fsm *fsm, /* XXX: depends on IO API */ switch (r->impl) { case IMPL_C: + case IMPL_CDATA: case IMPL_VMC: r->u.impl_c.h = h; r->u.impl_c.func = (int (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm_main"); @@ -418,6 +442,7 @@ fsm_runner_initialize(struct fsm *fsm, const struct fsm_options *opt, switch (impl) { case IMPL_C: + case IMPL_CDATA: case IMPL_LLVM: case IMPL_RUST: case IMPL_VMASM: @@ -448,6 +473,7 @@ fsm_runner_finalize(struct fsm_runner *r) switch (r->impl) { case IMPL_C: + case IMPL_CDATA: case IMPL_VMC: case IMPL_VMOPS: if (r->u.impl_c.h != NULL) { @@ -499,6 +525,7 @@ fsm_runner_run(const struct fsm_runner *r, const char *s, size_t n) switch (r->impl) { case IMPL_C: + case IMPL_CDATA: case IMPL_VMC: case IMPL_VMOPS: assert(r->u.impl_c.func != NULL); diff --git a/src/retest/runner.h b/src/retest/runner.h index 40b269eb0..70fb78926 100644 --- a/src/retest/runner.h +++ b/src/retest/runner.h @@ -27,6 +27,7 @@ enum error_type { enum implementation { IMPL_C, + IMPL_CDATA, IMPL_GO, IMPL_GOASM, IMPL_INTERPRET, diff --git a/tests/retest/Makefile b/tests/retest/Makefile index 94fa19b69..47e0b3452 100644 --- a/tests/retest/Makefile +++ b/tests/retest/Makefile @@ -8,7 +8,7 @@ DIR += ${TEST_OUTDIR.tests/retest} RETEST=${BUILD}/bin/retest -.for lang in vm asm c vmc llvm +.for lang in vm asm c cdata vmc llvm .for io in pair str # XXX: we don't have FSM_IO_STR for asm yet From 9f90b01e130c4b46696410d678a2624686da101e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sun, 13 Oct 2024 09:24:53 -0400 Subject: [PATCH 20/54] print: Add allocator handle to ir_print_f typedef. --- src/libfsm/print.c | 2 +- src/libfsm/print.h | 1 + src/libfsm/print/c.c | 2 ++ src/libfsm/print/cdata.c | 2 ++ src/libfsm/print/irdot.c | 2 ++ src/libfsm/print/irjson.c | 2 ++ 6 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 533b264f6..0615c0d1f 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -365,7 +365,7 @@ fsm_print(FILE *f, const struct fsm *fsm, } if (print_ir != NULL) { - r = print_ir(f, opt, hooks, ir); + r = print_ir(f, fsm->alloc, opt, hooks, ir); goto done; } diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 86c2fdfce..e3e485162 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -61,6 +61,7 @@ typedef int fsm_print_f(FILE *f, const struct fsm *fsm); typedef int ir_print_f(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ir *ir); diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index cc3927dc6..b3c053ce1 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -530,11 +530,13 @@ fsm_print_c_body(FILE *f, const struct ir *ir, int fsm_print_c(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ir *ir) { const char *prefix; + (void)alloc; assert(f != NULL); assert(opt != NULL); diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index fae96821d..3cc3fb9ed 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -1118,6 +1118,7 @@ populate_config_from_ir(struct cdata_config *config, const struct ir *ir) int fsm_print_cdata(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ir *ir) @@ -1127,6 +1128,7 @@ fsm_print_cdata(FILE *f, assert(hooks != NULL); assert(ir != NULL); + (void)alloc; (void)hooks; /* ignored, for now */ /* First pass, figure out totals and index sizes */ diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 37dc34296..94b4947ef 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -332,11 +332,13 @@ print_state(FILE *f, int fsm_print_ir(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ir *ir) { size_t i; + (void)alloc; assert(f != NULL); assert(opt != NULL); diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index a96250e0f..d4e98b5c6 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -222,11 +222,13 @@ print_state(FILE *f, int fsm_print_irjson(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ir *ir) { size_t i; + (void)alloc; assert(f != NULL); assert(opt != NULL); From 4fe0beb84c61501af002bb0a885b2da9a0790750 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sun, 13 Oct 2024 09:29:30 -0400 Subject: [PATCH 21/54] cdata: Use passed-in alloc handle and its interface. --- src/libfsm/print/cdata.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 3cc3fb9ed..0e2fd53b7 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -100,6 +100,8 @@ size_needed(size_t max_value) /* Configuration. Figure out whether we really need a uint32_t for edge offsets * or can get by with a uint16_t, etc. and make the output more dense. */ struct cdata_config { + const struct fsm_alloc *alloc; + fsm_state_t start; size_t state_count; @@ -637,11 +639,11 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs } static bool -append_endid(struct endid_buf *buf, uint64_t id) +append_endid(const struct fsm_alloc *alloc, struct endid_buf *buf, uint64_t id) { if (buf->used == buf->ceil) { const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; - unsigned *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); + unsigned *nbuf = f_realloc(alloc, buf->buf, nceil * sizeof(nbuf[0])); if (nbuf == NULL) { return false; } @@ -725,7 +727,7 @@ save_state_endids(struct cdata_config *config, const struct ir_state_endids *end * will be falsely associated with this state's as well. */ if (config->endid_buf.used > 0 && endids->ids[0] > config->endid_buf.buf[config->endid_buf.used - 1]) { - if (!append_endid(&config->endid_buf, 0)) { + if (!append_endid(config->alloc, &config->endid_buf, 0)) { return false; } } @@ -736,7 +738,7 @@ save_state_endids(struct cdata_config *config, const struct ir_state_endids *end if (endids->ids[i] > config->max_endid) { config->max_endid = endids->ids[i]; } - if (!append_endid(&config->endid_buf, endids->ids[i])) { + if (!append_endid(config->alloc, &config->endid_buf, endids->ids[i])) { return false; } } @@ -747,11 +749,11 @@ save_state_endids(struct cdata_config *config, const struct ir_state_endids *end } static bool -append_eager_output(struct eager_output_buf *buf, uint64_t id) +append_eager_output(const struct fsm_alloc *alloc, struct eager_output_buf *buf, uint64_t id) { if (buf->used == buf->ceil) { const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; - uint64_t *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); + uint64_t *nbuf = f_realloc(alloc, buf->buf, nceil * sizeof(nbuf[0])); if (nbuf == NULL) { return false; } @@ -825,7 +827,7 @@ save_state_eager_outputs(struct cdata_config *config, const struct ir_state_eage /* If necessary add a 0, as in save_state_endids above. */ if (config->eager_output_buf.used > 0 && eager_outputs->ids[0] > config->eager_output_buf.buf[config->eager_output_buf.used - 1]) { - if (!append_eager_output(&config->eager_output_buf, 0)) { + if (!append_eager_output(config->alloc, &config->eager_output_buf, 0)) { return false; } } @@ -837,7 +839,7 @@ save_state_eager_outputs(struct cdata_config *config, const struct ir_state_eage config->max_eager_output_id = eager_outputs->ids[i]; } - if (!append_eager_output(&config->eager_output_buf, eager_outputs->ids[i])) { + if (!append_eager_output(config->alloc, &config->eager_output_buf, eager_outputs->ids[i])) { return false; } } @@ -862,11 +864,11 @@ cmp_outgoing(const void *pa, const void *pb) } static bool -append_dst(struct dst_buf *buf, uint32_t dst) +append_dst(const struct fsm_alloc *alloc, struct dst_buf *buf, uint32_t dst) { if (buf->used == buf->ceil) { const size_t nceil = buf->ceil == 0 ? 8 : 2*buf->ceil; - uint32_t *nbuf = realloc(buf->buf, nceil * sizeof(nbuf[0])); + uint32_t *nbuf = f_realloc(alloc, buf->buf, nceil * sizeof(nbuf[0])); if (nbuf == NULL) { return false; } @@ -1009,7 +1011,7 @@ save_state_edge_group_destinations(struct cdata_config *config, struct state_inf const size_t base = dst_buf->used; for (size_t o_i = 0; o_i < outgoing_used; o_i++) { const struct range_info *r = &outgoing[o_i]; - if (!append_dst(dst_buf, r->dst_state)) { + if (!append_dst(config->alloc, dst_buf, r->dst_state)) { return false; } } @@ -1020,13 +1022,14 @@ save_state_edge_group_destinations(struct cdata_config *config, struct state_inf } static bool -populate_config_from_ir(struct cdata_config *config, const struct ir *ir) +populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *alloc, const struct ir *ir) { memset(config, 0x00, sizeof(*config)); + config->alloc = alloc; config->start = ir->start; config->state_count = ir->n; - config->state_info = calloc(ir->n, sizeof(config->state_info[0])); + config->state_info = f_calloc(config->alloc, ir->n, sizeof(config->state_info[0])); assert(config->state_info != NULL); /* could just memset this to 0xff, but this is explicit */ @@ -1133,7 +1136,7 @@ fsm_print_cdata(FILE *f, /* First pass, figure out totals and index sizes */ struct cdata_config config; - populate_config_from_ir(&config, ir); + populate_config_from_ir(&config, alloc, ir); #if LOG_SIZES fprintf(stderr, "// config: dst_state_count %zu, start %d, dst_buf.used %zd, endid_buf.used %zd, eager_output_buf.used %zd\n", @@ -1242,10 +1245,10 @@ fsm_print_cdata(FILE *f, if (!generate_interpreter(f, &config, opt, prefix)) { return -1; } } - free(config.dst_buf.buf); - free(config.endid_buf.buf); - free(config.eager_output_buf.buf); - free(config.state_info); + f_free(alloc, config.dst_buf.buf); + f_free(alloc, config.endid_buf.buf); + f_free(alloc, config.eager_output_buf.buf); + f_free(alloc, config.state_info); return 0; } From 6e9b7f10d4bc9452b06c19546a0f4948cbd19cd4 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sun, 13 Oct 2024 09:31:33 -0400 Subject: [PATCH 22/54] Fix strange comparison on a bool. While technically equivalent, this looks confusnig. I'm not sure if it was a typo (thinking of `eager_output_buf.used > 0`) or some kind of search/replace artifact. --- src/libfsm/print/cdata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 0e2fd53b7..b8ec31234 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -242,7 +242,7 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm id_type_str(config->t_endid_value), config->endid_buf.used); } - if (has_eager_outputs > 0) { + if (has_eager_outputs) { if (comments) { fprintf(f, "\n" From f2ac0826ded9415354fc81e048802e919f1538aa Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 14 Oct 2024 11:03:02 -0400 Subject: [PATCH 23/54] tmp: Add EXPENSIVE_CHECKS wrapper around fsm_exec's is_isdfa check. Normally this doesn't matter, because the expected API use is based on code generation rather than heavily using `fsm_exec`, but we are calling fsm_exec to check that the combined DFAs still match all of their original regexes. This is very expensive. (It similarly gets very cumbersome during fuzzing.) A better way to address this long-term is tracking on the FSM whether it's result of fsm_determinise (and optionally fsm_minimise), and updating that info to indicate that it reverted to an NFA if any other operations modify the FSM states/edges. That's outside the scope of this PR though. --- src/libfsm/exec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfsm/exec.c b/src/libfsm/exec.c index 077494b8f..e74cf7c7c 100644 --- a/src/libfsm/exec.c +++ b/src/libfsm/exec.c @@ -105,10 +105,12 @@ fsm_exec(const struct fsm *fsm, /* TODO: pass struct of callbacks to call during each event; transitions etc */ +#if EXPENSIVE_CHECKS if (!fsm_all(fsm, fsm_isdfa)) { errno = EINVAL; return -1; } +#endif if (!fsm_getstart(fsm, &state)) { errno = EINVAL; From 03033cc7196b2663d9953055d1a432fc7b6433dc Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 14 Oct 2024 11:12:32 -0400 Subject: [PATCH 24/54] cdata: Address warning, ensure closing `}` is always output. Previously `AMBIG_NONE` was missing the closing curly brace but the other variants added it. All three need it, so it's cleaner to move it outside the switch case. Also, make AMBIG_NONE output `(void)endid_base;`, because otherwise it leads to a warning about the unused variable. (It's probably not worth complicating the output template further to avoid using it in the first place here.) --- src/libfsm/print/cdata.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index b8ec31234..16c59d6cd 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -604,21 +604,21 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs switch (opt->ambig) { case AMBIG_NONE: + fprintf(f, + "\t\t(void)endid_base;\n"); break; case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, "\t\t*id = *endid_base;\n" - "\t\t(void)endid_count;\n" - "\t}\n"); + "\t\t(void)endid_count;\n"); break; case AMBIG_MULTIPLE: fprintf(f, "\t\t*%s = endid_base;\n" - "\t\t*%s = endid_count;\n" - "\t}\n", + "\t\t*%s = endid_count;\n", /* TODO: rename these to endid_ids and endid_count? * That will be an interface change. */ "ids", "count"); @@ -628,6 +628,9 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs assert(!"unreached"); abort(); } + + fprintf(f, + "\t}\n"); } /* If the end state has eager_outputs, set their flags. */ From ed2e4d0f3a482c9dca1eec88857a2e3a08ce4119 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Oct 2024 16:06:14 -0400 Subject: [PATCH 25/54] cdata: Move .end and (optional) .endid_offset into a separata array. These fields are only used at the end of input, and moving them into a different struct will make the per-state data accessed during DFA execution more compact. --- src/libfsm/print/cdata.c | 80 +++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 16c59d6cd..f25563e49 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -174,14 +174,10 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm "\ttypedef %s %s_cdata_state;\n", id_type_str(config->t_state_id), prefix); - /* TODO: move .end and .endid_offset into a separate table, they aren't accessed - * until the end of input, so this should slightly reduce cache misses. - * Do this once there's baseline timing info. */ fprintf(f, "\tstruct %s_cdata_dfa {\n" "\t\t%s_cdata_state start;\n" "\t\tstruct %s_cdata_state {\n" - "\t\t\tbool end; /* is this an end state? */\n" "\n", prefix, prefix, prefix); if (comments) { @@ -210,9 +206,6 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm } fprintf(f, "\t\t\t%s dst_table_offset;\n", id_type_str(config->t_dst_state_offset)); - if (has_endids) { - fprintf(f, "\t\t\t%s endid_offset;\n", id_type_str(config->t_endid_offset)); - } if (has_eager_outputs) { fprintf(f, "\t\t\t%s eager_output_offset;\n", id_type_str(config->t_eager_output_offset)); } @@ -221,6 +214,21 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm "\t\t} states[%zd];\n" "\n", config->state_count); + /* .end could be a single uint64_t bitset */ + if (comments) { + fprintf(f, "\t\t/* State-associated info only checked at end of input */\n"); + } + fprintf(f, + "\t\tstruct %s_state_end_info {\n" + "\t\t\tbool end; /* is this an end state? */\n", prefix); + if (has_endids) { + fprintf(f, "\t\t\t%s endid_offset; /* or %zu for NONE */\n", + id_type_str(config->t_endid_offset), config->endid_buf.used); + } + fprintf(f, + "\t\t} state_end_info[%zd];\n" + "\n", config->state_count); + if (comments) { fprintf(f, "\t\t/* Destination states for each edge group in each state,\n" @@ -272,8 +280,6 @@ generate_data(FILE *f, const struct cdata_config *config, const struct ir_state *s = &ir->states[s_i]; const struct state_info *si = &config->state_info[s_i]; - const bool is_end = s->isend; - const bool has_endids = si->endid != STATE_OFFSET_NONE; const bool has_eager_outputs = si->eager_output != STATE_OFFSET_NONE; fprintf(f, "\t\t\t[%zd] = {%s\n", s_i, s_i == config->start ? " /* start */" : ""); @@ -315,7 +321,6 @@ generate_data(FILE *f, const struct cdata_config *config, const size_t state_NONE = config->state_count; const size_t dst_table_NONE = config->dst_buf.used; - const size_t endid_NONE = config->endid_buf.used; const size_t eager_output_NONE = config->eager_output_buf.used; if (si->default_dst == STATE_OFFSET_NONE) { @@ -331,52 +336,61 @@ generate_data(FILE *f, const struct cdata_config *config, fprintf(f, "\t\t\t\t.default_dst = %zu,\n", si->default_dst); } - fprintf(f, "\t\t\t\t.end = %d,\n", is_end); - if (si->dst == STATE_OFFSET_NONE) { /* no non-default outgoing edges */ fprintf(f, "\t\t\t\t.dst_table_offset = %zd, /* NONE */\n", dst_table_NONE); } else { fprintf(f, "\t\t\t\t.dst_table_offset = %zd,\n", si->dst); } - /* Only include these if any state uses endids/eager_outputs, and - * if this state doesn't then use the end of the array as NONE. */ - if (config->endid_buf.used > 0) { - if (has_endids) { + if (config->eager_output_buf.used > 0) { + if (has_eager_outputs) { if (comments) { - fprintf(f, "\t\t\t\t/* endids:"); - for (size_t i = 0; i < s->endids.count; i++) { + fprintf(f, "\t\t\t\t/* eager_outputs:"); + for (size_t i = 0; i < s->eager_outputs->count; i++) { if (i > 0 && (i & 15) == 0) { fprintf(f, "\n\t\t\t\t *"); } - fprintf(f, " %u", s->endids.ids[i]); + fprintf(f, " %u", s->eager_outputs->ids[i]); } fprintf(f, " */\n"); } - fprintf(f, "\t\t\t\t.endid_offset = %zd,\n", si->endid); + fprintf(f, "\t\t\t\t.eager_output_offset = %zd,\n", si->eager_output); } else { - fprintf(f, "\t\t\t\t.endid_offset = %zd, /* NONE */\n", endid_NONE); + fprintf(f, "\t\t\t\t.eager_output_offset = %zd, /* NONE */\n", eager_output_NONE); } } - if (config->eager_output_buf.used > 0) { - if (has_eager_outputs) { + fprintf(f, "\t\t\t},\n"); + } + fprintf(f, "\t\t},\n"); + + fprintf(f, "\t\t.state_end_info = {\n"); + for (size_t s_i = 0; s_i < ir->n; s_i++) { + const size_t endid_NONE = config->endid_buf.used; + const struct ir_state *s = &ir->states[s_i]; + const struct state_info *si = &config->state_info[s_i]; + fprintf(f, "\t\t\t[%zu] = {\n", s_i); + fprintf(f, "\t\t\t\t.end = %d,\n", s->isend); + + /* Only include these if any state uses endids/eager_outputs, and + * if this state doesn't then use the end of the array as NONE. */ + if (config->endid_buf.used > 0) { + if (si->endid != STATE_OFFSET_NONE) { if (comments) { - fprintf(f, "\t\t\t\t/* eager_outputs:"); - for (size_t i = 0; i < s->eager_outputs->count; i++) { + fprintf(f, "\t\t\t\t/* endids:"); + for (size_t i = 0; i < s->endids.count; i++) { if (i > 0 && (i & 15) == 0) { fprintf(f, "\n\t\t\t\t *"); } - fprintf(f, " %u", s->eager_outputs->ids[i]); + fprintf(f, " %u", s->endids.ids[i]); } fprintf(f, " */\n"); } - fprintf(f, "\t\t\t\t.eager_output_offset = %zd,\n", si->eager_output); + fprintf(f, "\t\t\t\t.endid_offset = %zd,\n", si->endid); } else { - fprintf(f, "\t\t\t\t.eager_output_offset = %zd, /* NONE */\n", eager_output_NONE); + fprintf(f, "\t\t\t\t.endid_offset = %zd, /* NONE */\n", endid_NONE); } } - fprintf(f, "\t\t\t},\n"); } fprintf(f, "\t\t},\n"); @@ -574,8 +588,8 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs /* At the end of the input, check if the current state is an end. * If not, there's no match. */ fprintf(f, - "\tconst struct %s_cdata_state *state = &%s_dfa_data.states[cur_state];\n" - "\tif (!state->end) { return 0; /* no match */ }\n", prefix, prefix); + "\tconst struct %s_state_end_info *state_end = &%s_dfa_data.state_end_info[cur_state];\n" + "\tif (!state_end->end) { return 0; /* no match */ }\n", prefix, prefix); /* Set the passed-in reference to the endids, if any. */ if (has_endids) { @@ -586,8 +600,8 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs * by the first lower value. endid_table[] has an extra 0 appended * as a terminator for the last set. */ fprintf(f, - "\tif (state->endid_offset < %s_ENDID_TABLE_COUNT) {\n" - "\t\t%s *endid_scan = &%s_dfa_data.endid_table[state->endid_offset];\n" + "\tif (state_end->endid_offset < %s_ENDID_TABLE_COUNT) {\n" + "\t\t%s *endid_scan = &%s_dfa_data.endid_table[state_end->endid_offset];\n" "\t\tconst %s *endid_base = endid_scan;\n" "\t\tsize_t endid_count = 0;\n" "\t\tuint64_t cur, next;\n" From 32eeaf47f75a2ccf316ce734538c63ef9a2d2702 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Oct 2024 17:14:29 -0400 Subject: [PATCH 26/54] cdata: Intern individual words from 256-bitsets. Each state has two 256-bitsets, stored as a uint64_t[4], but the individual words in those have a lot of duplication. Add a table with every unique word, sorted descending by frequency, and replace the per-state labels and label_group_starts arrays with an array of offsets into the label_word table. Typically these offsets will fit in a uint8_t (though the code generation will switch to a uint16_t when necessary), making the per-state data much smaller. The label_word table's most commonly used entries are all grouped together and should stay in cache. --- src/libfsm/print/cdata.c | 157 +++++++++++++++++++++++++++++++++++---- 1 file changed, 141 insertions(+), 16 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index f25563e49..712711e06 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -120,6 +120,9 @@ struct cdata_config { enum id_type t_endid_value; enum id_type t_eager_output_value; + /* numeric type for entries in .bitset_words.pairs[] */ + enum id_type t_label_word_id; + struct dst_buf { size_t ceil; size_t used; @@ -154,6 +157,17 @@ struct cdata_config { size_t eager_output; } *state_info; + /* Collected 64-bit word counts for the .labels and + * .label_group_starts 256-bitsets. */ + struct bitset_words { + size_t used; + size_t ceil; + struct bitset_word_pair { + uint64_t word; + size_t count; + } *pairs; + } bitset_words; + #if LOG_REUSE struct reuse_stats { size_t miss; @@ -183,11 +197,11 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm if (comments) { fprintf(f, "\t\t\t/* To find the destination state for label character C,\n" - "\t\t\t * check if the bit C is set in .labels[]. If so, find the\n" - "\t\t\t * the 1 bit at or preceding C in .label_group_starts[],\n" - "\t\t\t * which represents the start of the Nth label group, the\n" - "\t\t\t * group label group that contains C. The dst state will be in\n" - "\t\t\t * .dst_table[.dst_table_offset + N]. This offset N is called\n" + "\t\t\t * check if the bit C is set in the word id'd by .labels[].\n" + "\t\t\t * If so, find the the 1 bit at or preceding C in\n" + "\t\t\t * .label_group_starts[], which represents the start of the Nth\n" + "\t\t\t * label group, the group label group that contains C. The dst state will\n" + "\t\t\t * be in .dst_table[.dst_table_offset + N]. This offset N is called\n" "\t\t\t * the rank, and .rank_sums has precomputed sums for each\n" "\t\t\t * word preceding .label_group_starts[C/64]. If .labels[]\n" "\t\t\t * isn't set for C, the destination is .default_dst, or the\n" @@ -196,10 +210,13 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm } fprintf(f, "\t\t\t%s_cdata_state default_dst; /* or %zu for NONE */\n" - "\t\t\tuint64_t labels[256/4]; /* which labels have non-default edges */\n" - "\t\t\tuint64_t label_group_starts[256/4]; /* start of each label group */\n" + "\t\t\t%s label_word_ids[4]; /* which labels have non-default edges */\n" + "\t\t\t%s label_group_start_word_ids[4]; /* start of each label group */\n" "\t\t\tuint8_t rank_sums[4]; /* rank at end of label_group_starts[n] */\n" - "\n", prefix, config->state_count); + "\n", + prefix, config->state_count, + id_type_str(config->t_label_word_id), + id_type_str(config->t_label_word_id)); if (comments) { fprintf(f, "\t\t\t/* Offsets into values in other tables */\n"); @@ -229,6 +246,14 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm "\t\t} state_end_info[%zd];\n" "\n", config->state_count); + if (comments) { + fprintf(f, + "\t\t/* Table of individual words used in label and label_group_start\n" + "\t\t * bitsets, in descending order by frequency. */\n"); + } + fprintf(f, + "\t\tuint64_t label_word_table[%zd];\n", config->bitset_words.used); + if (comments) { fprintf(f, "\t\t/* Destination states for each edge group in each state,\n" @@ -267,6 +292,22 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm return true; } +static void +lookup_label_ids(const struct cdata_config *config, const uint64_t *labels, unsigned ids[4]) { + for (size_t w_i = 0; w_i < 4; w_i++) { + bool found = false; + const uint64_t w = labels[w_i]; + for (size_t i = 0; i < config->bitset_words.used; i++) { + if (config->bitset_words.pairs[i].word == w) { + ids[w_i] = i; + found = true; + break; + } + } + assert(found); + } +} + static bool generate_data(FILE *f, const struct cdata_config *config, bool comments, const char *prefix, const struct ir *ir) @@ -295,8 +336,10 @@ generate_data(FILE *f, const struct cdata_config *config, } fprintf(f, "\n"); } - fprintf(f, "\t\t\t\t.labels = { 0x%lx, 0x%lx, 0x%lx, 0x%lx },\n", - si->labels[0], si->labels[1], si->labels[2], si->labels[3]); + unsigned label_ids[4]; + lookup_label_ids(config, si->labels, label_ids); + fprintf(f, "\t\t\t\t.label_word_ids = { %u, %u, %u, %u },\n", + label_ids[0], label_ids[1], label_ids[2], label_ids[3]); size_t dst_count = 0; if (comments) { @@ -311,8 +354,9 @@ generate_data(FILE *f, const struct cdata_config *config, } fprintf(f, "\n"); } - fprintf(f, "\t\t\t\t.label_group_starts = { 0x%lx, 0x%lx, 0x%lx, 0x%lx },\n", - si->label_group_starts[0], si->label_group_starts[1], si->label_group_starts[2], si->label_group_starts[3]); + lookup_label_ids(config, si->label_group_starts, label_ids); + fprintf(f, "\t\t\t\t.label_group_start_word_ids = { %u, %u, %u, %u },\n", + label_ids[0], label_ids[1], label_ids[2], label_ids[3]); /* rank_sums[0] is always 0, but allows us to avoid a subtraction in the inner loop, * and the space would be wasted otherwise anyway due to alignment. */ @@ -395,6 +439,20 @@ generate_data(FILE *f, const struct cdata_config *config, } fprintf(f, "\t\t},\n"); + fprintf(f, + "\t\t.label_word_table = {\n\t\t\t"); + for (size_t i = 0; i < config->bitset_words.used; i++) { + fprintf(f, " 0x%016lx,", config->bitset_words.pairs[i].word); + if ((i & 3) == 3) { + fprintf(f, "\n\t\t\t"); + } + } + if ((config->bitset_words.used & 3) != 3) { + fprintf(f, "\n"); + } + + fprintf(f, "\t\t},\n"); + fprintf(f, "\t\t.dst_table = {"); @@ -554,11 +612,14 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs "\t\tconst size_t w_i = c/64;\n" "\t\tconst size_t word_rem = c & 63;\n" "\t\tconst uint64_t bit = (uint64_t)1 << word_rem;\n" - "\t\tif (state->labels[w_i] & bit) { /* if state has label */\n" + "\t\tconst %s label_word_id = state->label_word_ids[w_i];\n" + "\t\tconst uint64_t label_word = %s_dfa_data.label_word_table[label_word_id];\n" + "\t\tif (label_word & bit) { /* if state has label */\n" "\t\t\tif (debug_traces) {\n" "\t\t\t\tfprintf(stderr, \"-- label '%%c' (0x%%02x) -> w_i %%zd, bit 0x%%016lx\\n\", isprint(c) ? c : 'c', c, w_i, bit);\n" "\t\t\t}\n" - "\t\t\tconst uint64_t lgs_word = state->label_group_starts[w_i];\n" + "\t\t\tconst uint64_t lgs_word_id = state->label_group_start_word_ids[w_i];\n" + "\t\t\tconst uint64_t lgs_word = %s_dfa_data.label_word_table[lgs_word_id];\n" "\t\t\tconst size_t back = (lgs_word & bit) ? 0 : 1; /* back to start of label group */\n" "\t\t\tconst uint64_t mask = bit - 1;\n" "\t\t\tconst uint64_t masked_word = lgs_word & mask;\n" @@ -583,7 +644,8 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs "\t\t\treturn 0; /* no match */\n" "\t\t}\n" "\t}\n", - popcount, prefix, prefix); + id_type_str(config->t_label_word_id), + prefix, prefix, popcount, prefix, prefix); /* At the end of the input, check if the current state is an end. * If not, there's no match. */ @@ -648,7 +710,13 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs } /* If the end state has eager_outputs, set their flags. */ - generate_eager_output_check(f, config, prefix); + if (has_eager_outputs) { + fprintf(f, "\t{\n"); /* add {} nesting so the tabs match */ + fprintf(f, "\t\tconst struct %s_cdata_state *state = &%s_dfa_data.states[cur_state];\n", + prefix, prefix); + generate_eager_output_check(f, config, prefix); + fprintf(f, "\t}\n"); + } /* Got a match. */ fprintf(f, "\treturn 1; /* match */\n"); @@ -898,6 +966,42 @@ append_dst(const struct fsm_alloc *alloc, struct dst_buf *buf, uint32_t dst) return true; } +static int +cmp_bitset_word_pair(const void *pa, const void *pb) +{ + const struct bitset_word_pair *a = (const struct bitset_word_pair *)pa; + const struct bitset_word_pair *b = (const struct bitset_word_pair *)pb; + + /* for sorting by descending count */ + return a->count < b->count ? 1 : a->count > b->count ? -1 : 0; +} + +static void +increment_bitset_word_count(const struct fsm_alloc *alloc, struct bitset_words *bws, uint64_t w) +{ + /* This table tends to stay fairly small, so linear search is probably good enough. */ + for (size_t i = 0; i < bws->used; i++) { + if (bws->pairs[i].word == w) { + bws->pairs[i].count++; + return; + } + } + + if (bws->used == bws->ceil) { + const size_t nceil = (bws->ceil == 0 ? 8 : 2*bws->ceil); + struct bitset_word_pair *npairs = f_realloc(alloc, + bws->pairs, nceil * sizeof(npairs[0])); + assert(npairs != NULL); + bws->ceil = nceil; + bws->pairs = npairs; + } + + struct bitset_word_pair *p = &bws->pairs[bws->used]; + p->word = w; + p->count = 1; + bws->used++; +} + static bool save_state_edge_group_destinations(struct cdata_config *config, struct state_info *si, size_t group_count, const struct ir_group *groups) @@ -966,6 +1070,12 @@ save_state_edge_group_destinations(struct cdata_config *config, struct state_inf } } + struct bitset_words *bws = &config->bitset_words; + for (size_t i = 0; i < 4; i++) { + increment_bitset_word_count(config->alloc, bws, si->labels[i]); + increment_bitset_word_count(config->alloc, bws, si->label_group_starts[i]); + } + /* Precompute label_group_starts[] rank sums so lookup only needs to * compute rank for the label's word, not every word preceding it. */ si->rank_sums[0] = 0; @@ -1058,6 +1168,14 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all si->eager_output = STATE_OFFSET_NONE; } + /* add a single entry for 0, in case the IR only has a single IR_NONE or IR_SAME state */ + config->bitset_words.ceil = 1; + config->bitset_words.used = 1; + config->bitset_words.pairs = calloc(1, sizeof(config->bitset_words.pairs[0])); + assert(config->bitset_words.pairs != NULL); + config->bitset_words.pairs[0].word = 0x0; + config->bitset_words.pairs[0].count = 1; + for (size_t s_i = 0; s_i < ir->n; s_i++) { const struct ir_state *s = &ir->states[s_i]; @@ -1124,6 +1242,12 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all * a pointer to the array of IDs. */ config->t_endid_value = UNSIGNED; //size_needed(config->max_endid); + /* Sort by use frequency, descending, so the most frequently used + * bitset words will stay in cache. */ + qsort(config->bitset_words.pairs, config->bitset_words.used, + sizeof(config->bitset_words.pairs[0]), cmp_bitset_word_pair); + config->t_label_word_id = size_needed(config->bitset_words.used); + config->t_eager_output_value = size_needed(config->max_eager_output_id); return true; @@ -1265,6 +1389,7 @@ fsm_print_cdata(FILE *f, f_free(alloc, config.dst_buf.buf); f_free(alloc, config.endid_buf.buf); f_free(alloc, config.eager_output_buf.buf); + f_free(alloc, config.bitset_words.pairs); f_free(alloc, config.state_info); return 0; From af0ae65bcf0ad6113fc52ea61cfb4adcd294d898 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 7 Nov 2024 17:16:37 -0500 Subject: [PATCH 27/54] Fix memory leak. The edge sets leak when halting with FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED. --- src/libfsm/determinise.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 9483218a1..9833fd878 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -240,6 +240,7 @@ fsm_determinise_with_config(struct fsm *nfa, assert(dfa->states[m->dfastate].edges == NULL); dfa->states[m->dfastate].edges = m->edges; + m->edges = NULL; /* transfer ownership */ /* * The current DFA state is an end state if any of its associated NFA @@ -616,6 +617,8 @@ map_free(struct map *map) if (b == NULL) { continue; } + /* free any edge sets that didn't get transferred */ + edge_set_free(map->alloc, b->edges); f_free(map->alloc, b); } From c5342a89d995c52327d51649d1c02a951299d04d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 9 Dec 2024 11:48:47 -0500 Subject: [PATCH 28/54] cdata: Buffer eager_outputs until match, rather than setting imediately. Previously, the CDATA codegen wrote eager_outputs directly into the caller's match bit buffer as they were encountered. Instead, set them in a stack-allocated buffer, and then copy them to the caller's if the DFA match succeeds overall. In order to avoid repeatedly checking for whether an eager_output has already been set (in the buffer), this collects the set of all distinct eager_output IDs and then remaps the array with eager output IDs to offsets into the unique set. This condenses the (sparse) set into a dense series 0..n that can be represented by flags in a stack-allocated bit vector (with a size known at compile time), and redundant eager_outputs harmlessly set flag bits that are already set. If the overall match succeeds, that bit vector is matched up with the unique ID array and the sparse values are written into the caller's buffer. Because the unique ID array is sorted, the relative ordering of the sparse and dense IDs is preserved (and 0 stays 0), so using non-ascending values as terminators still works. --- src/libfsm/print/cdata.c | 174 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 166 insertions(+), 8 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 712711e06..447dfb48c 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -25,6 +25,8 @@ #include #include +#include + #include "libfsm/internal.h" #include "libfsm/print.h" @@ -34,7 +36,6 @@ * This mostly exists to sidestep very expensive compilation for large data sets * using the other C modes. -sv */ - /* Whether to check the table buffers for previous instances of * the same sets of dst_states, endids, and eager_outputs. This * should always be an improvement, making the generated code @@ -119,6 +120,7 @@ struct cdata_config { /* numeric types for endid and eager_output table values */ enum id_type t_endid_value; enum id_type t_eager_output_value; + enum id_type t_distinct_eager_output_offset; /* numeric type for entries in .bitset_words.pairs[] */ enum id_type t_label_word_id; @@ -143,6 +145,9 @@ struct cdata_config { } eager_output_buf; size_t max_eager_output_id; + fsm_output_id_t *eager_output_ids; + size_t distinct_eager_output_id_count; + struct state_info { uint64_t labels[256/64]; uint64_t label_group_starts[256/64]; @@ -284,7 +289,18 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm "\t\t * terminated by non-increasing value. */\n"); } fprintf(f, "\t\t%s eager_output_table[%zd + 1];\n", - id_type_str(config->t_eager_output_value), config->eager_output_buf.used); + id_type_str(config->t_distinct_eager_output_offset), config->eager_output_buf.used); + + if (comments) { + fprintf(f, + "\n" + "\t\t/* All distinct eager_output ID that appear in the DFA,\n" + "\t\t * in ascending order. Used to convert the dense bitset for\n" + "\t\t * eager_outputs reached one or more times with their values. */\n"); + } + fprintf(f, "\t\t%s eager_output_ids[%zd + 1];\n", + id_type_str(config->t_eager_output_value), + config->distinct_eager_output_id_count); } fprintf(f, @@ -308,6 +324,94 @@ lookup_label_ids(const struct cdata_config *config, const uint64_t *labels, unsi } } +static int +cmp_eager_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(const fsm_output_id_t *)pa; + const fsm_output_id_t b = *(const fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static bool +collect_distinct_eager_output_ids(struct cdata_config *config, + const struct ir *ir) +{ + bool res = false; + fsm_output_id_t *id_array = NULL; + + /* state set, used as an fsm_output_id_t set. Both are unsigned int. */ + struct state_set *ids = NULL; + + if (config->eager_output_buf.used > 0) { + /* Add value of 0 at offset 0, because it's used as a list terminator. */ + if (!state_set_add(&ids, config->alloc, 0)) { + goto cleanup; + } + } + + for (size_t s_i = 0; s_i < ir->n; s_i++) { + const struct ir_state *s = &ir->states[s_i]; + struct ir_state_eager_output *eos = s->eager_outputs; + if (eos == NULL) { continue; } + + for (size_t id_i = 0; id_i < eos->count; id_i++) { + const fsm_output_id_t id = eos->ids[id_i]; + /* fprintf(stderr, "%s: state %zu, id %u\n", __func__, s_i, id); */ + if (!state_set_add(&ids, config->alloc, (fsm_state_t)id)) { + goto cleanup; + } + } + } + + config->distinct_eager_output_id_count = 0; + config->eager_output_ids = NULL; + + const size_t distinct_eager_output_ids = state_set_count(ids); + if (distinct_eager_output_ids == 0) { + state_set_free(ids); + return true; /* nothing to do */ + } + + id_array = f_malloc(config->alloc, distinct_eager_output_ids * sizeof(id_array[0])); + if (id_array == NULL) { goto cleanup; } + + struct state_iter iter; + state_set_reset(ids, &iter); + fsm_state_t s_id; + while (state_set_next(&iter, &s_id)) { + id_array[config->distinct_eager_output_id_count++] = (fsm_output_id_t)s_id; + } + assert(config->distinct_eager_output_id_count == distinct_eager_output_ids); + qsort(id_array, distinct_eager_output_ids, sizeof(id_array[0]), cmp_eager_output_id); + + config->eager_output_ids = id_array; + /* fprintf(stderr, "/// distinct eager outputs:\n"); + * for (size_t i = 0; i < config->distinct_eager_output_id_count; i++) { + * fprintf(stderr, "-- %zu: %zu\n", i, (size_t)id_array[i]); + * } */ + + state_set_free(ids); + return true; + +cleanup: + f_free(config->alloc, id_array); + state_set_free(ids); + return res; +} + +static void +generate_distinct_eager_output_id_table(FILE *f, const struct cdata_config *config) +{ + assert(config->distinct_eager_output_id_count > 0); + + fprintf(f, "\t\t.eager_output_ids = {\n\t\t\t"); + for (size_t i = 0; i < config->distinct_eager_output_id_count; i++) { + fprintf(f, " %u,", config->eager_output_ids[i]); + if ((i & 15) == 15) { fprintf(f, "\n\t\t\t"); } + } + fprintf(f, "\n\t\t},\n"); +} + static bool generate_data(FILE *f, const struct cdata_config *config, bool comments, const char *prefix, const struct ir *ir) @@ -480,12 +584,38 @@ generate_data(FILE *f, const struct cdata_config *config, if (config->eager_output_buf.used > 0) { fprintf(f, "\t\t.eager_output_table = {"); + for (size_t i = 0; i < config->eager_output_buf.used; i++) { if ((i & 15) == 0) { fprintf(f, "\n\t\t\t"); } - fprintf(f, " %lu,", config->eager_output_buf.buf[i]); + /* TODO This uses linear search and does redundant lookups, but + * the distinct eager_output IDs should be fairly small. + * This could be replaced with an id->offset map later. + * + * Note: Non-ascending values are used as a list terminator in + * both the original and remapped sets -- because 0 was added + * at the start of collect_distinct_eager_output_ids, it will + * stay as 0, and since the list of distinct IDs is sorted the + * mapping will preserve the relative order -- non-ascending + * values terminating runs of IDs will still be non-ascending. */ + fsm_output_id_t id = config->eager_output_buf.buf[i]; + unsigned id_offset = (unsigned)-1; + for (unsigned i = 0; i < config->distinct_eager_output_id_count; i++) { + if (config->eager_output_ids[i] == id) { + id_offset = i; + break; + } + } + assert(id_offset != (unsigned)-1); /* found */ + fprintf(f, " %u,", id_offset); } fprintf(f, "\n\t\t\t 0 /* end */,\n"); fprintf(f, "\n\t\t},\n"); + + /* Emit a sorted table of all the distinct eager output + * values. This, along with a bitset stack-allocated by + * the interpreter, will be used to keep track of which + * eager outputs should be set if the DFA as a whole matches. */ + generate_distinct_eager_output_id_table(f, config); } fprintf(f, "\t};\n"); @@ -513,15 +643,15 @@ generate_eager_output_check(FILE *f, const struct cdata_config *config, const ch "\t\t\t\tif (debug_traces) {\n" "\t\t\t\t\tfprintf(stderr, \"%%s: setting eager_output flag %%u\\n\", __func__, cur);\n" "\t\t\t\t}\n" - "\t\t\t\teager_output_buf[cur/64] |= (uint64_t)1 << (cur & 63);\n" + "\t\t\t\tuncommitted_eager_output_buf[cur/64] |= (uint64_t)1 << (cur & 63);\n" "\t\t\t\teo_scan++;\n" "\t\t\t\tnext = *eo_scan;\n" "\t\t\t} while (next > cur);\n" "\t\t}\n", prefix, - id_type_str(config->t_eager_output_value), + id_type_str(config->t_distinct_eager_output_offset), prefix, - id_type_str(config->t_eager_output_value)); + id_type_str(config->t_distinct_eager_output_offset)); } } @@ -545,6 +675,15 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs fprintf(f, "\tuint32_t cur_state = %s_dfa_data.start;\n", prefix); fprintf(f, "\n"); + /* Stack-allocated bitset buffer for uncommitted eager_outputs: if the Nth bit in this + * buffer is set, then after a successful DFA match the ID in dfa.eager_output_ids[N] + * should be reported as matching. The bitset is used to collect redundant eager_outputs, + * and to avoid reporting them to the caller until the overall DFA match result is known. */ + if (config->distinct_eager_output_id_count > 0) { + const size_t eager_output_words = config->distinct_eager_output_id_count/64 + 1; + fprintf(f, "\tuint64_t uncommitted_eager_output_buf[%zu] = {0};\n", eager_output_words); + } + /* Setting this to true will log out execution steps. */ const bool debug_traces = false; fprintf(f, @@ -716,6 +855,19 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs prefix, prefix); generate_eager_output_check(f, config, prefix); fprintf(f, "\t}\n"); + + /* commit eager_output matches to the caller's buffer */ + const size_t eager_output_words = config->distinct_eager_output_id_count/64 + 1; + fprintf(f, "\tfor (size_t w_i = 0; w_i < %zu; w_i++) {\n", eager_output_words); + fprintf(f, "\t\tconst uint64_t w = uncommitted_eager_output_buf[w_i];\n"); + fprintf(f, "\t\tif (w == 0) { continue; }\n"); + fprintf(f, "\t\tfor (size_t bit = 1; bit < 64; bit++) {\n"); + fprintf(f, "\t\t\tif (w & ((uint64_t)1 << bit)) {\n"); + fprintf(f, "\t\t\t\tconst uint64_t id = %s_dfa_data.eager_output_ids[64*w_i + bit];\n", prefix); + fprintf(f, "\t\t\t\teager_output_buf[id/64] |= ((uint64_t)1 << (id & 63));\n"); + fprintf(f, "\t\t\t}\n"); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t}\n"); } /* Got a match. */ @@ -776,8 +928,8 @@ save_state_endids(struct cdata_config *config, const struct ir_state_endids *end } } - /* If there's a potential match, check that it isn't followed by - * a value > the last, because it would falsely continue the run. */ + /* If there's a potential match, check that it is followed by + * a value <= the last, because otherwise it would falsely continue the run. */ if (e_i == endids->count && b_i + e_i + 1 < config->endid_buf.used && config->endid_buf.buf[b_i + e_i + 1] <= endids->ids[e_i - 1]) { @@ -1223,6 +1375,10 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all } } + if (!collect_distinct_eager_output_ids(config, ir)) { + goto alloc_fail; + } + /* Get the smallest numeric type that will fit all state IDs in * the current DFA, reserving one extra to use as an out-of-band * "NONE" value for fields like default_dst. These also the values @@ -1249,6 +1405,7 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all config->t_label_word_id = size_needed(config->bitset_words.used); config->t_eager_output_value = size_needed(config->max_eager_output_id); + config->t_distinct_eager_output_offset = size_needed(config->distinct_eager_output_id_count); return true; not_implemented: @@ -1391,6 +1548,7 @@ fsm_print_cdata(FILE *f, f_free(alloc, config.eager_output_buf.buf); f_free(alloc, config.bitset_words.pairs); f_free(alloc, config.state_info); + f_free(alloc, config.eager_output_ids); return 0; } From 8cd564b2b8ee9e345a8fb1b42058ca5b7533f9f7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 17 Jan 2025 13:34:31 -0500 Subject: [PATCH 29/54] Fix a bug in the CDATA interpreter's bitset checking. Because of the for loop init condition, this was checking bits `1 << 1..64` in every word -- it was unintentionally ignoring the least significant bit. It should check `1 << 0..64`. --- src/libfsm/print/cdata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 447dfb48c..1f209df7c 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -861,7 +861,7 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs fprintf(f, "\tfor (size_t w_i = 0; w_i < %zu; w_i++) {\n", eager_output_words); fprintf(f, "\t\tconst uint64_t w = uncommitted_eager_output_buf[w_i];\n"); fprintf(f, "\t\tif (w == 0) { continue; }\n"); - fprintf(f, "\t\tfor (size_t bit = 1; bit < 64; bit++) {\n"); + fprintf(f, "\t\tfor (size_t bit = 0; bit < 64; bit++) {\n"); fprintf(f, "\t\t\tif (w & ((uint64_t)1 << bit)) {\n"); fprintf(f, "\t\t\t\tconst uint64_t id = %s_dfa_data.eager_output_ids[64*w_i + bit];\n", prefix); fprintf(f, "\t\t\t\teager_output_buf[id/64] |= ((uint64_t)1 << (id & 63));\n"); From 6aa1962ef5ee5d7152ec3d9c0217ec5530838589 Mon Sep 17 00:00:00 2001 From: Ricky Hosfelt Date: Fri, 7 Feb 2025 14:45:17 -0500 Subject: [PATCH 30/54] update CI to lock to ubuntu 22.04 for clang 14.0.0 --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfb6182f2..35fa5c7f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: checkout: name: "Checkout" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Cache checkout @@ -44,7 +44,7 @@ jobs: pcre_suite: name: "Import PCRE suite" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ build ] # for cvtpcre steps: @@ -130,7 +130,7 @@ jobs: build: name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ checkout ] strategy: @@ -216,7 +216,7 @@ jobs: # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ checkout, build ] strategy: @@ -301,7 +301,7 @@ jobs: test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ build ] strategy: @@ -362,7 +362,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] @@ -470,7 +470,7 @@ jobs: test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ pcre_suite ] # and also build, but pcre_suite gives us that strategy: @@ -531,7 +531,7 @@ jobs: docs: name: Documentation - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ checkout ] env: @@ -582,7 +582,7 @@ jobs: install: name: Install - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ build, docs ] env: @@ -643,7 +643,7 @@ jobs: fpm: name: Package ${{ matrix.pkg }} - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ install ] env: From 924cb38479cc51277dc3133b935ee2c498e1870d Mon Sep 17 00:00:00 2001 From: Ricky Hosfelt Date: Fri, 7 Feb 2025 14:59:07 -0500 Subject: [PATCH 31/54] keep matrix.os so we can easily add more os variations in the future --- .github/workflows/ci.yml | 51 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35fa5c7f4..157506a14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,10 @@ jobs: pcre_suite: name: "Import PCRE suite" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-22.04 ] needs: [ build ] # for cvtpcre steps: @@ -71,7 +74,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -79,7 +82,7 @@ jobs: id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitrary build, just for cvtpcre - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -130,14 +133,14 @@ jobs: build: name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ checkout ] strategy: fail-fast: true matrix: san: [ NO_SANITIZER, AUSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -169,7 +172,7 @@ jobs: key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && steps.cache-build.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-22.04' && steps.cache-build.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake electric-fence @@ -216,14 +219,14 @@ jobs: # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ checkout, build ] strategy: fail-fast: false matrix: san: [ NO_SANITIZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake, pmake ] debug: [ EXPENSIVE_CHECKS, DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -258,7 +261,7 @@ jobs: run: find ${{ env.wc }} -type f -name '*.c' | sort -r | head -5 | xargs touch - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install pmake bmake pcregrep @@ -301,14 +304,14 @@ jobs: test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ build ] strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -329,7 +332,7 @@ jobs: key: checkout-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake pcregrep electric-fence @@ -362,7 +365,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] @@ -370,7 +373,7 @@ jobs: fail-fast: false matrix: san: [ FUZZER ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -388,7 +391,7 @@ jobs: key: checkout-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake @@ -470,14 +473,14 @@ jobs: test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ pcre_suite ] # and also build, but pcre_suite gives us that strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -492,7 +495,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && matrix.san == 'EFENCE' + if: matrix.os == 'ubuntu-22.04' && matrix.san == 'EFENCE' run: | uname -a sudo apt-get install electric-fence @@ -506,7 +509,7 @@ jobs: ${{ matrix.cc }} --version - name: Dependencies (Ubuntu/Go) - if: matrix.os == 'ubuntu' && (matrix.lang == 'go' || matrix.lang == 'goasm') + if: matrix.os == 'ubuntu-22.04' && (matrix.lang == 'go' || matrix.lang == 'goasm') run: | uname -a sudo apt-get install golang @@ -524,7 +527,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst @@ -587,7 +590,7 @@ jobs: env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -601,7 +604,7 @@ jobs: key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' && steps.cache-prefix.outputs.cache-hit != 'true' + if: env.os == 'ubuntu-22.04' && steps.cache-prefix.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake @@ -648,7 +651,7 @@ jobs: env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -661,7 +664,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' + if: env.os == 'ubuntu-22.04' run: | uname -a sudo gem install --no-document fpm From a99b9137424f44c760241447fc361d392fbd0dc7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 16 Jun 2025 15:28:45 -0400 Subject: [PATCH 32/54] cdata: Instead of asserting, the print interface returns -1 on error. Bubble up the allocation failure errors. Use `f_calloc` rather than `calloc`. --- src/libfsm/print/cdata.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 1f209df7c..29cf121cb 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -1309,7 +1309,9 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all config->state_count = ir->n; config->state_info = f_calloc(config->alloc, ir->n, sizeof(config->state_info[0])); - assert(config->state_info != NULL); + if (config->state_info == NULL) { + goto alloc_fail; + } /* could just memset this to 0xff, but this is explicit */ for (size_t s_i = 0; s_i < ir->n; s_i++) { @@ -1323,8 +1325,10 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all /* add a single entry for 0, in case the IR only has a single IR_NONE or IR_SAME state */ config->bitset_words.ceil = 1; config->bitset_words.used = 1; - config->bitset_words.pairs = calloc(1, sizeof(config->bitset_words.pairs[0])); - assert(config->bitset_words.pairs != NULL); + config->bitset_words.pairs = f_calloc(config->alloc, 1, sizeof(config->bitset_words.pairs[0])); + if (config->bitset_words.pairs == NULL) { + goto alloc_fail; + } config->bitset_words.pairs[0].word = 0x0; config->bitset_words.pairs[0].count = 1; @@ -1413,7 +1417,6 @@ populate_config_from_ir(struct cdata_config *config, const struct fsm_alloc *all return false; alloc_fail: - assert(!"alloc fail"); return false; } @@ -1434,7 +1437,9 @@ fsm_print_cdata(FILE *f, /* First pass, figure out totals and index sizes */ struct cdata_config config; - populate_config_from_ir(&config, alloc, ir); + if (!populate_config_from_ir(&config, alloc, ir)) { + return -1; + } #if LOG_SIZES fprintf(stderr, "// config: dst_state_count %zu, start %d, dst_buf.used %zd, endid_buf.used %zd, eager_output_buf.used %zd\n", From 050969689ef2abf5ee0fe7797e0df85279e88343 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 16 Jun 2025 15:38:10 -0400 Subject: [PATCH 33/54] cdata: Bubble up other allocation errors. --- src/libfsm/print/cdata.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 29cf121cb..4ca715452 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -1128,14 +1128,14 @@ cmp_bitset_word_pair(const void *pa, const void *pb) return a->count < b->count ? 1 : a->count > b->count ? -1 : 0; } -static void +static bool increment_bitset_word_count(const struct fsm_alloc *alloc, struct bitset_words *bws, uint64_t w) { /* This table tends to stay fairly small, so linear search is probably good enough. */ for (size_t i = 0; i < bws->used; i++) { if (bws->pairs[i].word == w) { bws->pairs[i].count++; - return; + return true; } } @@ -1143,7 +1143,7 @@ increment_bitset_word_count(const struct fsm_alloc *alloc, struct bitset_words * const size_t nceil = (bws->ceil == 0 ? 8 : 2*bws->ceil); struct bitset_word_pair *npairs = f_realloc(alloc, bws->pairs, nceil * sizeof(npairs[0])); - assert(npairs != NULL); + return false; bws->ceil = nceil; bws->pairs = npairs; } @@ -1152,6 +1152,7 @@ increment_bitset_word_count(const struct fsm_alloc *alloc, struct bitset_words * p->word = w; p->count = 1; bws->used++; + return true; } static bool @@ -1224,8 +1225,12 @@ save_state_edge_group_destinations(struct cdata_config *config, struct state_inf struct bitset_words *bws = &config->bitset_words; for (size_t i = 0; i < 4; i++) { - increment_bitset_word_count(config->alloc, bws, si->labels[i]); - increment_bitset_word_count(config->alloc, bws, si->label_group_starts[i]); + if (!increment_bitset_word_count(config->alloc, bws, si->labels[i])) { + return false; + } + if (!increment_bitset_word_count(config->alloc, bws, si->label_group_starts[i])) { + return false; + } } /* Precompute label_group_starts[] rank sums so lookup only needs to From c783ce8d775f59c0f4ffdcbd4feba486f1d60ac0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 31 Jan 2025 15:08:48 -0500 Subject: [PATCH 34/54] Rename test to _single, add _multiple variant. These test combining `[""]` and `["", ""]`. --- .../eager_output/eager_output_at_start_multiple.c | 15 +++++++++++++++ ..._at_start.c => eager_output_at_start_single.c} | 0 2 files changed, 15 insertions(+) create mode 100644 tests/eager_output/eager_output_at_start_multiple.c rename tests/eager_output/{eager_output_at_start.c => eager_output_at_start_single.c} (100%) diff --git a/tests/eager_output/eager_output_at_start_multiple.c b/tests/eager_output/eager_output_at_start_multiple.c new file mode 100644 index 000000000..5c6f4577c --- /dev/null +++ b/tests/eager_output/eager_output_at_start_multiple.c @@ -0,0 +1,15 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + [0] = "", + [1] = "", + }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_at_start.c b/tests/eager_output/eager_output_at_start_single.c similarity index 100% rename from tests/eager_output/eager_output_at_start.c rename to tests/eager_output/eager_output_at_start_single.c From 0b601deba84bb09ff68d8a43875e2d89d6e6ead8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 31 Jan 2025 15:09:24 -0500 Subject: [PATCH 35/54] Fix memory leak in fsm_eager_output_compact, found while fuzzing. --- src/libfsm/eager_output.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c index e00e96cd1..00fa1b5f0 100644 --- a/src/libfsm/eager_output.c +++ b/src/libfsm/eager_output.c @@ -408,7 +408,10 @@ fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_c assert(ob->state < mapping_count); const fsm_state_t nstate = mapping[ob->state]; - if (nstate == FSM_STATE_REMAP_NO_STATE) { continue; } + if (nstate == FSM_STATE_REMAP_NO_STATE) { + f_free(fsm->alloc, ob->entry); + continue; + } const uint64_t hash = hash_id(nstate); From efaa884e8b6752d9b9adc279e1905b458415ccce Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 4 Feb 2025 12:45:08 -0500 Subject: [PATCH 36/54] Fix fsm_union_repeated_pattern_group's anchoring linkage. Previously this didn't handle mixed anchoring correctly, potentially leading to false positives the case represented by eager_output_alt_mixing_anchored_and_unanchored.c. See comments in fsm_union_repeated_pattern_group for details. Fuzzing did not turn up any new issues. Another commit after this will make a few small interface changes and update callers. --- src/libfsm/union.c | 751 ++++++++++++++---- ...utput_alt_mixing_anchored_and_unanchored.c | 29 + .../eager_output_at_start_multiple_anchored.c | 15 + .../eager_output_at_start_single_anchored.c | 22 + tests/eager_output/utils.c | 90 +-- 5 files changed, 716 insertions(+), 191 deletions(-) create mode 100644 tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c create mode 100644 tests/eager_output/eager_output_at_start_multiple_anchored.c create mode 100644 tests/eager_output/eager_output_at_start_single_anchored.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 126181992..ed71384ff 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -21,9 +21,54 @@ #include "internal.h" #include +#include + #include "eager_output.h" +#include "endids.h" #define LOG_UNION_ARRAY 0 +#define LOG_ANALYZE_GROUP_NFA_RESULTS 0 +#define LOG_UNION_REPEATED_PATTERN_GROUP 0 + +#define NO_STATE (fsm_state_t)-1 + +/* State/edge info gathered about an NFA. Used by fsm_union_repeated_pattern_group. */ +struct analysis_info { + bool nullable; /* Does the NFA match the empty string? */ + fsm_state_t start; /* start state */ + + /* The states with a /./ self edge representing the unanchored + * start and end, or NO_STATE. There can be at most one of each. */ + fsm_state_t unanchored_start_loop; + fsm_state_t unanchored_end_loop; + + /* The end state following the unanchored end loop. */ + fsm_state_t unanchored_end_loop_end; + + /* State that links to paths only reachable from the beginning of input. */ + fsm_state_t anchored_start; + + /* States leading to an anchored end. */ + struct state_set *anchored_ends; + + /* States with an outgoing labeled edge to the unanchored end loop. Input + * following those edges has matched, but may still consume trailing input. + * These edges correspond to edges leaving capture group 0 in PCRE. */ + struct state_set *eager_matches; + + /* Edges leading to states that can only match at the start of input. */ + struct edge_set *anchored_firsts; + + /* Edges leading to states that can begin an unanchored match, + * potentially after other combined patterns have matched. */ + struct edge_set *repeatable_firsts; + + /* A new state that may be added while replacing the unanchored_end_loop, + * if present. This state exists to set an eager output ID and have an + * epsilon edge to the combined NFA's global_unanchored_end_loop. + * The old unanchored_end_loop will be disconnected. */ + fsm_state_t eager_match_state; +}; struct fsm * fsm_union(struct fsm *a, struct fsm *b, @@ -157,10 +202,441 @@ fsm_union_array(size_t fsm_count, return res; } -#define LOG_UNION_REPEATED_PATTERN_GROUP 0 +static bool +has_dot_self_edge(const struct fsm *nfa, fsm_state_t s_i) +{ + const struct fsm_state *s = &nfa->states[s_i]; + + struct edge_group_iter ei; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &ei); + struct edge_group_iter_info info; + while (edge_set_group_iter_next(&ei, &info)) { + if (info.to != s_i) { continue; } + for (size_t i = 0; i < 256/64; i++) { + if (info.symbols[i] != (uint64_t)-1) { continue; } + } + return true; + } + + return false; +} + +#if LOG_ANALYZE_GROUP_NFA_RESULTS +static void +dump_state_set(FILE *f, const char *name, const struct state_set *set) +{ + struct state_iter si; + fsm_state_t s; + if (state_set_empty(set)) { return; } + + fprintf(f, " - %s:", name); + state_set_reset(set, &si); + while (state_set_next(&si, &s)) { + fprintf(f, " %d", s); + } + fprintf(f, "\n"); +} + +static void +dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set *edges) +{ + struct edge_group_iter iter; + struct edge_group_iter_info info; + if (edge_set_empty(edges)) { return; } + + fprintf(f, " - %s:", name); + edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + fprintf(f, " %d->%d", from, info.to); + } + fprintf(f, "\n"); +} +#endif + +/* If there's a labeled edge to an end state, check if the label set is + * only [\n] and there's also an epsilon edge to the same end state. + * This represents an anchored end in the NFA. */ +static bool +has_epsilon_and_newline_edges_to_end(const struct fsm *nfa, fsm_state_t s_i, fsm_state_t *dst_end) +{ + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + + if (state_set_empty(s->epsilons)) { return false; } + if (edge_set_empty(s->edges)) { return false; } + + struct edge_group_iter iter; + struct edge_group_iter_info info; + + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + /* Look for an edge set with only '\n' */ + if ((info.symbols[0] != (1ULL << '\n')) + || info.symbols[1] || info.symbols[2] || info.symbols[3]) { + continue; + } + + if (fsm_isend(nfa, info.to)) { + struct state_iter si; + fsm_state_t os_i; + state_set_reset(s->epsilons, &si); + while (state_set_next(&si, &os_i)) { + if (os_i == info.to) { + *dst_end = info.to; + return true; + } + } + } + } + + return false; +} + +static bool +has_labeled_edge_to_unanchored_end_loop(const struct fsm *nfa, + fsm_state_t s_i, fsm_state_t unanchored_end_loop) +{ + if (unanchored_end_loop == NO_STATE) { return false; } + assert(unanchored_end_loop < nfa->statecount); + + /* The unanchored_end_loop's self-edge doesn't count here. */ + if (s_i == unanchored_end_loop) { return false; } + + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (info.to == unanchored_end_loop) { + return true; + } + } + return false; +} + +static bool +start_state_epsilon_closure_matches_empty_string__iter(const struct fsm *nfa, + fsm_state_t s_i, struct state_set **seen, bool *result) +{ + if (*result) { return true; } + if (!state_set_add(seen, nfa->alloc, s_i)) { return false; } + + if (fsm_isend(nfa, s_i)) { + *result = true; + return true; + } + + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (!state_set_contains(*seen, ns_i)) { + if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, ns_i, seen, result)) { + return false; + } + if (*result) { break; } + } + } + + return true; +} + +/* Does the start state's epsilon closure match the empty string? + * Returns false on error, otherwise returns true and sets *result. */ +static bool +start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, fsm_state_t start, bool *result) +{ + struct state_set *seen = NULL; /* empty set */ + if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, start, &seen, result)) { return false; } + state_set_free(seen); + + return true; +} + +static bool +analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) +{ + memset(ainfo, 0x00, sizeof(*ainfo)); + ainfo->start = NO_STATE; + ainfo->unanchored_start_loop = NO_STATE; + ainfo->anchored_start = NO_STATE; + ainfo->unanchored_end_loop = NO_STATE; + ainfo->unanchored_end_loop_end = NO_STATE; + ainfo->eager_match_state = NO_STATE; + + if (!fsm_getstart(nfa, &ainfo->start)) { + return false; + } + + const size_t state_count = fsm_countstates(nfa); + assert(ainfo->start < state_count); + + { + const struct fsm_state *s = &nfa->states[ainfo->start]; + + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (ns_i == ainfo->start) { continue; } + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + assert(ns_i < state_count); + const struct fsm_state *ns = &nfa->states[ns_i]; + + /* if there's a state in the start state's epsilon closure that + * has a dot self-edge, it's the unanchored start loop */ + if (has_dot_self_edge(nfa, ns_i)) { + assert(ainfo->unanchored_start_loop == NO_STATE); + ainfo->unanchored_start_loop = ns_i; + + /* copy its non-self labeled edges to ainfo->repeatable_firsts */ + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != ns_i) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } else { + /* likewise, a state without a dot self-edge is the anchored start */ + assert(ainfo->anchored_start == NO_STATE); + ainfo->anchored_start = ns_i; + + /* copy its labeled edges to ainfo->anchored_firsts */ + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (!edge_set_add_bulk(&ainfo->anchored_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } + } + + /* If the start state always matches, set a flag noting that it will need special handling + * later. It's arguably pointless to combine "" with other regexes, because it will always + * trivially match, but otherwise it would never match. */ + if (!start_state_epsilon_closure_matches_empty_string(nfa, ainfo->start, &ainfo->nullable)) { + goto alloc_fail; + } + + /* If there's a state with a dot self-edge and an epsilon edge to an end state, it's + * the unanchored end loop. There should only be one. */ + for (size_t s_i = 0; s_i < state_count; s_i++) { + const struct fsm_state *s = &nfa->states[s_i]; + if (has_dot_self_edge(nfa, s_i)) { + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (fsm_isend(nfa, ns_i)) { + assert(ainfo->unanchored_end_loop == NO_STATE); + ainfo->unanchored_end_loop = s_i; + ainfo->unanchored_end_loop_end = ns_i; + break; + } + } + if (ainfo->unanchored_end_loop != NO_STATE) { break; } + } + } + + /* Collect states that lead to an anchored end or eager match. */ + for (size_t s_i = 0; s_i < state_count; s_i++) { + fsm_state_t dst_end = NO_STATE; + if (has_epsilon_and_newline_edges_to_end(nfa, s_i, &dst_end)) { + if (!state_set_add(&ainfo->anchored_ends, nfa->alloc, dst_end)) { + goto alloc_fail; + } + } + + if (has_labeled_edge_to_unanchored_end_loop(nfa, s_i, ainfo->unanchored_end_loop)) { + if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { + goto alloc_fail; + } + } + } -/* Combine an array of FSMs into a single FSM in one pass, with an extra loop - * so that more than one pattern with eager outputs can match. */ +#if LOG_ANALYZE_GROUP_NFA_RESULTS + { + fprintf(stderr, "# analysis_info start %d, usl %d, uel %d, uele %d\n", + ainfo->start, ainfo->unanchored_start_loop, ainfo->unanchored_end_loop, ainfo->unanchored_end_loop_end); + dump_state_set(stderr, "anchored_ends", ainfo->anchored_ends); + dump_state_set(stderr, "eager_matches", ainfo->eager_matches); + dump_edge_set(stderr, "anchored_firsts", ainfo->anchored_start, ainfo->anchored_firsts); + dump_edge_set(stderr, "repeatable_firsts", ainfo->unanchored_start_loop, ainfo->repeatable_firsts); + } +#endif + return true; + +alloc_fail: + fprintf(stderr, "alloc fail\n"); + return false; +} + +/* Replace any labeled edges on nfa->states[from_state] going to old_to + * with a new edge leading to new_to. There currently isn't a function in + * the libfsm API for this (and it shouldn't be necessary in general), but + * if it gets one later this can be replaced. */ +static bool +replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to) +{ + if (old_to == NO_STATE) { + /* nothing to do */ + return true; + } + assert(new_to < nfa->statecount); + assert(from_state < nfa->statecount); + + struct fsm_state *from = &nfa->states[from_state]; + struct edge_set *old_edges = from->edges; + struct edge_set *new_edges = edge_set_new(); + + /* copy, replacing edges to old_to */ + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(old_edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (!edge_set_add_bulk(&new_edges, nfa->alloc, info.symbols, + info.to == old_to ? new_to : info.to)) { + return false; + } + } + + edge_set_free(nfa->alloc, old_edges); + from->edges = new_edges; + return true; +} + +/* Make a couple changes to the group NFA so that it can be combined correctly: + * + * - If the group NFA has an unanchored_end_loop, add a new state, + * eager_match_state, which will be a waypoint between edges that previously + * led to the unanchored_end_loop and the global NFA's global_unanchored_end_loop + * (so it can potentially also match other group NFAs with unanchored starts). + * This state will get an eager output ID. + * + * - Set an end ID on every anchored end state, so halting on these counts as a match. */ +static bool +modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t id_base) +{ + const bool nullable_and_unanchored_end = ainfo->nullable + && ainfo->unanchored_end_loop != NO_STATE; + + /* Add the eager match state if there are eager match states + * or a nullable unanchored end. This will link to the global NFA's + * unanchored_end_loop. */ + if (!state_set_empty(ainfo->eager_matches) || nullable_and_unanchored_end) { + if (!fsm_addstate(nfa, &ainfo->eager_match_state)) { + return false; + } + + /* Set eager match ID on new eager_match_state. */ + const fsm_output_id_t oid = (fsm_output_id_t)(id + id_base); + if (!fsm_seteageroutput(nfa, ainfo->eager_match_state, oid)) { + return false; + } + + /* For every state in eager_matches, replace every edge leading to + * the unanchored_end_loop with an edge with the same labels to + * eager_match_state. */ + struct state_iter si; + state_set_reset(ainfo->eager_matches, &si); + fsm_state_t ems_i; + while (state_set_next(&si, &ems_i)) { + if (!replace_labeled_edge(nfa, ems_i, + ainfo->unanchored_end_loop, ainfo->eager_match_state)) { + return false; + } + + /* The state must not link to the unanchored end loop anymore. + * Doing so will cause a combinatorial explosion that makes + * combining more ~10 NFAs incredibly expensive. */ + struct edge_group_iter iter; + struct edge_group_iter_info info; + assert(ems_i < nfa->statecount); + const struct fsm_state *ems = &nfa->states[ems_i]; + edge_set_group_iter_reset(ems->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to != ainfo->unanchored_end_loop); + } + } + } + + /* If the group NFA matches the empty string and has an unanchored end, then + * link its unanchored start state to the eager match state. This ensures + * all inputs will match this group NFA when combined. */ + if (nullable_and_unanchored_end) { + assert(ainfo->start != NO_STATE); + assert(ainfo->eager_match_state != NO_STATE); + + if (ainfo->unanchored_start_loop != NO_STATE) { + struct fsm_state *s = &nfa->states[ainfo->unanchored_start_loop]; + if (!state_set_add(&s->epsilons, nfa->alloc, ainfo->eager_match_state)) { + return false; + } + } + } + + /* If there are anchored ends, set an endid on them */ + if (!state_set_empty(ainfo->anchored_ends)) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t anchored_end_state; + const fsm_end_id_t end_id = (fsm_end_id_t)(id + id_base); + while (state_set_next(&si, &anchored_end_state)) { + if (!fsm_endid_set(nfa, anchored_end_state, end_id)) { + return false; + } + } + } + + return true; +} + +static void +rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) +{ + if (base == 0) { return; } + +#define SHIFT(S) if (ainfo-> S != NO_STATE) { ainfo-> S += base; } + SHIFT(start); + SHIFT(unanchored_start_loop); + SHIFT(unanchored_end_loop); + SHIFT(unanchored_end_loop_end); + SHIFT(anchored_start); + SHIFT(eager_match_state); +#undef SHIFT + + state_set_rebase(&ainfo->anchored_ends, base); + state_set_rebase(&ainfo->eager_matches, base); + + edge_set_rebase(&ainfo->anchored_firsts, base); + edge_set_rebase(&ainfo->repeatable_firsts, base); + +} + +static void +free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) +{ + state_set_free(ainfo->anchored_ends); + state_set_free(ainfo->eager_matches); + edge_set_free(alloc, ainfo->anchored_firsts); + edge_set_free(alloc, ainfo->repeatable_firsts); +} + +/* Combine an array of FSMs into a single FSM that attempts to match them + * all in one pass, with an extra loop so that more than one pattern with + * eager outputs can match. */ struct fsm * fsm_union_repeated_pattern_group(size_t entry_count, struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases) @@ -168,9 +644,11 @@ fsm_union_repeated_pattern_group(size_t entry_count, const struct fsm_alloc *alloc = entries[0].fsm->alloc; const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; - if (entry_count == 1) { - return entries[0].fsm; - } + /* TODO: make this an extra argument */ + const size_t id_base = 1; + + struct analysis_info *ainfos = f_calloc(alloc, entry_count, sizeof(ainfos[0])); + if (ainfos == NULL) { goto fail; } size_t est_total_states = 0; for (size_t i = 0; i < entry_count; i++) { @@ -183,205 +661,200 @@ fsm_union_repeated_pattern_group(size_t entry_count, est_total_states += count; } + for (size_t i = 0; i < entry_count; i++) { + struct fsm *fsm = entries[i].fsm; + + /* Identify various states in the NFA that will be relevant to combining. */ + if (!analyze_group_nfa(fsm, &ainfos[i])) { + goto fail; + } + + /* Change the NFA structure so it can better link into the combined FSM, + * and set endids and/or output IDs as appropriate. */ + if (!modify_group_nfa(fsm, i, &ainfos[i], id_base)) { + goto fail; + } + } + est_total_states += 5; /* new start and end, new unanchored start and end loops */ struct fsm *res = fsm_new_statealloc(alloc, est_total_states); if (res == NULL) { return NULL; } - /* collected end states */ - struct ends_buf { - size_t ceil; - size_t used; - fsm_state_t *states; - } ends = { .ceil = 0 }; - - /* The new overall start state, which will have an epsilon edge to... */ + /* The new overall start state */ fsm_state_t global_start; if (!fsm_addstate(res, &global_start)) { goto fail; } - /* states linking to the starts of unanchored and anchored subgraphs, respectively. */ - fsm_state_t global_start_loop, global_start_anchored; - if (!fsm_addstate(res, &global_start_loop)) { goto fail; } - if (!fsm_addstate(res, &global_start_anchored)) { goto fail; } + /* States linking to the starts of unanchored and anchored subgraphs, respectively. + * Matching other group NFAs loops back to the global_unanchored_start_loop, but + * patterns anchored at the ^start are only reachable via global_anchored_start. */ + fsm_state_t global_unanchored_start_loop, global_anchored_start; + if (!fsm_addstate(res, &global_unanchored_start_loop)) { goto fail; } + if (!fsm_addstate(res, &global_anchored_start)) { goto fail; } /* The unanchored end loop state, and an end state with no outgoing edges. */ - fsm_state_t global_end_loop, global_end; + fsm_state_t global_unanchored_end_loop, global_end; if (!fsm_addstate(res, &global_end)) { goto fail; } - if (!fsm_addstate(res, &global_end_loop)) { goto fail; } + if (!fsm_addstate(res, &global_unanchored_end_loop)) { goto fail; } - /* link the start to the start loop and anchored start, and the start loop to itself */ + /* link the start to the global unanchored start loop and anchored start. */ if (log) { - fprintf(stderr, "link_before: global_start %d -> global_start_loop %d and global_start_anchored %d\n", - global_start, global_start_loop, global_start_anchored); + fprintf(stderr, "link_before: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", + global_start, global_unanchored_start_loop, global_anchored_start); } - if (!fsm_addedge_epsilon(res, global_start, global_start_loop)) { goto fail; } - if (!fsm_addedge_epsilon(res, global_start, global_start_anchored)) { goto fail; } - if (!fsm_addedge_any(res, global_start_loop, global_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } - /* link the end loop and end */ + /* Link the global unanchored start loop to itself. */ + if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop and global end. */ if (log) { - fprintf(stderr, "link_before: global_end_loop %d -> global_end %d (and -> self)\n", global_end_loop, global_end); + fprintf(stderr, "link_before: global_unanchored_end_loop %d -> global_end %d (and -> self)\n", global_unanchored_end_loop, global_end); } - if (!fsm_addedge_epsilon(res, global_end_loop, global_end)) { goto fail; } - if (!fsm_addedge_any(res, global_end_loop, global_end_loop)) { goto fail; } + if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } if (bases != NULL) { memset(bases, 0x00, entry_count * sizeof(bases[0])); } + /* For each group FSM, link its unanchored and anchored start states + * and eager_match_state to the global ones. */ for (size_t fsm_i = 0; fsm_i < entry_count; fsm_i++) { - ends.used = 0; /* reset */ - struct fsm *fsm = entries[fsm_i].fsm; entries[fsm_i].fsm = NULL; /* transfer ownership */ const size_t state_count = fsm_countstates(fsm); - - fsm_state_t fsm_start; - if (!fsm_getstart(fsm, &fsm_start)) { + struct analysis_info *ainfo = &ainfos[fsm_i]; + if (ainfo->start == NO_STATE) { fsm_free(fsm); /* no start, just discard */ continue; } + assert(ainfo->start < state_count); - for (fsm_state_t s_i = 0; s_i < state_count; s_i++) { - if (fsm_isend(fsm, s_i)) { - if (ends.used == ends.ceil) { /* grow? */ - size_t nceil = (ends.ceil == 0 ? 4 : 2*ends.ceil); - fsm_state_t *nstates = f_realloc(alloc, - ends.states, nceil * sizeof(nstates[0])); - if (nstates == NULL) { goto fail; } - ends.ceil = nceil; - ends.states = nstates; - } - ends.states[ends.used++] = s_i; - } - } - - if (ends.used == 0) { - fsm_free(fsm); /* no ends, just discard */ - continue; - } - - /* When combining these, remove self-edges from any states on the FSMs to be - * combined that also have eager output IDs. We are about to add an epsilon edge - * from each to a shared state that won't have eager output IDs. - * - * Eager output matching should be idempotent, so carrying it to other reachable - * state is redundant, and it leads to a combinatorial explosion that blows up the - * state count while determinising the combined FSM otherwise. - * - * For example, if /aaa/, /bbb/, and /ccc/ are combined into a DFA that repeats - * the sub-patterns (like `^.*(?:(aaa)|(bbb)|(ccc))+.*$`), the self-edge at each - * eager output state would combine with every reachable state from then on, - * leading to a copy of the whole reachable subgraph colored by every - * combination of eager output IDs: aaa, bbb, ccc, aaa+bbb, aaa+ccc, - * bbb+ccc, aaa+bbb+ccc. Instead of three relatively separate subgraphs - * that set the eager output at their last state, one for each pattern, - * it leads to 8 (2**3) subgraph clusters because it encodes _each - * distinct combination_ in the DFA. This becomes incredibly expensive - * as the combined pattern count increases; it's essentially what I'm - * trying to avoid by adding eager output support in the first place. - * - * FIXME: instead of actively removing these, filter in fsm_determinise? */ - if (fsm_eager_output_has_eager_output(fsm)) { - /* for any state that has eager outputs and a self edge, - * remove the self edge before further linkage */ - for (fsm_state_t s = 0; s < fsm->statecount; s++) { - const size_t eager_output_count = fsm_eager_output_count(fsm, s); - if (eager_output_count == 0) { continue; } - struct edge_set *edges = fsm->states[s].edges; - struct edge_set *new = edge_set_new(); - - struct edge_group_iter iter; - struct edge_group_iter_info info; - edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - if (info.to != s) { - if (!edge_set_add_bulk(&new, fsm->alloc, - info.symbols, info.to)) { - goto fail; - } - } - } - edge_set_free(fsm->alloc, edges); - fsm->states[s].edges = new; - } - } - - /* call fsm_merge; we really don't care which is which */ + /* Call fsm_merge; we really don't care which is which. */ struct fsm_combine_info combine_info; struct fsm *merged = fsm_merge(res, fsm, &combine_info); if (merged == NULL) { goto fail; } - /* update offsets if res had its state IDs shifted forward */ + /* Update offsets if res had its state IDs shifted forward. */ global_start += combine_info.base_a; - global_start_loop += combine_info.base_a; - global_start_anchored += combine_info.base_a;; + global_unanchored_start_loop += combine_info.base_a; + global_anchored_start += combine_info.base_a; global_end += combine_info.base_a; - global_end_loop += combine_info.base_a; + global_unanchored_end_loop += combine_info.base_a; - /* also update offsets for the FSM's states */ - fsm_start += combine_info.base_b; - for (size_t i = 0; i < ends.used; i++) { - ends.states[i] += combine_info.base_b; - } + /* Also update offsets for the group FSM's states. */ + rebase_analysis_info(ainfo, combine_info.base_b); if (bases != NULL) { bases[fsm_i].state = combine_info.base_b; bases[fsm_i].capture = combine_info.capture_base_b; } - if (log) { - fprintf(stderr, "%s: fsm[%zd].start: %d\n", __func__, fsm_i, fsm_start); - for (size_t i = 0; i < ends.used; i++) { - fprintf(stderr, "%s: fsm[%zd].ends[%zd]: %d\n", __func__, fsm_i, i, ends.states[i]); + /* Link the FSM's eager match state back to the global_unanchored_end_loop, so that after + * matching it in an unanchored way it can continue attempting to match other combined + * patterns that aren't anchored at their start. Also link it to the global end, so + * it will be retained during determinisation and minimisation. */ + if (ainfo->eager_match_state != NO_STATE) { + if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_unanchored_end_loop)) { + goto fail; + } + if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_end)) { + goto fail; + } + + /* If the NFA matches the empty string and is not anchored at the end, then + * add an epsilon edge from the global start directly to its eager match state. + * This ensures all inputs will match this group NFA when combined. */ + if (ainfo->nullable) { + assert(ainfo->unanchored_end_loop != NO_STATE); + if (!fsm_addedge_epsilon(merged, global_start, ainfo->eager_match_state)) { + goto fail; + } + } + } else { + /* If the NFA matches an end-anchored empty string, then add an epsilon edge from + * the global start to an anchored end, which has an endid. */ + if (ainfo->nullable && !state_set_empty(ainfo->anchored_ends)) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t anchored_end_state; + + while (state_set_next(&si, &anchored_end_state)) { + if (!fsm_addedge_epsilon(merged, global_start, anchored_end_state)) { + goto fail; + } + /* It should only be necessary to link one, since that's enough + * for determinisation to carry the end id back to the start's + * epsilon closure. */ + break; + } } } - /* link to the FSM's start state */ - const fsm_state_t start_src = entries[fsm_i].anchored_start ? global_start_anchored : global_start_loop; - if (!fsm_addedge_epsilon(merged, start_src, fsm_start)) { goto fail; } - if (log) { - fprintf(stderr, "%s: linking %s %d to fsm[%zd]'s start %d (anchored? %d)\n", - __func__, - entries[fsm_i].anchored_start ? "global_start_anchored" : "global_start_loop", - start_src, fsm_i, fsm_start, entries[fsm_i].anchored_start); + struct edge_group_iter iter; + struct edge_group_iter_info info; + + /* Link the global_anchored_start to group FSM paths that are start-anchored + * and can only match at the start of input. */ + edge_set_group_iter_reset(ainfo->anchored_firsts, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(global_anchored_start < merged->statecount); + struct fsm_state *anchored_start = &merged->states[global_anchored_start]; + if (!edge_set_add_bulk(&anchored_start->edges, merged->alloc, + info.symbols, info.to)) { + goto fail; + } } - /* link from the FSM's ends */ - const fsm_state_t end_dst = entries[fsm_i].anchored_end ? global_end : global_end_loop; - for (size_t i = 0; i < ends.used; i++) { - if (log) { - fprintf(stderr, "%s: linking fsm[%zd]'s end[%zd] %d (anchored? %d) to %s %d\n", - __func__, fsm_i, i, ends.states[i], entries[fsm_i].anchored_end, - entries[fsm_i].anchored_end ? "global_end" : "global_end_loop", - end_dst); + /* Link the global_unanchored_start_loop to group FSM paths that aren't + * start-anchored. */ + edge_set_group_iter_reset(ainfo->repeatable_firsts, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + struct fsm_state *unanchored_start = &merged->states[global_unanchored_start_loop]; + if (!edge_set_add_bulk(&unanchored_start->edges, merged->alloc, + info.symbols, info.to)) { + goto fail; } - if (!fsm_addedge_epsilon(merged, ends.states[i], end_dst)) { goto fail; } } res = merged; } - /* Link from the global_end_loop to the global_start_loop, so patterns with an - * unanchored start can follow other patterns with an unanchored end. */ + /* Link from the global_unanchored_end_loop to the global_unanchored_start_loop, + * so patterns with an unanchored start can follow other patterns with an unanchored + * end, possibly with other ignored input between them. */ if (log) { fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", - __func__, global_start, global_start_loop, global_start_anchored, global_end_loop, global_end); - fprintf(stderr, "%s: linking global_end_loop %d to global_start_loop %d\n", - __func__, global_end_loop, global_start_loop); + __func__, global_start, global_unanchored_start_loop, global_anchored_start, global_unanchored_end_loop, global_end); + fprintf(stderr, "%s: linking global_unanchored_end_loop %d to global_unanchored_start_loop %d\n", + __func__, global_unanchored_end_loop, global_unanchored_start_loop); fprintf(stderr, "%s: setting global_start %d and end %d\n", __func__, global_start, global_end); } - if (!fsm_addedge_epsilon(res, global_end_loop, global_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } /* This needs to be set after merging, because that clears the start state. */ fsm_setstart(res, global_start); fsm_setend(res, global_end, 1); - f_free(alloc, ends.states); + for (size_t i = 0; i < entry_count; i++) { + free_analysis(alloc, &ainfos[i]); + } + + f_free(alloc, ainfos); + return res; fail: - f_free(alloc, ends.states); + if (ainfos != NULL) { + for (size_t i = 0; i < entry_count; i++) { + free_analysis(alloc, &ainfos[i]); + } + f_free(alloc, ainfos); + } + return NULL; } diff --git a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c new file mode 100644 index 000000000..06965e014 --- /dev/null +++ b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c @@ -0,0 +1,29 @@ +#include "utils.h" + +/* Test for false positive matches when combining patterns + * with both anchored and unanchored subtrees. */ + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "a(?:x|$)", + "a(?:y|$)", + }, + .inputs = { + { .input = "a", .expected_ids = { 1, 2 } }, + { .input = "aZ", .expect_fail = true }, + { .input = "Za", .expected_ids = { 1, 2 } }, + { .input = "ax", .expected_ids = { 1 } }, + { .input = "axZ", .expected_ids = { 1 } }, + { .input = "ay", .expected_ids = { 2 } }, + { .input = "ayZ", .expected_ids = { 2 } }, + { .input = "axa", .expected_ids = { 1, 2 } }, + { .input = "aya", .expected_ids = { 1, 2 } }, + { .input = "axay", .expected_ids = { 1, 2 } }, + { .input = "ayax", .expected_ids = { 1, 2 } }, + }, + }; + + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_at_start_multiple_anchored.c b/tests/eager_output/eager_output_at_start_multiple_anchored.c new file mode 100644 index 000000000..ce242472f --- /dev/null +++ b/tests/eager_output/eager_output_at_start_multiple_anchored.c @@ -0,0 +1,15 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + [0] = "$", + [1] = "^$", + }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_at_start_single_anchored.c b/tests/eager_output/eager_output_at_start_single_anchored.c new file mode 100644 index 000000000..e7ce16050 --- /dev/null +++ b/tests/eager_output/eager_output_at_start_single_anchored.c @@ -0,0 +1,22 @@ +#include "utils.h" + +int main(void) +{ + const struct eager_output_test test_unanchored_start = { + .patterns = { "$" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + + const struct eager_output_test test_anchored_start = { + .patterns = { "^$" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + + bool pass = run_test(&test_unanchored_start, false, false); + pass = run_test(&test_anchored_start, false, false) && pass; + return pass; +} diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c index 4bee8d848..f7eb23069 100644 --- a/tests/eager_output/utils.c +++ b/tests/eager_output/utils.c @@ -45,6 +45,8 @@ dump(const struct fsm *fsm) int run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids) { + (void)force_endids; /* TODO: unused, remove. */ + struct fsm_union_entry entries[MAX_PATTERNS] = {0}; allow_extra_outputs = false; @@ -82,45 +84,14 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); assert(fsm != NULL); - /* Zero is used to terminate expected_ids, so don't use it here. */ - const fsm_output_id_t output_id = (fsm_output_id_t) (i + 1); - const fsm_end_id_t end_id = (fsm_end_id_t) (i + 1); - - /* Set either an end ID or an eager output ID, depending on - * whether the fsm is anchored at the end or not. */ - if (e->anchored_end || force_endids) { - ret = fsm_setendid(fsm, end_id); - } else { - ret = fsm_seteageroutputonends(fsm, output_id); - } - assert(ret == 1); - if (log) { fprintf(stderr, "==== source DFA %zd (pre det+min)\n", i); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fsm_endid_dump(stderr, fsm); - fprintf(stderr, "====\n"); - } - - ret = fsm_determinise(fsm); - assert(ret == 1); - - if (log) { - fprintf(stderr, "==== source DFA %zd (post det)\n", i); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "====\n"); - } - - ret = fsm_minimise(fsm); - assert(ret == 1); - - if (log) { - fprintf(stderr, "==== source DFA %zd (post det+min)\n", i); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } e->fsm = fsm; @@ -133,11 +104,13 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo if (log) { fprintf(stderr, "==== combined (pre det+min)\n"); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "--- endids:\n"); - fsm_endid_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } if (log) { @@ -151,9 +124,11 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo if (log) { fprintf(stderr, "==== combined (post det)\n"); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } ret = fsm_minimise(fsm); @@ -164,11 +139,13 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo if (log) { fprintf(stderr, "==== combined (post det+min)\n"); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "--- endids:\n"); - fsm_endid_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } struct cb_info outputs = { 0 }; @@ -218,13 +195,22 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo /* Copy endid outputs into outputs.ids[], since for testing * purposes we don't care about the difference between eager - * output and endids here -- the values don't overlap. */ + * output and endids here. */ assert(outputs.used + endid_count <= MAX_IDS); for (size_t endid_i = 0; endid_i < endid_count; endid_i++) { if (log) { fprintf(stderr, "-- adding endid %zd: %d\n", endid_i, endid_buf[endid_i]); } - outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + bool found = false; + for (size_t o_i = 0; o_i < outputs.used; o_i++) { + if (outputs.ids[o_i] == endid_buf[endid_i]) { + found = true; + break; + } + } + if (!found) { + outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + } } } From 24817d3edaab51860c7d39e7395243a73fb64310 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 4 Feb 2025 13:57:35 -0500 Subject: [PATCH 37/54] fsm_union_repeated_pattern_group: Interface changes. - Instead of taking an array of `struct fsm_union_array *` pointers, this now takes an array of `struct fsm *` pointers. The other fields on `fsm_union_array` are no longer used, so the extra struct layer has been removed. - This now takes an extra argument, id_base, because each nfa[i] will get end IDs and/or output IDs (i + id_base) set on them. Previously these were set by the caller. - Rename parameters, to emphasize that the FSMs must be NFAs. - Update the test code for the interface changes. - Remove flags from the test code that are no longer used. --- include/fsm/bool.h | 23 ++++++---- src/libfsm/union.c | 43 +++++++++---------- tests/eager_output/eager_output1.c | 2 +- tests/eager_output/eager_output2.c | 2 +- tests/eager_output/eager_output3.c | 2 +- tests/eager_output/eager_output4.c | 2 +- tests/eager_output/eager_output5.c | 2 +- tests/eager_output/eager_output6.c | 2 +- tests/eager_output/eager_output7.c | 30 +------------ ...utput_alt_mixing_anchored_and_unanchored.c | 2 +- .../eager_output_at_start_multiple.c | 2 +- .../eager_output_at_start_multiple_anchored.c | 2 +- .../eager_output_at_start_single.c | 2 +- .../eager_output_at_start_single_anchored.c | 4 +- tests/eager_output/eager_output_fr1.c | 2 +- tests/eager_output/eager_output_fr2.c | 2 +- tests/eager_output/eager_output_fr3.c | 2 +- .../eager_output_mixed_anchored_unanchored.c | 2 +- tests/eager_output/utils.c | 38 ++++------------ tests/eager_output/utils.h | 2 +- 20 files changed, 62 insertions(+), 106 deletions(-) diff --git a/include/fsm/bool.h b/include/fsm/bool.h index 4d9f1889a..c2c2d80ed 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -52,15 +52,22 @@ struct fsm * fsm_union_array(size_t fsm_count, struct fsm **fsms, struct fsm_combined_base_pair *bases); -struct fsm_union_entry { - struct fsm *fsm; - bool anchored_start; - bool anchored_end; -}; - +/* Combine an array of NFAs into a single NFA that attempts to match them + * all in one pass, with an extra loop so that more than one pattern with + * eager outputs can match. Ownership of the NFAs is transferred, they will + * be combined (or freed, if they don't have a start state). + * + * This MUST be called with NFAs constructed via re_comp, Calling it with + * manually constructed NFAs or DFAs is unsupported. + * + * This will set end IDs and/or output IDs representing matching each + * of the original NFAs on the combined result, where nfas[i] will + * get ID of (id_base + i). + * + * Returns NULL on error. */ struct fsm * -fsm_union_repeated_pattern_group(size_t entry_count, - struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases); +fsm_union_repeated_pattern_group(size_t nfa_count, + struct fsm **nfas, struct fsm_combined_base_pair *bases, size_t id_base); struct fsm * fsm_intersect(struct fsm *a, struct fsm *b); diff --git a/src/libfsm/union.c b/src/libfsm/union.c index ed71384ff..f381d7cd6 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -638,31 +638,28 @@ free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) * all in one pass, with an extra loop so that more than one pattern with * eager outputs can match. */ struct fsm * -fsm_union_repeated_pattern_group(size_t entry_count, - struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases) +fsm_union_repeated_pattern_group(size_t nfa_count, + struct fsm **nfas, struct fsm_combined_base_pair *bases, size_t id_base) { - const struct fsm_alloc *alloc = entries[0].fsm->alloc; + const struct fsm_alloc *alloc = nfas[0]->alloc; const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; - /* TODO: make this an extra argument */ - const size_t id_base = 1; - - struct analysis_info *ainfos = f_calloc(alloc, entry_count, sizeof(ainfos[0])); + struct analysis_info *ainfos = f_calloc(alloc, nfa_count, sizeof(ainfos[0])); if (ainfos == NULL) { goto fail; } size_t est_total_states = 0; - for (size_t i = 0; i < entry_count; i++) { - assert(entries[i].fsm); - if (entries[i].fsm->alloc != alloc) { + for (size_t i = 0; i < nfa_count; i++) { + assert(nfas[i]); + if (nfas[i]->alloc != alloc) { errno = EINVAL; return NULL; } - const size_t count = fsm_countstates(entries[i].fsm); + const size_t count = fsm_countstates(nfas[i]); est_total_states += count; } - for (size_t i = 0; i < entry_count; i++) { - struct fsm *fsm = entries[i].fsm; + for (size_t i = 0; i < nfa_count; i++) { + struct fsm *fsm = nfas[i]; /* Identify various states in the NFA that will be relevant to combining. */ if (!analyze_group_nfa(fsm, &ainfos[i])) { @@ -716,17 +713,17 @@ fsm_union_repeated_pattern_group(size_t entry_count, if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } if (bases != NULL) { - memset(bases, 0x00, entry_count * sizeof(bases[0])); + memset(bases, 0x00, nfa_count * sizeof(bases[0])); } - /* For each group FSM, link its unanchored and anchored start states + /* For each group NFA, link its unanchored and anchored start states * and eager_match_state to the global ones. */ - for (size_t fsm_i = 0; fsm_i < entry_count; fsm_i++) { - struct fsm *fsm = entries[fsm_i].fsm; - entries[fsm_i].fsm = NULL; /* transfer ownership */ + for (size_t nfa_i = 0; nfa_i < nfa_count; nfa_i++) { + struct fsm *fsm = nfas[nfa_i]; + nfas[nfa_i] = NULL; /* transfer ownership */ const size_t state_count = fsm_countstates(fsm); - struct analysis_info *ainfo = &ainfos[fsm_i]; + struct analysis_info *ainfo = &ainfos[nfa_i]; if (ainfo->start == NO_STATE) { fsm_free(fsm); /* no start, just discard */ continue; @@ -749,8 +746,8 @@ fsm_union_repeated_pattern_group(size_t entry_count, rebase_analysis_info(ainfo, combine_info.base_b); if (bases != NULL) { - bases[fsm_i].state = combine_info.base_b; - bases[fsm_i].capture = combine_info.capture_base_b; + bases[nfa_i].state = combine_info.base_b; + bases[nfa_i].capture = combine_info.capture_base_b; } /* Link the FSM's eager match state back to the global_unanchored_end_loop, so that after @@ -840,7 +837,7 @@ fsm_union_repeated_pattern_group(size_t entry_count, fsm_setstart(res, global_start); fsm_setend(res, global_end, 1); - for (size_t i = 0; i < entry_count; i++) { + for (size_t i = 0; i < nfa_count; i++) { free_analysis(alloc, &ainfos[i]); } @@ -850,7 +847,7 @@ fsm_union_repeated_pattern_group(size_t entry_count, fail: if (ainfos != NULL) { - for (size_t i = 0; i < entry_count; i++) { + for (size_t i = 0; i < nfa_count; i++) { free_analysis(alloc, &ainfos[i]); } f_free(alloc, ainfos); diff --git a/tests/eager_output/eager_output1.c b/tests/eager_output/eager_output1.c index f20ef77b7..4900f89e0 100644 --- a/tests/eager_output/eager_output1.c +++ b/tests/eager_output/eager_output1.c @@ -8,5 +8,5 @@ int main(void) { .input = "abc", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output2.c b/tests/eager_output/eager_output2.c index cdac204e2..6a10eec1c 100644 --- a/tests/eager_output/eager_output2.c +++ b/tests/eager_output/eager_output2.c @@ -13,5 +13,5 @@ int main(void) { .input = "XabeX", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output3.c b/tests/eager_output/eager_output3.c index c11bc58a4..b6320ef79 100644 --- a/tests/eager_output/eager_output3.c +++ b/tests/eager_output/eager_output3.c @@ -12,5 +12,5 @@ int main(void) { .input = "abe", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output4.c b/tests/eager_output/eager_output4.c index 47cd32029..2e0f17f13 100644 --- a/tests/eager_output/eager_output4.c +++ b/tests/eager_output/eager_output4.c @@ -9,5 +9,5 @@ int main(void) { .input = "Xabcde", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output5.c b/tests/eager_output/eager_output5.c index 4551c68b1..6d2ce4eb8 100644 --- a/tests/eager_output/eager_output5.c +++ b/tests/eager_output/eager_output5.c @@ -10,5 +10,5 @@ int main(void) { .input = "abbc", .expected_ids = { 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output6.c b/tests/eager_output/eager_output6.c index 5431d0981..188541f39 100644 --- a/tests/eager_output/eager_output6.c +++ b/tests/eager_output/eager_output6.c @@ -30,5 +30,5 @@ int main(void) }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output7.c b/tests/eager_output/eager_output7.c index 3d123878b..94e9f1787 100644 --- a/tests/eager_output/eager_output7.c +++ b/tests/eager_output/eager_output7.c @@ -2,24 +2,6 @@ int main(void) { - /* Run this test with env FORCE_ENDIDS=N ... to see how much more - * expensive it is to combine the first N patterns using endids, - * rather than eager_outputs. It becomes VERY slow for >= 9 or so. - * (Note that the checks probably will not pass for N < 4, because - * it will start skipping appear in the early test inputs.) */ - bool force_endids = false; - size_t force_endid_count = 0; - { - const char *str = getenv("FORCE_ENDIDS"); - if (str != NULL) { - force_endid_count = atoi(str); - if (force_endid_count == 0) { - force_endid_count = 26; - } - force_endids = true; - } - } - struct eager_output_test test = { .patterns = { [0] = "apple", @@ -89,15 +71,5 @@ int main(void) }, }; - /* truncate patterns to the first N */ - if (force_endids) { - assert(force_endid_count > 0 && force_endid_count <= 26); - test.patterns[force_endid_count] = NULL; - - /* truncate test inputs to just the first couple, since - * later inputs use later patterns */ - test.inputs[5].input = NULL; - } - - return run_test(&test, false, force_endids); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c index 06965e014..ad125b9b1 100644 --- a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c +++ b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c @@ -25,5 +25,5 @@ int main(void) }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start_multiple.c b/tests/eager_output/eager_output_at_start_multiple.c index 5c6f4577c..ddc9530f5 100644 --- a/tests/eager_output/eager_output_at_start_multiple.c +++ b/tests/eager_output/eager_output_at_start_multiple.c @@ -11,5 +11,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start_multiple_anchored.c b/tests/eager_output/eager_output_at_start_multiple_anchored.c index ce242472f..0cf7e9b70 100644 --- a/tests/eager_output/eager_output_at_start_multiple_anchored.c +++ b/tests/eager_output/eager_output_at_start_multiple_anchored.c @@ -11,5 +11,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start_single.c b/tests/eager_output/eager_output_at_start_single.c index 407aa4e77..8ba5f2ad1 100644 --- a/tests/eager_output/eager_output_at_start_single.c +++ b/tests/eager_output/eager_output_at_start_single.c @@ -8,5 +8,5 @@ int main(void) { .input = "", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start_single_anchored.c b/tests/eager_output/eager_output_at_start_single_anchored.c index e7ce16050..a9c13ef67 100644 --- a/tests/eager_output/eager_output_at_start_single_anchored.c +++ b/tests/eager_output/eager_output_at_start_single_anchored.c @@ -16,7 +16,7 @@ int main(void) }, }; - bool pass = run_test(&test_unanchored_start, false, false); - pass = run_test(&test_anchored_start, false, false) && pass; + bool pass = run_test(&test_unanchored_start); + pass = run_test(&test_anchored_start) && pass; return pass; } diff --git a/tests/eager_output/eager_output_fr1.c b/tests/eager_output/eager_output_fr1.c index e8e5f3395..97eb34312 100644 --- a/tests/eager_output/eager_output_fr1.c +++ b/tests/eager_output/eager_output_fr1.c @@ -9,5 +9,5 @@ int main(void) { .input = "ab", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_fr2.c b/tests/eager_output/eager_output_fr2.c index 404e98644..23bd3103c 100644 --- a/tests/eager_output/eager_output_fr2.c +++ b/tests/eager_output/eager_output_fr2.c @@ -9,5 +9,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_fr3.c b/tests/eager_output/eager_output_fr3.c index c7e4127a6..0d15a4a68 100644 --- a/tests/eager_output/eager_output_fr3.c +++ b/tests/eager_output/eager_output_fr3.c @@ -9,5 +9,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_mixed_anchored_unanchored.c b/tests/eager_output/eager_output_mixed_anchored_unanchored.c index a586f9840..7afb272db 100644 --- a/tests/eager_output/eager_output_mixed_anchored_unanchored.c +++ b/tests/eager_output/eager_output_mixed_anchored_unanchored.c @@ -42,5 +42,5 @@ int main(void) }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c index f7eb23069..dfd2b952b 100644 --- a/tests/eager_output/utils.c +++ b/tests/eager_output/utils.c @@ -43,15 +43,11 @@ dump(const struct fsm *fsm) } int -run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids) +run_test(const struct eager_output_test *test) { - (void)force_endids; /* TODO: unused, remove. */ + struct fsm *nfas[MAX_PATTERNS] = {0}; - struct fsm_union_entry entries[MAX_PATTERNS] = {0}; - - allow_extra_outputs = false; - - size_t fsms_used = 0; + size_t nfas_used = 0; int ret = 0; int log = 0; @@ -68,24 +64,12 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo for (size_t i = 0; i < MAX_PATTERNS; i++) { const char *p = test->patterns[i]; if (test->patterns[i] == NULL) { break; } - const size_t len = strlen(p); - struct fsm_union_entry *e = &entries[fsms_used]; - - /* For sake of these patterns, they are anchored if the first/last - * character is '^' and '$', respectively. This is too simplistic - * for the general case, though. */ - if (len > 0) { - if (p[0] == '^') { e->anchored_start = true; } - if (p[len - 1] == '$') { e->anchored_end = true; } - /* fprintf(stderr, "%s: p[%zd]: '%s', start %d, end %d\n", */ - /* __func__, fsms_used, p, e->anchored_start, e->anchored_end); */ - } struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); assert(fsm != NULL); if (log) { - fprintf(stderr, "==== source DFA %zd (pre det+min)\n", i); + fprintf(stderr, "==== source NFA %zd\n", i); if (log > 1) { dump(fsm); fsm_eager_output_dump(stderr, fsm); @@ -94,12 +78,12 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo } } - e->fsm = fsm; - fsms_used++; + nfas[i] = fsm; + nfas_used++; } - /* If there's only one pattern this just returns fsms[0]. */ - struct fsm *fsm = fsm_union_repeated_pattern_group(fsms_used, entries, NULL); + const size_t id_base = 1; /* offset by 1 because 0 is used as end-of-list */ + struct fsm *fsm = fsm_union_repeated_pattern_group(nfas_used, nfas, NULL, id_base); assert(fsm != NULL); if (log) { @@ -238,11 +222,7 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo assert(ret == 1); } - if (!allow_extra_outputs) { - assert(outputs.used == expected_id_count); - } else { - assert(outputs.used >= expected_id_count); - } + assert(outputs.used >= expected_id_count); size_t floor = 0; for (size_t exp_i = 0; exp_i < outputs.used; exp_i++) { diff --git a/tests/eager_output/utils.h b/tests/eager_output/utils.h index 672c01977..02f8427c9 100644 --- a/tests/eager_output/utils.h +++ b/tests/eager_output/utils.h @@ -48,7 +48,7 @@ int cmp_output(const void *pa, const void *pb); int -run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids); +run_test(const struct eager_output_test *test); struct cb_info { size_t used; From 971bac6d718eeb1657265f3f8cb56b6eff04fd66 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 5 Feb 2025 15:35:44 -0500 Subject: [PATCH 38/54] fsm_union_repeated_pattern_group: fix linkage for mixed start anchoring. --- src/libfsm/union.c | 64 ++++++++++++++++--- ...tput_mixed_anchored_and_unanchored_start.c | 18 ++++++ 2 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index f381d7cd6..e3c7311ae 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -357,6 +357,56 @@ start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, fsm_stat return true; } +static bool +copy_noncycle_labeled_edges_from_epsilon_closure_iter(struct analysis_info *ainfo, + const struct fsm *nfa, fsm_state_t usl_i, fsm_state_t s_i, struct state_set **seen) +{ + const struct fsm_state *s = &nfa->states[s_i]; + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + /* copy its labeled edges to ainfo->repeatable_firsts, + * unless they lead back to the unanchored_start_loop */ + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != usl_i) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, + nfa->alloc, info.symbols, info.to)) { + return false; + } + } + } + + if (!state_set_add(seen, nfa->alloc, usl_i)) { return false; } + + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (state_set_contains(*seen, ns_i)) { continue; } + if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, usl_i, ns_i, seen)) { + return false; + } + } + + return true; +} + +static bool +copy_noncycle_labeled_edges_from_epsilon_closure(struct analysis_info *ainfo, + const struct fsm *nfa, fsm_state_t unanchored_start_loop_id) +{ + struct state_set *seen = NULL; /* empty set */ + if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, + unanchored_start_loop_id, unanchored_start_loop_id, &seen)) { + return false; + } + state_set_free(seen); + + return true; +} + static bool analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) { @@ -396,15 +446,11 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) assert(ainfo->unanchored_start_loop == NO_STATE); ainfo->unanchored_start_loop = ns_i; - /* copy its non-self labeled edges to ainfo->repeatable_firsts */ - edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); - while (edge_set_group_iter_next(&egi, &info)) { - if (info.to != ns_i) { - if (!edge_set_add_bulk(&ainfo->repeatable_firsts, - nfa->alloc, info.symbols, info.to)) { - goto alloc_fail; - } - } + /* Copy labeled edges from the unanchored start loop and its epsilon + * closure to ainfo->repeatable_firsts, except for edges leading back + * to the unanchored start loop. */ + if (!copy_noncycle_labeled_edges_from_epsilon_closure(ainfo, nfa, ns_i)) { + goto alloc_fail; } } else { /* likewise, a state without a dot self-edge is the anchored start */ diff --git a/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c b/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c new file mode 100644 index 000000000..d18a67de7 --- /dev/null +++ b/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c @@ -0,0 +1,18 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "(^|[^A-Z])abc", + }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "xabc", .expected_ids = { 1 } }, + { .input = "xyz abc", .expected_ids = { 1 } }, + { .input = "Xabc", .expect_fail = true }, + }, + }; + + return run_test(&test); +} From d4e2d2c6fffa9a5f0171a391f236f1fe198335eb Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 11:38:10 -0500 Subject: [PATCH 39/54] Add tests, fix anchoring bugs in fsm_union_repeated_pattern_group. There are two bugs captured in eager_output_unanchored_end_plus.c: - Regexes ending in '+' weren't combining correctly, because analysis wasn't properly handling the construction for matching but optionally repeating the last character. - Eager matching after consuming a single character from the start state wasn't linked correctly to the global_unanchored_start_loop, so while the labeled edges were copied the eager output was lost. The other test files are focused on variants of that -- the + and start cases individually, and when + precedes a `()` subtree with more than one character. --- src/libfsm/union.c | 474 ++++++++++++------ .../eager_output_unanchored_end_plus.c | 19 + .../eager_output_unanchored_end_plus_min.c | 18 + .../eager_output_unanchored_end_plus_min2.c | 19 + ...ger_output_unanchored_end_plus_multichar.c | 19 + 5 files changed, 394 insertions(+), 155 deletions(-) create mode 100644 tests/eager_output/eager_output_unanchored_end_plus.c create mode 100644 tests/eager_output/eager_output_unanchored_end_plus_min.c create mode 100644 tests/eager_output/eager_output_unanchored_end_plus_min2.c create mode 100644 tests/eager_output/eager_output_unanchored_end_plus_multichar.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index e3c7311ae..9acdddf01 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -27,8 +27,11 @@ #include "endids.h" #define LOG_UNION_ARRAY 0 -#define LOG_ANALYZE_GROUP_NFA_RESULTS 0 +#define LOG_ANALYZE_GROUP_NFA 0 +#define LOG_AFTER_MODIFY_GROUP_NFA 0 +#define LOG_ANALYZE_GROUP_NFA_RESULTS (0 || LOG_ANALYZE_GROUP_NFA > 1) #define LOG_UNION_REPEATED_PATTERN_GROUP 0 +#define LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT 0 #define NO_STATE (fsm_state_t)-1 @@ -68,6 +71,9 @@ struct analysis_info { * epsilon edge to the combined NFA's global_unanchored_end_loop. * The old unanchored_end_loop will be disconnected. */ fsm_state_t eager_match_state; + + /* These states need an epsilon edge added to the eager_matched_state. */ + struct state_set *needs_indirect_epsilon_edge_to_eager_match_state; }; struct fsm * @@ -253,37 +259,48 @@ dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set } #endif -/* If there's a labeled edge to an end state, check if the label set is - * only [\n] and there's also an epsilon edge to the same end state. - * This represents an anchored end in the NFA. */ +/* For each state in the epsilon closure, if there's a labeled edge + * to an end state, check if the label set is only [\n] and there's + * also an epsilon edge to the same end state. + * If so, this represents an anchored end in the NFA. */ static bool -has_epsilon_and_newline_edges_to_end(const struct fsm *nfa, fsm_state_t s_i, fsm_state_t *dst_end) +has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, struct state_set *eclosure, + fsm_state_t s_i, fsm_state_t *dst_end) { - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; + struct state_iter si; + state_set_reset(eclosure, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + assert(ns_i < nfa->statecount); + const struct fsm_state *ns = &nfa->states[ns_i]; - if (state_set_empty(s->epsilons)) { return false; } - if (edge_set_empty(s->edges)) { return false; } + if (state_set_empty(ns->epsilons)) { continue; } + if (edge_set_empty(ns->edges)) { continue; } - struct edge_group_iter iter; - struct edge_group_iter_info info; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + /* Look for an edge set with only '\n' */ + if ((info.symbols[0] != (1ULL << '\n')) + || info.symbols[1] || info.symbols[2] || info.symbols[3]) { + continue; + } - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - /* Look for an edge set with only '\n' */ - if ((info.symbols[0] != (1ULL << '\n')) - || info.symbols[1] || info.symbols[2] || info.symbols[3]) { - continue; - } + /* If it's an end, look for an epsilon leeding to the same destination */ + if (fsm_isend(nfa, info.to)) { + struct state_iter inner_si; + fsm_state_t os_i; - if (fsm_isend(nfa, info.to)) { - struct state_iter si; - fsm_state_t os_i; - state_set_reset(s->epsilons, &si); - while (state_set_next(&si, &os_i)) { - if (os_i == info.to) { - *dst_end = info.to; - return true; + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + + state_set_reset(s->epsilons, &inner_si); + while (state_set_next(&inner_si, &os_i)) { + if (os_i == info.to) { + *dst_end = info.to; + return true; + } } } } @@ -293,123 +310,89 @@ has_epsilon_and_newline_edges_to_end(const struct fsm *nfa, fsm_state_t s_i, fsm } static bool -has_labeled_edge_to_unanchored_end_loop(const struct fsm *nfa, - fsm_state_t s_i, fsm_state_t unanchored_end_loop) +has_labeled_edge_to_eclosure_with_unanchored_end_loop(const struct fsm *nfa, + struct state_set **eclosures, + fsm_state_t s_i, fsm_state_t unanchored_end_loop, + fsm_state_t *indirect_dst) { if (unanchored_end_loop == NO_STATE) { return false; } assert(unanchored_end_loop < nfa->statecount); - /* The unanchored_end_loop's self-edge doesn't count here. */ - if (s_i == unanchored_end_loop) { return false; } - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; - struct edge_group_iter iter; - struct edge_group_iter_info info; - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - if (info.to == unanchored_end_loop) { - return true; - } - } - return false; -} - -static bool -start_state_epsilon_closure_matches_empty_string__iter(const struct fsm *nfa, - fsm_state_t s_i, struct state_set **seen, bool *result) -{ - if (*result) { return true; } - if (!state_set_add(seen, nfa->alloc, s_i)) { return false; } - - if (fsm_isend(nfa, s_i)) { - *result = true; - return true; - } - - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; + const struct state_set *s_eclosure = eclosures[s_i]; + /* For every state in s_i's epsilon closure, check if it has + * a labeled edge to a state with the unanchored_end_loop + * in its epsilon closure. */ struct state_iter si; - state_set_reset(s->epsilons, &si); + state_set_reset(s_eclosure, &si); fsm_state_t ns_i; while (state_set_next(&si, &ns_i)) { - if (!state_set_contains(*seen, ns_i)) { - if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, ns_i, seen, result)) { - return false; - } - if (*result) { break; } - } - } + /* The unanchored_end_loop's self-edge doesn't count here. */ + if (ns_i == unanchored_end_loop) { continue; } - return true; -} - -/* Does the start state's epsilon closure match the empty string? - * Returns false on error, otherwise returns true and sets *result. */ -static bool -start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, fsm_state_t start, bool *result) -{ - struct state_set *seen = NULL; /* empty set */ - if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, start, &seen, result)) { return false; } - state_set_free(seen); - - return true; -} - -static bool -copy_noncycle_labeled_edges_from_epsilon_closure_iter(struct analysis_info *ainfo, - const struct fsm *nfa, fsm_state_t usl_i, fsm_state_t s_i, struct state_set **seen) -{ - const struct fsm_state *s = &nfa->states[s_i]; + /* FIXME: this should only apply to the original state, not its epsilon closure...right? */ + if (ns_i != s_i) { continue; } - struct edge_group_iter egi; - struct edge_group_iter_info info; + assert(ns_i < nfa->statecount); + const struct fsm_state *ns = &nfa->states[ns_i]; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to < nfa->statecount); + const struct state_set *to_eclosure = eclosures[info.to]; + + struct state_iter dst_si; + state_set_reset(to_eclosure, &dst_si); + fsm_state_t dst_s_i; + while (state_set_next(&dst_si, &dst_s_i)) { + if (dst_s_i == unanchored_end_loop) { + if (info.to != unanchored_end_loop) { + *indirect_dst = info.to; + } - /* copy its labeled edges to ainfo->repeatable_firsts, - * unless they lead back to the unanchored_start_loop */ - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); - while (edge_set_group_iter_next(&egi, &info)) { - if (info.to != usl_i) { - if (!edge_set_add_bulk(&ainfo->repeatable_firsts, - nfa->alloc, info.symbols, info.to)) { - return false; + return true; + } } } } - if (!state_set_add(seen, nfa->alloc, usl_i)) { return false; } - - struct state_iter si; - state_set_reset(s->epsilons, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - if (state_set_contains(*seen, ns_i)) { continue; } - if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, usl_i, ns_i, seen)) { - return false; - } - } - - return true; + return false; } static bool -copy_noncycle_labeled_edges_from_epsilon_closure(struct analysis_info *ainfo, - const struct fsm *nfa, fsm_state_t unanchored_start_loop_id) +start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, const struct state_set *eclosure) { - struct state_set *seen = NULL; /* empty set */ - if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, - unanchored_start_loop_id, unanchored_start_loop_id, &seen)) { - return false; + struct state_iter si; + state_set_reset(eclosure, &si); + + fsm_state_t s_i; + while (state_set_next(&si, &s_i)) { + if (fsm_isend(nfa, s_i)) { return true; } } - state_set_free(seen); - return true; + return false; } +static const struct fsm_options dump_nfa_opt = { + .io = FSM_IO_STR, + .ambig = AMBIG_MULTIPLE, + .case_ranges = 1, + .consolidate_edges = 1, + .group_edges = 1, +}; + static bool analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) { + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "==== %s\n", __func__); + if (LOG_ANALYZE_GROUP_NFA > 1) { + fsm_print(stderr, nfa, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + } + } + memset(ainfo, 0x00, sizeof(*ainfo)); ainfo->start = NO_STATE; ainfo->unanchored_start_loop = NO_STATE; @@ -425,42 +408,67 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) const size_t state_count = fsm_countstates(nfa); assert(ainfo->start < state_count); - { - const struct fsm_state *s = &nfa->states[ainfo->start]; + struct state_set **eclosures = epsilon_closure(nfa); + if (eclosures == NULL) { + return false; + } + /* First pass: Iterate over the start state's epsilon edges, + * attempting to identify the unanchored start loop and anchored + * start states (if present). + * + * Note: This uses the start state's epsilon set rather than its + * epsilon closure because (by construction) the unanchored + * start loop and anchored start states will both be directly + * connected to the start state. Using the epsilon closure can + * mis-identify the unanchored *end* loop as the start loop, if + * there is a path with only epsilon edges between them. */ + struct state_iter si; + state_set_reset(nfa->states[ainfo->start].epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (ns_i == ainfo->start) { continue; } + + /* If there's a state in the start state's epsilon set that + * has a dot self-edge, it's the unanchored start loop. */ + if (has_dot_self_edge(nfa, ns_i)) { + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "%s: unanchored_start_loop found on state %d\n", __func__, ns_i); + } + /* there can be only one */ + assert(ainfo->unanchored_start_loop == NO_STATE + || ainfo->unanchored_start_loop == ns_i); + ainfo->unanchored_start_loop = ns_i; + continue; + } else { + /* Otherwise, a state without a dot self-edge is the anchored start. */ + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); + } + assert(ainfo->anchored_start == NO_STATE || ainfo->anchored_start == ns_i); + ainfo->anchored_start = ns_i; + continue; + } + } + + /* Copy labeled edges from the unanchored start loop and + * its epsilon closure to ainfo->repeatable_firsts, except + * for edges leading back to the unanchored start loop. */ + if (ainfo->unanchored_start_loop != NO_STATE) { struct state_iter si; - state_set_reset(s->epsilons, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - if (ns_i == ainfo->start) { continue; } + state_set_reset(eclosures[ainfo->unanchored_start_loop], &si); + fsm_state_t cs_i; + while (state_set_next(&si, &cs_i)) { + assert(cs_i < nfa->statecount); + const struct fsm_state *cs = &nfa->states[cs_i]; /* closure state */ struct edge_group_iter egi; struct edge_group_iter_info info; - assert(ns_i < state_count); - const struct fsm_state *ns = &nfa->states[ns_i]; - - /* if there's a state in the start state's epsilon closure that - * has a dot self-edge, it's the unanchored start loop */ - if (has_dot_self_edge(nfa, ns_i)) { - assert(ainfo->unanchored_start_loop == NO_STATE); - ainfo->unanchored_start_loop = ns_i; - - /* Copy labeled edges from the unanchored start loop and its epsilon - * closure to ainfo->repeatable_firsts, except for edges leading back - * to the unanchored start loop. */ - if (!copy_noncycle_labeled_edges_from_epsilon_closure(ainfo, nfa, ns_i)) { - goto alloc_fail; - } - } else { - /* likewise, a state without a dot self-edge is the anchored start */ - assert(ainfo->anchored_start == NO_STATE); - ainfo->anchored_start = ns_i; - - /* copy its labeled edges to ainfo->anchored_firsts */ - edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); - while (edge_set_group_iter_next(&egi, &info)) { - if (!edge_set_add_bulk(&ainfo->anchored_firsts, + edge_set_group_iter_reset(cs->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != ainfo->unanchored_start_loop) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, nfa->alloc, info.symbols, info.to)) { goto alloc_fail; } @@ -469,12 +477,33 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) } } + /* Copy labeled edges from the anchored start and its epsilon + * closure to ainfo->anchored_firsts. */ + if (ainfo->anchored_start != NO_STATE) { + struct state_iter si; + state_set_reset(eclosures[ainfo->anchored_start], &si); + fsm_state_t cs_i; + while (state_set_next(&si, &cs_i)) { + assert(cs_i < nfa->statecount); + const struct fsm_state *cs = &nfa->states[cs_i]; + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + edge_set_group_iter_reset(cs->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (!edge_set_add_bulk(&ainfo->anchored_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } + /* If the start state always matches, set a flag noting that it will need special handling * later. It's arguably pointless to combine "" with other regexes, because it will always * trivially match, but otherwise it would never match. */ - if (!start_state_epsilon_closure_matches_empty_string(nfa, ainfo->start, &ainfo->nullable)) { - goto alloc_fail; - } + ainfo->nullable = start_state_epsilon_closure_matches_empty_string(nfa, eclosures[ainfo->start]); /* If there's a state with a dot self-edge and an epsilon edge to an end state, it's * the unanchored end loop. There should only be one. */ @@ -499,17 +528,24 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) /* Collect states that lead to an anchored end or eager match. */ for (size_t s_i = 0; s_i < state_count; s_i++) { fsm_state_t dst_end = NO_STATE; - if (has_epsilon_and_newline_edges_to_end(nfa, s_i, &dst_end)) { + if (has_epsilon_and_newline_edges_to_same_end(nfa, eclosures[s_i], s_i, &dst_end)) { if (!state_set_add(&ainfo->anchored_ends, nfa->alloc, dst_end)) { goto alloc_fail; } } - if (has_labeled_edge_to_unanchored_end_loop(nfa, s_i, ainfo->unanchored_end_loop)) { + fsm_state_t indirect_dst = NO_STATE; + if (has_labeled_edge_to_eclosure_with_unanchored_end_loop(nfa, eclosures, s_i, ainfo->unanchored_end_loop, &indirect_dst)) { if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { goto alloc_fail; } } + + if (indirect_dst != NO_STATE) { + if (!state_set_add(&ainfo->needs_indirect_epsilon_edge_to_eager_match_state, nfa->alloc, indirect_dst)) { + goto alloc_fail; + } + } } #if LOG_ANALYZE_GROUP_NFA_RESULTS @@ -522,10 +558,17 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) dump_edge_set(stderr, "repeatable_firsts", ainfo->unanchored_start_loop, ainfo->repeatable_firsts); } #endif + + closure_free(nfa, eclosures, state_count); + return true; alloc_fail: fprintf(stderr, "alloc fail\n"); + if (eclosures != NULL) { + closure_free(nfa, eclosures, state_count); + } + return false; } @@ -534,7 +577,8 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) * the libfsm API for this (and it shouldn't be necessary in general), but * if it gets one later this can be replaced. */ static bool -replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to) +replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to, + bool *found) { if (old_to == NO_STATE) { /* nothing to do */ @@ -552,6 +596,7 @@ replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to struct edge_group_iter_info info; edge_set_group_iter_reset(old_edges, EDGE_GROUP_ITER_ALL, &iter); while (edge_set_group_iter_next(&iter, &info)) { + if (info.to == old_to) { *found = true; } if (!edge_set_add_bulk(&new_edges, nfa->alloc, info.symbols, info.to == old_to ? new_to : info.to)) { return false; @@ -577,6 +622,7 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t { const bool nullable_and_unanchored_end = ainfo->nullable && ainfo->unanchored_end_loop != NO_STATE; + const bool log = 0 || (LOG_ANALYZE_GROUP_NFA > 0); /* Add the eager match state if there are eager match states * or a nullable unanchored end. This will link to the global NFA's @@ -585,25 +631,47 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t if (!fsm_addstate(nfa, &ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: added eager_match_state %d\n", __func__, ainfo->eager_match_state); + } /* Set eager match ID on new eager_match_state. */ const fsm_output_id_t oid = (fsm_output_id_t)(id + id_base); if (!fsm_seteageroutput(nfa, ainfo->eager_match_state, oid)) { return false; } + if (log) { + fprintf(stderr, "%s: set eager_output id %d on eager_match_state %d\n", + __func__, oid, ainfo->eager_match_state); + } /* For every state in eager_matches, replace every edge leading to * the unanchored_end_loop with an edge with the same labels to - * eager_match_state. */ + * eager_match_state. + * + * If the labeled edge does not directly lead to the unanchored_end_loop, + * then add an epsilon edge from wherever it leads to eager_match_state + * instead. */ struct state_iter si; state_set_reset(ainfo->eager_matches, &si); fsm_state_t ems_i; while (state_set_next(&si, &ems_i)) { + bool found = false; if (!replace_labeled_edge(nfa, ems_i, - ainfo->unanchored_end_loop, ainfo->eager_match_state)) { + ainfo->unanchored_end_loop, ainfo->eager_match_state, &found)) { return false; } + if (log) { + if (found) { + fprintf(stderr, "%s: replacing labeled edges from eager_match_state %d to unanchored_end_loop %d with edge to new eager_match_state %d\n", + __func__, ems_i, ainfo->unanchored_end_loop, ainfo->eager_match_state); + } else if (!found && ainfo->unanchored_end_loop != NO_STATE) { + fprintf(stderr, "%s: not found: labeled edges from eager_match_state %d to unanchored_end_loop %d\n", + __func__, ems_i, ainfo->unanchored_end_loop); + } + } + /* The state must not link to the unanchored end loop anymore. * Doing so will cause a combinatorial explosion that makes * combining more ~10 NFAs incredibly expensive. */ @@ -616,6 +684,16 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t assert(info.to != ainfo->unanchored_end_loop); } } + + state_set_reset(ainfo->needs_indirect_epsilon_edge_to_eager_match_state, &si); + fsm_state_t intermediate_i; + while (state_set_next(&si, &intermediate_i)) { + if (!fsm_addedge_epsilon(nfa, intermediate_i, ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: adding epsilon edge from intermediate eager match state %d to new eager_match_state %d\n", + __func__, intermediate_i, ainfo->eager_match_state); + } + } } /* If the group NFA matches the empty string and has an unanchored end, then @@ -630,6 +708,10 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t if (!state_set_add(&s->epsilons, nfa->alloc, ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: adding epsilon edge from unanchored_start_loop %d to eager_match_state %d\n", + __func__, ainfo->unanchored_start_loop, ainfo->eager_match_state); + } } } @@ -643,9 +725,21 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t if (!fsm_endid_set(nfa, anchored_end_state, end_id)) { return false; } + if (log) { + fprintf(stderr, "%s: setting endid %d on anchored_end_state %d\n", + __func__, end_id, anchored_end_state); + } } } +#if LOG_AFTER_MODIFY_GROUP_NFA + fprintf(stderr, "=== after %s\n", __func__); + fsm_print(stderr, nfa, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + fsm_endid_dump(stderr, nfa); + fsm_eager_output_dump(stderr, nfa); + +#endif + return true; } @@ -740,6 +834,8 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addstate(res, &global_end)) { goto fail; } if (!fsm_addstate(res, &global_unanchored_end_loop)) { goto fail; } + /* do this later, combining NFAs may rebase the state IDs */ +#if 0 /* link the start to the global unanchored start loop and anchored start. */ if (log) { fprintf(stderr, "link_before: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", @@ -757,6 +853,7 @@ fsm_union_repeated_pattern_group(size_t nfa_count, } if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } +#endif if (bases != NULL) { memset(bases, 0x00, nfa_count * sizeof(bases[0])); @@ -781,6 +878,10 @@ fsm_union_repeated_pattern_group(size_t nfa_count, struct fsm *merged = fsm_merge(res, fsm, &combine_info); if (merged == NULL) { goto fail; } + if (log) { + fprintf(stderr, "merged: bases a %d and b %d\n", combine_info.base_a, combine_info.base_b); + } + /* Update offsets if res had its state IDs shifted forward. */ global_start += combine_info.base_a; global_unanchored_start_loop += combine_info.base_a; @@ -807,6 +908,11 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_end)) { goto fail; } + if (log) { + fprintf(stderr, "eager_match_state: adding epsilon EMS %d -> global_unanchored_end_loop %d and EMS %d -> global_end %d\n", + ainfo->eager_match_state, global_unanchored_end_loop, + ainfo->eager_match_state, global_end); + } /* If the NFA matches the empty string and is not anchored at the end, then * add an epsilon edge from the global start directly to its eager match state. @@ -816,6 +922,10 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(merged, global_start, ainfo->eager_match_state)) { goto fail; } + if (log) { + fprintf(stderr, "nullable: global_start %d -> eager_match_state %d\n", + global_start, ainfo->eager_match_state); + } } } else { /* If the NFA matches an end-anchored empty string, then add an epsilon edge from @@ -829,6 +939,11 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(merged, global_start, anchored_end_state)) { goto fail; } + if (log) { + fprintf(stderr, "nullable & anchored ends: global_start %d -> anchored_end_state %d\n", + global_start, anchored_end_state); + } + /* It should only be necessary to link one, since that's enough * for determinisation to carry the end id back to the start's * epsilon closure. */ @@ -850,6 +965,10 @@ fsm_union_repeated_pattern_group(size_t nfa_count, info.symbols, info.to)) { goto fail; } + if (log) { + fprintf(stderr, "anchored_firsts: adding global_anchored_start %d -> info.to %d (with same labels)\n", + global_anchored_start, info.to); + } } /* Link the global_unanchored_start_loop to group FSM paths that aren't @@ -861,15 +980,53 @@ fsm_union_repeated_pattern_group(size_t nfa_count, info.symbols, info.to)) { goto fail; } + + if (log) { + fprintf(stderr, "repeatable_firsts: adding global_unanchored_start_loop %d -> info.to %d (same edges)\n", + global_unanchored_start_loop, info.to); + } + } + + + /* Add an epsilon edge from the global unanchored start loop to the NFA's. + * Without this, eager outputs for eager matches in the start state's epsilon + * closure may get lost during determinisation. */ + if (ainfo->unanchored_start_loop != NO_STATE) { + if (!fsm_addedge_epsilon(merged, global_unanchored_start_loop, ainfo->unanchored_start_loop)) { + goto fail; + } + + if (log) { + fprintf(stderr, "repeatable_firsts: adding an epsilon edge from global_unanchored_start_loop %d to NFA unanchored_start_loop %d\n", + global_unanchored_start_loop, ainfo->unanchored_start_loop); + } } res = merged; } + /* link the start to the global unanchored start loop and anchored start. */ + if (log) { + fprintf(stderr, "linking: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", + global_start, global_unanchored_start_loop, global_anchored_start); + } + if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } + + /* Link the global unanchored start loop to itself. */ + if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop and global end. */ + if (log) { + fprintf(stderr, "linking: global_unanchored_end_loop %d -> global_end %d (and -> self)\n", global_unanchored_end_loop, global_end); + } + if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } + /* Link from the global_unanchored_end_loop to the global_unanchored_start_loop, * so patterns with an unanchored start can follow other patterns with an unanchored * end, possibly with other ignored input between them. */ - if (log) { + if (log || LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT) { fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", __func__, global_start, global_unanchored_start_loop, global_anchored_start, global_unanchored_end_loop, global_end); fprintf(stderr, "%s: linking global_unanchored_end_loop %d to global_unanchored_start_loop %d\n", @@ -887,6 +1044,13 @@ fsm_union_repeated_pattern_group(size_t nfa_count, free_analysis(alloc, &ainfos[i]); } + if (LOG_UNION_REPEATED_PATTERN_GROUP || LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT) { + fprintf(stderr, "==== %s output (combined, pre det+min)\n", __func__); + fsm_print(stderr, res, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + fsm_endid_dump(stderr, res); + fsm_eager_output_dump(stderr, res); + } + f_free(alloc, ainfos); return res; diff --git a/tests/eager_output/eager_output_unanchored_end_plus.c b/tests/eager_output/eager_output_unanchored_end_plus.c new file mode 100644 index 000000000..81354ffcb --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abcx+", + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcx", .expected_ids = { 1 } }, + { .input = "abcxx", .expected_ids = { 1 } }, + { .input = "z", .expected_ids = { 2 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_min.c b/tests/eager_output/eager_output_unanchored_end_plus_min.c new file mode 100644 index 000000000..3d270514e --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_min.c @@ -0,0 +1,18 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abcx+", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcx", .expected_ids = { 1 } }, + { .input = "abcxx", .expected_ids = { 1 } }, + { .input = "z", .expect_fail = true }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_min2.c b/tests/eager_output/eager_output_unanchored_end_plus_min2.c new file mode 100644 index 000000000..20d735fcc --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_min2.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "z", .expected_ids = { 1 } }, + { .input = "Xz", .expected_ids = { 1 } }, + { .input = "XzX", .expected_ids = { 1 } }, + { .input = "zX", .expected_ids = { 1 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_multichar.c b/tests/eager_output/eager_output_unanchored_end_plus_multichar.c new file mode 100644 index 000000000..40d3e5d24 --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_multichar.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abc(xy)+", + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcxy", .expected_ids = { 1 } }, + { .input = "abcxyxy", .expected_ids = { 1 } }, + { .input = "z", .expected_ids = { 2 } }, + }, + }; + + return run_test(&test); +} From b7fd1bc059c1878aee621c362992475190ac3a6f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 12:12:15 -0500 Subject: [PATCH 40/54] Interface change: Add 'const'. --- src/libfsm/closure.c | 4 ++-- src/libfsm/internal.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index 3993afcda..165fa4964 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -128,7 +128,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s } struct state_set ** -epsilon_closure(struct fsm *fsm) +epsilon_closure(const struct fsm *fsm) { struct state_set **closures; fsm_state_t s; @@ -190,7 +190,7 @@ epsilon_closure(struct fsm *fsm) } void -closure_free(struct fsm *fsm, struct state_set **closures, size_t n) +closure_free(const struct fsm *fsm, struct state_set **closures, size_t n) { fsm_state_t s; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 46997c82a..094723fdb 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -94,10 +94,10 @@ state_hasnondeterminism(const struct fsm *fsm, fsm_state_t state, struct bm *bm) * for states, with wrapper to populate malloced array of user-facing structs. */ struct state_set ** -epsilon_closure(struct fsm *fsm); +epsilon_closure(const struct fsm *fsm); void -closure_free(struct fsm *fsm, struct state_set **closures, size_t n); +closure_free(const struct fsm *fsm, struct state_set **closures, size_t n); /* * Internal free function that invokes free(3) by default, or a user-provided From b79100a92f8f08d7eb4ab0f9a191467fdeb04118 Mon Sep 17 00:00:00 2001 From: Ricky Hosfelt Date: Fri, 7 Feb 2025 14:45:17 -0500 Subject: [PATCH 41/54] update CI to lock to ubuntu 22.04 for clang 14.0.0 --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfb6182f2..35fa5c7f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: checkout: name: "Checkout" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Cache checkout @@ -44,7 +44,7 @@ jobs: pcre_suite: name: "Import PCRE suite" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ build ] # for cvtpcre steps: @@ -130,7 +130,7 @@ jobs: build: name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ checkout ] strategy: @@ -216,7 +216,7 @@ jobs: # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ checkout, build ] strategy: @@ -301,7 +301,7 @@ jobs: test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ build ] strategy: @@ -362,7 +362,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] @@ -470,7 +470,7 @@ jobs: test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 needs: [ pcre_suite ] # and also build, but pcre_suite gives us that strategy: @@ -531,7 +531,7 @@ jobs: docs: name: Documentation - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ checkout ] env: @@ -582,7 +582,7 @@ jobs: install: name: Install - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ build, docs ] env: @@ -643,7 +643,7 @@ jobs: fpm: name: Package ${{ matrix.pkg }} - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ install ] env: From 9040973d886e3040990c6dc2bff9614371fa7196 Mon Sep 17 00:00:00 2001 From: Ricky Hosfelt Date: Fri, 7 Feb 2025 14:59:07 -0500 Subject: [PATCH 42/54] keep matrix.os so we can easily add more os variations in the future --- .github/workflows/ci.yml | 51 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35fa5c7f4..157506a14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,10 @@ jobs: pcre_suite: name: "Import PCRE suite" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-22.04 ] needs: [ build ] # for cvtpcre steps: @@ -71,7 +74,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -79,7 +82,7 @@ jobs: id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitrary build, just for cvtpcre - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -130,14 +133,14 @@ jobs: build: name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ checkout ] strategy: fail-fast: true matrix: san: [ NO_SANITIZER, AUSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -169,7 +172,7 @@ jobs: key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && steps.cache-build.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-22.04' && steps.cache-build.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake electric-fence @@ -216,14 +219,14 @@ jobs: # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ checkout, build ] strategy: fail-fast: false matrix: san: [ NO_SANITIZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake, pmake ] debug: [ EXPENSIVE_CHECKS, DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -258,7 +261,7 @@ jobs: run: find ${{ env.wc }} -type f -name '*.c' | sort -r | head -5 | xargs touch - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install pmake bmake pcregrep @@ -301,14 +304,14 @@ jobs: test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ build ] strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -329,7 +332,7 @@ jobs: key: checkout-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake pcregrep electric-fence @@ -362,7 +365,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] @@ -370,7 +373,7 @@ jobs: fail-fast: false matrix: san: [ FUZZER ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -388,7 +391,7 @@ jobs: key: checkout-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake @@ -470,14 +473,14 @@ jobs: test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} needs: [ pcre_suite ] # and also build, but pcre_suite gives us that strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -492,7 +495,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && matrix.san == 'EFENCE' + if: matrix.os == 'ubuntu-22.04' && matrix.san == 'EFENCE' run: | uname -a sudo apt-get install electric-fence @@ -506,7 +509,7 @@ jobs: ${{ matrix.cc }} --version - name: Dependencies (Ubuntu/Go) - if: matrix.os == 'ubuntu' && (matrix.lang == 'go' || matrix.lang == 'goasm') + if: matrix.os == 'ubuntu-22.04' && (matrix.lang == 'go' || matrix.lang == 'goasm') run: | uname -a sudo apt-get install golang @@ -524,7 +527,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst @@ -587,7 +590,7 @@ jobs: env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -601,7 +604,7 @@ jobs: key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' && steps.cache-prefix.outputs.cache-hit != 'true' + if: env.os == 'ubuntu-22.04' && steps.cache-prefix.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake @@ -648,7 +651,7 @@ jobs: env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -661,7 +664,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' + if: env.os == 'ubuntu-22.04' run: | uname -a sudo gem install --no-document fpm From 64ba90348866d1747b54a50f6a09b66226a4a6b4 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 14:56:08 -0500 Subject: [PATCH 43/54] fuzz/target.c: fsm_union_repeated_pattern_group interface changes. --- fuzz/target.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 316c5ad57..b7e0f3f7b 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -508,8 +508,6 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - enum re_is_anchored_res anchorage[MAX_PATTERNS] = {0}; - /* for each pattern, attempt to compile to a DFA */ for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { const char *p = env.patterns[p_i]; @@ -528,14 +526,9 @@ fuzz_eager_output(const uint8_t *data, size_t size) continue; /* invalid regex */ } - const fsm_output_id_t endid = (fsm_output_id_t)p_i; - ret = fsm_seteageroutputonends(fsm, endid); - assert(ret == 1); - if (verbose) { fprintf(stderr, "==== pattern %zd, pre det\n", p_i); fsm_dump(stderr, fsm); - fsm_eager_output_dump(stderr, fsm); fprintf(stderr, "====\n"); fsm_state_t c = fsm_countstates(fsm); @@ -544,12 +537,6 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - ret = fsm_determinise(fsm); - assert(ret == 1); - - ret = fsm_minimise(fsm); - assert(ret == 1); - fsm_state_t start; if (!fsm_getstart(fsm, &start)) { fsm_free(fsm); @@ -578,7 +565,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) /* copy and combine fsms into one DFA */ { size_t used = 0; - struct fsm_union_entry entries[MAX_PATTERNS] = {0}; + struct fsm *nfas[MAX_PATTERNS] = {0}; for (size_t i = 0; i < env.fsm_count; i++) { /* there can be gaps, fsms[] lines up with patterns[] */ @@ -604,9 +591,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - entries[used].fsm = cp; - entries[used].anchored_start = anchorage[i] & RE_IS_ANCHORED_START; - entries[used].anchored_end = anchorage[i] & RE_IS_ANCHORED_END; + nfas[used] = cp; used++; } @@ -615,7 +600,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) } /* consumes entries[] */ - struct fsm *fsm = fsm_union_repeated_pattern_group(used, entries, NULL); + struct fsm *fsm = fsm_union_repeated_pattern_group(used, nfas, NULL, 0); assert(fsm != NULL); if (verbose) { From e80d3c521895a9a42d27905ff8c5e95a311d562e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 14:56:49 -0500 Subject: [PATCH 44/54] union: Fix trivial memory leak. --- src/libfsm/union.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 9acdddf01..7ef7bb278 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -770,6 +770,7 @@ free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) { state_set_free(ainfo->anchored_ends); state_set_free(ainfo->eager_matches); + state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); edge_set_free(alloc, ainfo->anchored_firsts); edge_set_free(alloc, ainfo->repeatable_firsts); } From e91d25bc266d88a500c30dd1314fa5d38854bbb4 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 14:57:01 -0500 Subject: [PATCH 45/54] union.c: Add comments for assertions. Fuzzing has produced inputs that cause this to fail, but they all depend on embedded '\0' characters. I wasn't able to reproduce the failure without those present, but I will investigate further later. For now, adding a TODO. --- src/libfsm/union.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 7ef7bb278..a0790c3f9 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -435,7 +435,17 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) if (LOG_ANALYZE_GROUP_NFA) { fprintf(stderr, "%s: unanchored_start_loop found on state %d\n", __func__, ns_i); } - /* there can be only one */ + + /* TODO: There is only one unanchored start loop, but in obscure cases it may + * be difficult to distinguish between the USL and the unanchored end loop or + * other intermediate .* loops. The real USL will strictly appear before any + * other such loops in the graph. + * + * For now, assert that there is only one, because it's safer to have this + * loudly fail at compile time than produce an incorrect graph. Fuzzing has + * produced some inputs that make this fail, but currently they seem to + * depend on having a '\0' character embedded in the middle, which would + * normally be rejected by this point. */ assert(ainfo->unanchored_start_loop == NO_STATE || ainfo->unanchored_start_loop == ns_i); ainfo->unanchored_start_loop = ns_i; @@ -445,7 +455,10 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) if (LOG_ANALYZE_GROUP_NFA) { fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); } + + /* TODO: This, too, can fail in obscure cases and needs further investigation. */ assert(ainfo->anchored_start == NO_STATE || ainfo->anchored_start == ns_i); + ainfo->anchored_start = ns_i; continue; } From 7a434af8fe007894649e0ac58190632d4cee0d40 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 14 Feb 2025 13:41:24 -0500 Subject: [PATCH 46/54] Switch to collecting an anchored_start state set, not just one state. `(^|wax-)((?:banana|^apple))` is an example of a regex that needs multiple anchored_start states linked in order to combine correctly. --- src/libfsm/union.c | 22 ++++++++++--------- ...ger_output_mixed_start_anchor_regression.c | 20 +++++++++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 tests/eager_output/eager_output_mixed_start_anchor_regression.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index a0790c3f9..9062e1dbd 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -48,8 +48,8 @@ struct analysis_info { /* The end state following the unanchored end loop. */ fsm_state_t unanchored_end_loop_end; - /* State that links to paths only reachable from the beginning of input. */ - fsm_state_t anchored_start; + /* States that link to paths only reachable from the beginning of input. */ + struct state_set *anchored_starts; /* States leading to an anchored end. */ struct state_set *anchored_ends; @@ -396,7 +396,6 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) memset(ainfo, 0x00, sizeof(*ainfo)); ainfo->start = NO_STATE; ainfo->unanchored_start_loop = NO_STATE; - ainfo->anchored_start = NO_STATE; ainfo->unanchored_end_loop = NO_STATE; ainfo->unanchored_end_loop_end = NO_STATE; ainfo->eager_match_state = NO_STATE; @@ -456,10 +455,9 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); } - /* TODO: This, too, can fail in obscure cases and needs further investigation. */ - assert(ainfo->anchored_start == NO_STATE || ainfo->anchored_start == ns_i); - - ainfo->anchored_start = ns_i; + if (!state_set_add(&ainfo->anchored_starts, nfa->alloc, ns_i)) { + goto alloc_fail; + } continue; } } @@ -492,9 +490,12 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) /* Copy labeled edges from the anchored start and its epsilon * closure to ainfo->anchored_firsts. */ - if (ainfo->anchored_start != NO_STATE) { + struct state_iter si_anchored_start; + state_set_reset(ainfo->anchored_starts, &si_anchored_start); + fsm_state_t anchored_start; + while (state_set_next(&si_anchored_start, &anchored_start)) { struct state_iter si; - state_set_reset(eclosures[ainfo->anchored_start], &si); + state_set_reset(eclosures[anchored_start], &si); fsm_state_t cs_i; while (state_set_next(&si, &cs_i)) { assert(cs_i < nfa->statecount); @@ -766,11 +767,11 @@ rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) SHIFT(unanchored_start_loop); SHIFT(unanchored_end_loop); SHIFT(unanchored_end_loop_end); - SHIFT(anchored_start); SHIFT(eager_match_state); #undef SHIFT state_set_rebase(&ainfo->anchored_ends, base); + state_set_rebase(&ainfo->anchored_starts, base); state_set_rebase(&ainfo->eager_matches, base); edge_set_rebase(&ainfo->anchored_firsts, base); @@ -782,6 +783,7 @@ static void free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) { state_set_free(ainfo->anchored_ends); + state_set_free(ainfo->anchored_starts); state_set_free(ainfo->eager_matches); state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); edge_set_free(alloc, ainfo->anchored_firsts); diff --git a/tests/eager_output/eager_output_mixed_start_anchor_regression.c b/tests/eager_output/eager_output_mixed_start_anchor_regression.c new file mode 100644 index 000000000..2965dc7aa --- /dev/null +++ b/tests/eager_output/eager_output_mixed_start_anchor_regression.c @@ -0,0 +1,20 @@ +#include "utils.h" + +/* Regression: This is a case that requires an anchored_start state set + * rather than a single optional anchored_start state ID to link + * correctly. */ + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "(^|wax-)((?:banana|^apple))", + "(^|wax-)(orange)", + }, + .inputs = { + { .input = "banana", .expected_ids = { 1 } }, + }, + }; + + return run_test(&test); +} From a72fc08e4f08c91cd510fd4edea0d24b252eb0ab Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 17 Jun 2025 09:24:20 -0400 Subject: [PATCH 47/54] retest: extra error checking for retest's fileno, isatty Something strange is happening in CI, and these may be involved. --- src/retest/main.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/retest/main.c b/src/retest/main.c index 6baa0ab7b..86a7474be 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -1232,7 +1232,15 @@ main(int argc, char *argv[]) int optlevel = 1; /* is output to a tty or not? */ - tty_output = isatty(fileno(stdout)); + int fileno_stdout = fileno(stdout); + if (fileno_stdout == -1) { + perror("fileno"); + } else { + tty_output = isatty(fileno_stdout); + if (tty_output == -1) { + perror("isatty"); + } + } /* note these defaults are the opposite than for fsm(1) */ opt.anonymous_states = 1; From f565b93c6429aa004646ea31475d06f76d175ef2 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 17 Jun 2025 10:32:44 -0400 Subject: [PATCH 48/54] cdata: change return types to void, since these functions can't fail. --- src/libfsm/print/cdata.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 1f209df7c..2bab1351f 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -183,7 +183,7 @@ struct cdata_config { #endif }; -static bool +static void generate_struct_definition(FILE *f, const struct cdata_config *config, bool comments, const char *prefix) { const bool has_endids = config->endid_buf.used > 0; @@ -305,7 +305,6 @@ generate_struct_definition(FILE *f, const struct cdata_config *config, bool comm fprintf(f, "\t};\n"); - return true; } static void @@ -412,7 +411,7 @@ generate_distinct_eager_output_id_table(FILE *f, const struct cdata_config *conf fprintf(f, "\n\t\t},\n"); } -static bool +static void generate_data(FILE *f, const struct cdata_config *config, bool comments, const char *prefix, const struct ir *ir) { @@ -619,8 +618,6 @@ generate_data(FILE *f, const struct cdata_config *config, } fprintf(f, "\t};\n"); - - return true; } static void @@ -655,7 +652,7 @@ generate_eager_output_check(FILE *f, const struct cdata_config *config, const ch } } -static bool +static void generate_interpreter(FILE *f, const struct cdata_config *config, const struct fsm_options *opt, const char *prefix) { const bool has_endids = config->endid_buf.used > 0; @@ -872,7 +869,6 @@ generate_interpreter(FILE *f, const struct cdata_config *config, const struct fs /* Got a match. */ fprintf(f, "\treturn 1; /* match */\n"); - return true; } static bool @@ -1530,17 +1526,17 @@ fsm_print_cdata(FILE *f, fprintf(f, ")\n"); fprintf(f, "{\n"); - if (!generate_struct_definition(f, &config, opt->comments, prefix)) { return -1; } - if (!generate_data(f, &config, opt->comments, prefix, ir)) { return -1; } - if (!generate_interpreter(f, &config, opt, prefix)) { return -1; } + generate_struct_definition(f, &config, opt->comments, prefix); + generate_data(f, &config, opt->comments, prefix, ir); + generate_interpreter(f, &config, opt, prefix); fprintf(f, "}\n"); fprintf(f, "\n"); } else { /* caller sets up the function head */ - if (!generate_struct_definition(f, &config, opt->comments, prefix)) { return -1; } - if (!generate_data(f, &config, opt->comments, prefix, ir)) { return -1; } - if (!generate_interpreter(f, &config, opt, prefix)) { return -1; } + generate_struct_definition(f, &config, opt->comments, prefix); + generate_data(f, &config, opt->comments, prefix, ir); + generate_interpreter(f, &config, opt, prefix); } f_free(alloc, config.dst_buf.buf); From 8f1838bbe69354af57fe754682048ea96271fbd2 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 17 Jun 2025 10:56:29 -0400 Subject: [PATCH 49/54] cdata: Error check. --- src/libfsm/print/cdata.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 2bab1351f..f44c028e1 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -1430,7 +1430,9 @@ fsm_print_cdata(FILE *f, /* First pass, figure out totals and index sizes */ struct cdata_config config; - populate_config_from_ir(&config, alloc, ir); + if (!populate_config_from_ir(&config, alloc, ir)) { + return -1; + } #if LOG_SIZES fprintf(stderr, "// config: dst_state_count %zu, start %d, dst_buf.used %zd, endid_buf.used %zd, eager_output_buf.used %zd\n", From 285f4449c08bb61277daaa4463750b43221717a3 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 17 Jun 2025 17:52:52 -0400 Subject: [PATCH 50/54] cdata: Fix NULL check. This looks like a merge error. --- src/libfsm/print/cdata.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 70d79d808..37c695a3c 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -1139,7 +1139,9 @@ increment_bitset_word_count(const struct fsm_alloc *alloc, struct bitset_words * const size_t nceil = (bws->ceil == 0 ? 8 : 2*bws->ceil); struct bitset_word_pair *npairs = f_realloc(alloc, bws->pairs, nceil * sizeof(npairs[0])); - return false; + if (npairs == NULL) { + return false; + } bws->ceil = nceil; bws->pairs = npairs; } From 6078cdff247a7618e9e1bb4006dfc238af9da4ee Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 18 Jun 2025 09:30:29 -0400 Subject: [PATCH 51/54] github workflows: ensure ${env.wc} exists. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 157506a14..cc802fe38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: checkout: name: "Checkout" runs-on: ubuntu-22.04 + run: | + mkdir -p ${{ env.wc }} steps: - name: Cache checkout From 43c851ed461e01c0b4ca7c923a3fa68fe28ca35f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 18 Jun 2025 09:50:43 -0400 Subject: [PATCH 52/54] cdata: Add a few comments about internal structs. --- src/libfsm/print/cdata.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/cdata.c b/src/libfsm/print/cdata.c index 37c695a3c..07351ca81 100644 --- a/src/libfsm/print/cdata.c +++ b/src/libfsm/print/cdata.c @@ -34,7 +34,9 @@ /* Print mode that generates C data literals for the DFA, plus a small interepreter. * This mostly exists to sidestep very expensive compilation for large data sets - * using the other C modes. -sv */ + * using the other C modes. When they generate long runs of nested switch/case + * statements, gcc and clang spend lots of time and RAM on analyzing them, and it + * can lead to builds taking several hours or even OOM-ing. -sv */ /* Whether to check the table buffers for previous instances of * the same sets of dst_states, endids, and eager_outputs. This @@ -125,12 +127,14 @@ struct cdata_config { /* numeric type for entries in .bitset_words.pairs[] */ enum id_type t_label_word_id; + /* buffer for edge destination states */ struct dst_buf { size_t ceil; size_t used; uint32_t *buf; } dst_buf; + /* buffer for endids */ struct endid_buf { size_t ceil; size_t used; @@ -138,6 +142,7 @@ struct cdata_config { } endid_buf; size_t max_endid; + /* buffer for eager output IDs */ struct eager_output_buf { size_t ceil; size_t used; @@ -145,6 +150,9 @@ struct cdata_config { } eager_output_buf; size_t max_eager_output_id; + /* Collection of distinct eager output IDs, in ascending order. + * This is used to map sequentially allocated internal IDs to the + * caller's eager output IDs, which are just arbitrary integers. */ fsm_output_id_t *eager_output_ids; size_t distinct_eager_output_id_count; From 3827dc51a1a1c4e1f49f4aadee6ffd0b0d1238ca Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 20 Jun 2025 09:09:25 -0400 Subject: [PATCH 53/54] Revert "github workflows: ensure ${env.wc} exists." This reverts commit 6078cdff247a7618e9e1bb4006dfc238af9da4ee. This is not the right place to do this, and this wasn't the actual root cause of the CI failures. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc802fe38..157506a14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,6 @@ jobs: checkout: name: "Checkout" runs-on: ubuntu-22.04 - run: | - mkdir -p ${{ env.wc }} steps: - name: Cache checkout From 26f90c4d92f25e8bf3a2c09803243266e1e4431c Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 28 Aug 2025 21:28:34 +0100 Subject: [PATCH 54/54] Stray const. --- src/libfsm/print/c.c | 2 +- src/libfsm/print/vmc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index b3c053ce1..3a6778545 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -584,7 +584,7 @@ fsm_print_c(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, ",\n"); - fprintf(f, "\tconst unsigned *id"); + fprintf(f, "\tunsigned *id"); break; case AMBIG_MULTIPLE: diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 2fef2da65..e6d0eaece 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -588,7 +588,7 @@ fsm_print_vmc(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, ",\n"); - fprintf(f, "\tconst unsigned *id"); + fprintf(f, "\tunsigned *id"); break; case AMBIG_MULTIPLE: