From db702ff73d08ee8120863e59a7c698e001de816f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 16 Feb 2023 13:52:31 -0500 Subject: [PATCH 001/373] Add src/adt/idmap.c, a state -> ID set map. --- Makefile | 1 + include/adt/idmap.h | 58 ++++++ src/adt/Makefile | 1 + src/adt/idmap.c | 392 ++++++++++++++++++++++++++++++++++++++ tests/idmap/Makefile | 19 ++ tests/idmap/idmap_basic.c | 136 +++++++++++++ 6 files changed, 607 insertions(+) create mode 100644 include/adt/idmap.h create mode 100644 src/adt/idmap.c create mode 100644 tests/idmap/Makefile create mode 100644 tests/idmap/idmap_basic.c diff --git a/Makefile b/Makefile index a94baac61..73a675013 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,7 @@ SUBDIR += src SUBDIR += tests/capture SUBDIR += tests/complement SUBDIR += tests/gen +SUBDIR += tests/idmap SUBDIR += tests/intersect #SUBDIR += tests/ir # XXX: fragile due to state numbering SUBDIR += tests/eclosure diff --git a/include/adt/idmap.h b/include/adt/idmap.h new file mode 100644 index 000000000..064fd15d1 --- /dev/null +++ b/include/adt/idmap.h @@ -0,0 +1,58 @@ +#ifndef IDMAP_H +#define IDMAP_H + +/* Mapping between one fsm_state_t and a set of + * unsigned IDs. The implementation assumes that both + * IDs are sequentially assigned and don't need a sparse + * mapping -- it will handle 10 -> [1, 3, 47] well, but + * not 1000000 -> [14, 524288, 1073741823]. */ + +#include + +#include "fsm/fsm.h" +#include "fsm/alloc.h" + +struct idmap; /* Opaque handle. */ + +struct idmap * +idmap_new(const struct fsm_alloc *alloc); + +void +idmap_free(struct idmap *m); + +/* Associate a value with a state (if not already present.) + * Returns 1 on success, or 0 on allocation failure. */ +int +idmap_set(struct idmap *m, fsm_state_t state_id, unsigned value); + +/* How many values are associated with an ID? */ +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id); + +/* Get the values associated with an ID. + * + * Returns 1 on success and writes them into the buffer, in ascending + * order, with the count in *written (if non-NULL). + * + * Returns 0 on error (insufficient buffer space). */ +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written); + +/* Iterator callback. */ +typedef void +idmap_iter_fun(fsm_state_t state_id, unsigned value, void *opaque); + +/* Iterate over the ID map. State IDs may be yielded out of order, + * values will be in ascending order. */ +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque); + +/* Iterate over the values associated with a single state + * (in ascending order). */ +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque); + +#endif diff --git a/src/adt/Makefile b/src/adt/Makefile index 05199f2dc..64ad7429f 100644 --- a/src/adt/Makefile +++ b/src/adt/Makefile @@ -2,6 +2,7 @@ SRC += src/adt/alloc.c SRC += src/adt/bitmap.c +SRC += src/adt/idmap.c SRC += src/adt/internedstateset.c SRC += src/adt/priq.c SRC += src/adt/path.c diff --git a/src/adt/idmap.c b/src/adt/idmap.c new file mode 100644 index 000000000..ca169b71e --- /dev/null +++ b/src/adt/idmap.c @@ -0,0 +1,392 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "adt/idmap.h" + +#include "adt/alloc.h" +#include "adt/hash.h" +#include "adt/u64bitset.h" + +#include +#include +#include + +#define NO_STATE ((fsm_state_t)-1) + +#define DEF_BUCKET_COUNT 4 + +struct idmap { + const struct fsm_alloc *alloc; + unsigned bucket_count; + unsigned buckets_used; + + /* All buckets' values are assumed to be large + * enough to store this value, and they will all + * grow as necessary. */ + unsigned max_value; + + /* Basic linear-probing, add-only hash table. */ + struct idmap_bucket { + fsm_state_t state; /* Key. NO_STATE when empty. */ + + /* values[] is always either NULL or has at least + * max_value + 1 bits; all grow on demand. */ + uint64_t *values; + } *buckets; +}; + +static unsigned +value_words(unsigned max_value) { + if (max_value == 0) { + /* Still allocate one word, for storing 0. */ + return 1; + } else { + return u64bitset_words(max_value); + } +} + +struct idmap * +idmap_new(const struct fsm_alloc *alloc) +{ + struct idmap *res = NULL; + struct idmap_bucket *buckets = NULL; + + res = f_malloc(alloc, sizeof(*res)); + if (res == NULL) { + goto cleanup; + } + + buckets = f_calloc(alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + goto cleanup; + } + + for (size_t i = 0; i < DEF_BUCKET_COUNT; i++) { + buckets[i].state = NO_STATE; + } + + res->alloc = alloc; + res->buckets_used = 0; + res->bucket_count = DEF_BUCKET_COUNT; + res->max_value = 0; + res->buckets = buckets; + + return res; + +cleanup: + f_free(alloc, res); + f_free(alloc, buckets); + return NULL; +} + +void +idmap_free(struct idmap *m) +{ + if (m == NULL) { + return; + } + + for (size_t i = 0; i < m->bucket_count; i++) { + if (m->buckets[i].state == NO_STATE) { + continue; + } + f_free(m->alloc, m->buckets[i].values); + } + + f_free(m->alloc, m->buckets); + f_free(m->alloc, m); +} + +static int +grow_bucket_values(struct idmap *m, unsigned old_words, unsigned new_words) +{ + assert(new_words > old_words); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + assert(b->values == NULL); + continue; + } + + uint64_t *nv = f_calloc(m->alloc, + new_words, sizeof(nv[0])); + if (nv == NULL) { + return 0; + } + + for (size_t w_i = 0; w_i < old_words; w_i++) { + nv[w_i] = b->values[w_i]; + } + f_free(m->alloc, b->values); + b->values = nv; + } + return 1; +} + +static int +grow_buckets(struct idmap *m) +{ + const size_t ocount = m->bucket_count; + const size_t ncount = 2*ocount; + assert(ncount > m->bucket_count); + + struct idmap_bucket *nbuckets = f_calloc(m->alloc, + ncount, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + nbuckets[nb_i].state = NO_STATE; + } + + const size_t nmask = ncount - 1; + + for (size_t ob_i = 0; ob_i < ocount; ob_i++) { + const struct idmap_bucket *ob = &m->buckets[ob_i]; + if (ob->state == NO_STATE) { + continue; + } + + const uint64_t h = hash_id(ob->state); + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + struct idmap_bucket *nb = &nbuckets[(h + nb_i) & nmask]; + if (nb->state == NO_STATE) { + nb->state = ob->state; + nb->values = ob->values; + break; + } else { + assert(nb->state != ob->state); + /* collision */ + continue; + } + } + } + + f_free(m->alloc, m->buckets); + + m->buckets = nbuckets; + m->bucket_count = ncount; + + return 1; +} + +int +idmap_set(struct idmap *m, fsm_state_t state_id, + unsigned value) +{ + assert(state_id != NO_STATE); + + const uint64_t h = hash_id(state_id); + if (value > m->max_value) { + const unsigned ovw = value_words(m->max_value); + const unsigned nvw = value_words(value); + /* If this value won't fit in the existing value + * arrays, then grow them all. We do not track the + * number of bits in each individual array. */ + if (nvw > ovw && !grow_bucket_values(m, ovw, nvw)) { + return 0; + } + m->max_value = value; + } + + assert(m->max_value >= value); + + if (m->buckets_used >= m->bucket_count/2) { + if (!grow_buckets(m)) { + return 0; + } + } + + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == state_id) { + assert(b->values != NULL); + u64bitset_set(b->values, value); + return 1; + } else if (b->state == NO_STATE) { + b->state = state_id; + assert(b->values == NULL); + + const unsigned vw = value_words(m->max_value); + b->values = f_calloc(m->alloc, + vw, sizeof(b->values[0])); + if (b->values == NULL) { + return 0; + } + m->buckets_used++; + + u64bitset_set(b->values, value); + return 1; + } else { + continue; /* collision */ + } + + } + + assert(!"unreachable"); + return 0; +} + +static const struct idmap_bucket * +get_bucket(const struct idmap *m, fsm_state_t state_id) +{ + const uint64_t h = hash_id(state_id); + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == NO_STATE) { + return NULL; + } else if (b->state == state_id) { + return b; + } + } + + return NULL; +} + +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return 0; + } + assert(b->values != NULL); + + size_t res = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + /* This could use popcount64(w). */ + if (w == 0) { + continue; + } + for (uint64_t bit = 1; bit; bit <<= 1) { + if (w & bit) { + res++; + } + } + } + + return res; +} + +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + if (written != NULL) { + *written = 0; + } + return 1; + } + + size_t buf_offset = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + if (buf_offset * sizeof(buf[0]) >= buf_size) { + return 0; + } + buf[buf_offset] = 64*w_i + b_i; + buf_offset++; + } + } + } + + if (written != NULL) { + *written = buf_offset; + } + return 1; +} + +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + continue; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + cb(b->state, v, opaque); + } + } + } + } +} + +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + /* if N contiguous bits are all zero, skip them all at once */ +#define BLOCK_BITS 16 + uint64_t block = ((uint64_t)1 << BLOCK_BITS) - 1; + size_t block_count = 0; + + uint64_t b_i = 0; + while (b_i < 64) { + if ((w & block) == 0) { + block <<= BLOCK_BITS; + b_i += BLOCK_BITS; + continue; + } + + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + cb(b->state, v, opaque); + block_count++; + } + b_i++; + block <<= 1; + } + +#define CHECK 0 +#if CHECK + size_t check_count = 0; + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + check_count++; + } + } + assert(block_count == check_count); +#endif + } +} diff --git a/tests/idmap/Makefile b/tests/idmap/Makefile new file mode 100644 index 000000000..aee01f565 --- /dev/null +++ b/tests/idmap/Makefile @@ -0,0 +1,19 @@ +.include "../../share/mk/top.mk" + +TEST.tests/idmap != ls -1 tests/idmap/idmap*.c +TEST_SRCDIR.tests/idmap = tests/idmap +TEST_OUTDIR.tests/idmap = ${BUILD}/tests/idmap + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +INCDIR.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += src/adt +.endfor + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +test:: ${TEST_OUTDIR.tests/idmap}/res${n} +SRC += ${TEST_SRCDIR.tests/idmap}/idmap${n}.c +CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += -UNDEBUG -D_DEFAULT_SOURCE -std=c99 +${TEST_OUTDIR.tests/idmap}/run${n}: ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c} -o ${TEST_OUTDIR.tests/idmap}/run${n} ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o +${TEST_OUTDIR.tests/idmap}/res${n}: ${TEST_OUTDIR.tests/idmap}/run${n} + ( ${TEST_OUTDIR.tests/idmap}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/idmap}/res${n} +.endfor diff --git a/tests/idmap/idmap_basic.c b/tests/idmap/idmap_basic.c new file mode 100644 index 000000000..19f44d56e --- /dev/null +++ b/tests/idmap/idmap_basic.c @@ -0,0 +1,136 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#define DEF_LIMIT 10 +#define DEF_SEED 0 + +/* Thes numbers were chose to get a reasonable variety, + * but also some duplicated values as the input grows. */ +#define MAX_GEN_VALUES 23 +#define ID_MASK ((1 << 9) - 1) +#define VALUE_MASK ((1 << 10) - 1) + +static void +dump_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + /* fprintf(stderr, " -- state %d, value %u\n", state_id, value); */ + assert(state_id <= ID_MASK); + assert(value <= VALUE_MASK); + (void)opaque; +} + +static int +cmp_u(const void *pa, const void *pb) +{ + const unsigned a = *(unsigned *)pa; + const unsigned b = *(unsigned *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +int main(int argc, char **argv) { + const size_t limit = (argc > 1 ? atoi(argv[1]) : DEF_LIMIT); + const unsigned seed = (argc > 2 ? atoi(argv[2]) : DEF_SEED); + + (void)argc; + (void)argv; + struct idmap *m = idmap_new(NULL); + + srandom(seed); + + /* Fill the table with random data */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t value_count = random() % MAX_GEN_VALUES; + + for (size_t v_i = 0; v_i < value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + if (!idmap_set(m, id, v)) { + assert(!"failed to set"); + } + } + } + + idmap_iter(m, dump_cb, NULL); + + srandom(seed); + + size_t got_buf_ceil = MAX_GEN_VALUES; + unsigned *got_buf = malloc(got_buf_ceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + + /* Reset the PRNG and read back the same data. */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t generated_value_count = random() % MAX_GEN_VALUES; + + /* Note: This can occasionally differ from + * generated_value_count, because the same id or values + * may have been generated more than once. As long as + * all the values match, it's fine. */ + const size_t value_count = idmap_get_value_count(m, id); + + if (value_count > got_buf_ceil) { + size_t nceil = got_buf_ceil; + while (nceil <= value_count) { + nceil *= 2; + } + free(got_buf); + got_buf = malloc(nceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + got_buf_ceil = nceil; + } + + size_t written; + if (!idmap_get(m, id, + got_buf_ceil * sizeof(got_buf[0]), got_buf, + &written)) { + assert(!"failed to get"); + } + assert(written == value_count); + + unsigned gen_buf[MAX_GEN_VALUES]; + + for (size_t v_i = 0; v_i < generated_value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + gen_buf[v_i] = v; + } + qsort(gen_buf, generated_value_count, sizeof(gen_buf[0]), cmp_u); + + /* Every generated value should appear in the buffer. + * There may be more in the buffer; ignore them. */ + size_t v_i = 0; + for (size_t gen_i = 0; gen_i < generated_value_count; gen_i++) { + int found = 0; + const unsigned gv = gen_buf[gen_i]; + assert(value_count <= got_buf_ceil); + /* got_buf should be sorted, so we can pick up where we left off */ + while (v_i < value_count) { + if (gv == got_buf[v_i]) { + /* Intentionally don't increment v_i on match, + * because gen_buf can repeat values. */ + found = 1; + break; + } + v_i++; + } + if (!found) { + fprintf(stderr, "NOT FOUND: state %d -- value: %u\n", + id, gv); + return EXIT_FAILURE; + } + } + } + + free(got_buf); + idmap_free(m); + return EXIT_SUCCESS; +} From 41a92d797b8caca61453555c4b7ccbff52ef2058 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 1 May 2023 15:37:20 -0400 Subject: [PATCH 002/373] Switch adt code to `uint64_t` for hashes, not `unsigned long`. Remove include/adt/mappingset.h, it's no longer used. --- include/adt/hashrec.h | 2 +- include/adt/mappingset.h | 42 ---------------------------------------- include/adt/stateset.h | 4 +++- src/adt/hashrec.c | 2 +- src/adt/stateset.c | 2 +- 5 files changed, 6 insertions(+), 46 deletions(-) delete mode 100644 include/adt/mappingset.h diff --git a/include/adt/hashrec.h b/include/adt/hashrec.h index 54816286e..545a20960 100644 --- a/include/adt/hashrec.h +++ b/include/adt/hashrec.h @@ -7,7 +7,7 @@ #ifndef ADT_HASHREC_H #define ADT_HASHREC_H -unsigned long +uint64_t hashrec(const void *p, size_t n); #endif diff --git a/include/adt/mappingset.h b/include/adt/mappingset.h deleted file mode 100644 index d4f91105e..000000000 --- a/include/adt/mappingset.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 Shannon F. Stewman - * - * See LICENCE for the full copyright terms. - */ - -#ifndef ADT_MAPPINGSET_H -#define ADT_MAPPINGSET_H - -struct fsm_alloc; -struct mapping_set; -struct mapping; - -struct mapping_iter { - struct hashset_iter iter; -}; - -struct mapping_set * -mapping_set_create(const struct fsm_alloc *a, - unsigned long (*hash)(const struct mapping *a), - int (*cmp)(const void *a, const void *b)); - -void -mapping_set_free(struct mapping_set *set); - -struct mapping * -mapping_set_add(struct mapping_set *set, struct mapping *item); - -struct mapping * -mapping_set_contains(const struct mapping_set *set, const struct mapping *item); - -void -mapping_set_clear(struct mapping_set *set); - -struct mapping * -mapping_set_first(const struct mapping_set *set, struct mapping_iter *it); - -struct mapping * -mapping_set_next(struct mapping_iter *it); - -#endif - diff --git a/include/adt/stateset.h b/include/adt/stateset.h index 83e835467..becd263ad 100644 --- a/include/adt/stateset.h +++ b/include/adt/stateset.h @@ -7,6 +7,8 @@ #ifndef ADT_STATESET_H #define ADT_STATESET_H +#include + struct set; struct fsm_alloc; struct state_set; @@ -72,7 +74,7 @@ state_set_rebase(struct state_set **set, fsm_state_t base); void state_set_replace(struct state_set **set, fsm_state_t old, fsm_state_t new); -unsigned long +uint64_t state_set_hash(const struct state_set *set); #endif diff --git a/src/adt/hashrec.c b/src/adt/hashrec.c index 6a341710b..7348cbc38 100644 --- a/src/adt/hashrec.c +++ b/src/adt/hashrec.c @@ -23,7 +23,7 @@ static const unsigned char hashk[] = { 0x14, 0xa8, 0xff, 0x36, 0x15, 0x16, 0x2c, 0xf7, 0xf4, 0xce, 0xb8, 0x66, 0x74, 0xf4, 0x3d, 0x64, }; -unsigned long +uint64_t hashrec(const void *p, size_t n) { uint64_t h = 0; diff --git a/src/adt/stateset.c b/src/adt/stateset.c index c1cff9933..eefd704f6 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -659,7 +659,7 @@ state_set_replace(struct state_set **setp, fsm_state_t old, fsm_state_t new) } } -unsigned long +uint64_t state_set_hash(const struct state_set *set) { if (set == NULL) { From ec30958032f930f4d7c3b4bba5554308b8fc9d1b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 2 May 2023 13:45:59 -0400 Subject: [PATCH 003/373] stateset: Add `EXPENSIVE_CHECKS` guard around expensive asserts. This and src/libfsm/internal.h's EXPENSIVE_CHECKS should move to a common place later. --- src/adt/stateset.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index eefd704f6..ab3542161 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -15,6 +15,12 @@ #include #include +/* FIXME: This should use the same define as currently exists in + * src/fsm/internal.h. This is used here because the calls to + * state_set_contains change the order of growth. */ +#define EXPENSIVE_CHECKS 0 + + /* * TODO: now fsm_state_t is a numeric index, this could be a dynamically * allocated bitmap, instead of a set.inc's array of items. @@ -276,7 +282,9 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->i = 1; } +#if EXPENSIVE_CHECKS assert(state_set_contains(set, state)); +#endif return 1; } @@ -478,7 +486,9 @@ state_set_remove(struct state_set **setp, fsm_state_t state) set->i--; } +#if EXPENSIVE_CHECKS assert(!state_set_contains(set, state)); +#endif } int From d1ce686861ab8575a3257a3b1875a42b4e95d80c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 2 May 2023 13:46:58 -0400 Subject: [PATCH 004/373] Namespace epsilon_closure and closure_free with "fsm_". These symbols are exported in the library. --- src/fsm/main.c | 4 ++-- src/libfsm/closure.c | 4 ++-- src/libfsm/epsilons.c | 4 ++-- src/libfsm/internal.h | 4 ++-- src/libfsm/libfsm.syms | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/fsm/main.c b/src/fsm/main.c index cd5eea90f..0f97a9a81 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -598,7 +598,7 @@ main(int argc, char *argv[]) size_t n; struct state_iter it; - closures = epsilon_closure(fsm); + closures = fsm_epsilon_closure(fsm); if (closures == NULL) { return -1; } @@ -619,7 +619,7 @@ main(int argc, char *argv[]) printf("\n"); } - closure_free(closures, fsm->statecount); + fsm_closure_free(closures, fsm->statecount); return 0; } else { diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index fa2d0c783..9ebf48eb9 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -128,7 +128,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s } struct state_set ** -epsilon_closure(struct fsm *fsm) +fsm_epsilon_closure(struct fsm *fsm) { struct state_set **closures; fsm_state_t s; @@ -190,7 +190,7 @@ epsilon_closure(struct fsm *fsm) } void -closure_free(struct state_set **closures, size_t n) +fsm_closure_free(struct state_set **closures, size_t n) { fsm_state_t s; diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index e87d9d974..06da7739e 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -66,7 +66,7 @@ fsm_remove_epsilons(struct fsm *nfa) assert(nfa != NULL); TIME(&pre); - eclosures = epsilon_closure(nfa); + eclosures = fsm_epsilon_closure(nfa); TIME(&post); DIFF_MSEC("epsilon_closure", pre, post, NULL); @@ -167,7 +167,7 @@ fsm_remove_epsilons(struct fsm *nfa) res = 1; cleanup: if (eclosures != NULL) { - closure_free(eclosures, state_count); + fsm_closure_free(eclosures, state_count); } return res; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 4e25b3950..1dde1fc96 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -162,10 +162,10 @@ state_hasnondeterminism(const struct fsm *fsm, fsm_state_t state, struct bm *bm) * for states, with wrapper to populate malloced array of user-facing structs. */ struct state_set ** -epsilon_closure(struct fsm *fsm); +fsm_epsilon_closure(struct fsm *fsm); void -closure_free(struct state_set **closures, size_t n); +fsm_closure_free(struct state_set **closures, size_t n); /* * Internal free function that invokes free(3) by default, or a user-provided diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 3474fa289..f9f6bf003 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -115,8 +115,8 @@ fsm_print_cfrag # XXX: workaround for lx make_ir # XXX: workaround for lx free_ir # XXX: workaround for lx -epsilon_closure # XXX: workaround for fsm -closure_free # XXX: workaround for fsm +fsm_epsilon_closure # XXX: workaround for fsm +fsm_closure_free # XXX: workaround for fsm fsm_mergeab From deb14a8cf410c55ac7496deb316f063bf6e95018 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 3 May 2023 08:52:01 -0400 Subject: [PATCH 005/373] tests/capture/: Delete outdated capture tests. These rely on either direct FSM construction or setting captures via a capture path, but captures are now implemented by a completely different mechanism. New tests will be added with the capture code in a future commit. --- tests/capture/capture0.c | 27 ---- tests/capture/capture1.c | 28 ---- tests/capture/capture2.c | 31 ---- tests/capture/capture3.c | 221 -------------------------- tests/capture/capture4.c | 242 ----------------------------- tests/capture/capture5.c | 150 ------------------ tests/capture/capture_concat1.c | 133 ---------------- tests/capture/capture_concat2.c | 133 ---------------- tests/capture/capture_long_trail.c | 28 ---- tests/capture/capture_union1.c | 140 ----------------- tests/capture/capture_union2.c | 117 -------------- 11 files changed, 1250 deletions(-) delete mode 100644 tests/capture/capture0.c delete mode 100644 tests/capture/capture1.c delete mode 100644 tests/capture/capture2.c delete mode 100644 tests/capture/capture3.c delete mode 100644 tests/capture/capture4.c delete mode 100644 tests/capture/capture5.c delete mode 100644 tests/capture/capture_concat1.c delete mode 100644 tests/capture/capture_concat2.c delete mode 100644 tests/capture/capture_long_trail.c delete mode 100644 tests/capture/capture_union1.c delete mode 100644 tests/capture/capture_union2.c diff --git a/tests/capture/capture0.c b/tests/capture/capture0.c deleted file mode 100644 index 4e7d0e3fa..000000000 --- a/tests/capture/capture0.c +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" - -/* /a(bcd)e/ */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abcde", - { - { 1, 4 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture1.c b/tests/capture/capture1.c deleted file mode 100644 index e9fe53ab9..000000000 --- a/tests/capture/capture1.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" -/* (a(b(c))) */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abc", - { - { 0, 3 }, - { 1, 3 }, - { 2, 3 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture2.c b/tests/capture/capture2.c deleted file mode 100644 index 20a1c1bac..000000000 --- a/tests/capture/capture2.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" - -/* (a(b((c))(d))) */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abcd", - { - { 0, 4 }, - { 1, 4 }, - { 2, 3 }, - { 2, 3 }, - { 3, 4 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture3.c b/tests/capture/capture3.c deleted file mode 100644 index 9d4d284ab..000000000 --- a/tests/capture/capture3.c +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "captest.h" - -/* Combine 3 fully disjoint FSMs: - * - * - 0: "(a(b))" - * - 1: "(cd(e))" - * - 2: "(fgh(i))" - * - * Shift the captures for 1 and 2 forward and use/combine - * opaques on them to track which one(s) matched. - * - * This tracking of which DFA matched should be more directly - * supported by the API later. */ - -static void -check(const struct fsm *fsm, const char *string, - unsigned end_id, unsigned capture_base); - -static void -det_and_min(const char *tag, struct fsm *fsm); - -int main(void) { - struct fsm *f_ab = captest_fsm_of_string("ab", 0); - struct fsm *f_cde = captest_fsm_of_string("cde", 1); - struct fsm *f_fghi = captest_fsm_of_string("fghi", 2); - struct fsm *f_all = NULL; - unsigned captures; - - struct fsm_combined_base_pair bases[3]; - struct fsm *fsms[3]; - - assert(f_ab); - assert(f_cde); - assert(f_fghi); - - /* set captures */ -#define SET_CAPTURE(FSM, STATE, CAPTURE, TYPE) \ - if (!fsm_set_capture_action(FSM, STATE, CAPTURE, TYPE)) { \ - fprintf(stderr, "failed to set capture on line %d\n", \ - __LINE__); \ - exit(EXIT_FAILURE); \ - } - - /* (a(b)) */ - if (!fsm_capture_set_path(f_ab, 0, 0, 2)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_ab, 1, 1, 2)) { - exit(EXIT_FAILURE); - } - - /* (cd(e)) */ - if (!fsm_capture_set_path(f_cde, 0, 0, 3)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_cde, 1, 2, 3)) { - exit(EXIT_FAILURE); - } - - /* (fgh(i)) */ - if (!fsm_capture_set_path(f_fghi, 0, 0, 4)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_fghi, 1, 3, 4)) { - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "\n=== f_ab...\n"); - fsm_print_fsm(stderr, f_ab); - fsm_capture_dump(stderr, "#### f_ab", f_ab); - - fprintf(stderr, "\n=== f_cde...\n"); - fsm_print_fsm(stderr, f_cde); - fsm_capture_dump(stderr, "#### f_cde", f_cde); - - fprintf(stderr, "\n=== f_fghi...\n"); - fsm_print_fsm(stderr, f_fghi); - fsm_capture_dump(stderr, "#### f_fghi", f_fghi); -#endif - - /* determinise and minimise each before unioning */ - det_and_min("ab", f_ab); - det_and_min("cde", f_cde); - det_and_min("fghi", f_fghi); - - /* union them */ - fsms[0] = f_ab; - fsms[1] = f_cde; - fsms[2] = f_fghi; - - f_all = fsm_union_array(3, fsms, bases); - assert(f_all != NULL); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "=== unioned f_ab with f_cde... (CB ab: %u, cde: %u)\n", - bases[0].capture, bases[1].capture); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "#### f_all", f_all); -#endif - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "=== unioned f_all with f_fghi... (CB fghi: %u), %u captures\n", - bases[2].capture, fsm_countcaptures(f_all)); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "#### f_all #2", f_all); -#endif - - if (!fsm_determinise(f_all)) { - fprintf(stderr, "NOPE %d\n", __LINE__); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "#### f_all", f_all); -#endif - - captures = fsm_countcaptures(f_all); - if (captures != 6) { - fprintf(stderr, "expected 6 captures, got %u\n", captures); - exit(EXIT_FAILURE); - } - - check(f_all, "ab", 0, bases[0].capture); - check(f_all, "cde", 1, bases[1].capture); - check(f_all, "fghi", 2, bases[2].capture); - - - fsm_free(f_all); - - return 0; -} - -static void -det_and_min(const char *tag, struct fsm *fsm) -{ - if (!fsm_determinise(fsm)) { - fprintf(stderr, "Failed to determise '%s'\n", tag); - exit(EXIT_FAILURE); - } - - if (!fsm_minimise(fsm)) { - fprintf(stderr, "Failed to minimise '%s'\n", tag); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, tag, fsm); -#endif - -} - -static void -check(const struct fsm *fsm, const char *string, - unsigned end_id, unsigned capture_base) -{ - int exec_res; - size_t i; - struct captest_input input; - fsm_state_t end; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - const size_t length = strlen(string); - const unsigned cb = capture_base; /* alias */ - - input.string = string; - input.pos = 0; - - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; - } - - exec_res = fsm_exec(fsm, captest_getc, &input, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "fsm_exec: %d for '%s', expected 1\n", - exec_res, string); - exit(EXIT_FAILURE); - } - - /* check end ID */ - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - /* check captures */ - if (0) { - fprintf(stderr, "captures for '%s' (cb %u): [%ld, %ld], [%ld, %ld]\n", - string, capture_base, - captures[0 + cb].pos[0], captures[0 + cb].pos[1], - captures[1 + cb].pos[0], captures[1 + cb].pos[1]); - } - - assert(captures[0 + cb].pos[0] == 0); - assert(captures[0 + cb].pos[1] == length); - assert(captures[1 + cb].pos[0] == length - 1); - assert(captures[1 + cb].pos[1] == length); -} diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c deleted file mode 100644 index 170cbe8b0..000000000 --- a/tests/capture/capture4.c +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "captest.h" - -/* Combine 2 mostly overlapping FSMs: - * - 0: "(abc)" - * - 1: "(ab*c)" - * and check for false positives in the match. - */ - -static struct fsm * -build_and_combine(unsigned *cb_a, unsigned *cb_b); - -static void -det_and_min(const char *tag, struct fsm *fsm); - -static struct fsm * -build_ab_c(void); - -static void -check(const struct fsm *fsm, const char *string, - unsigned expected_ends, - unsigned cb_a, size_t pa_0, size_t pa_1, - unsigned cb_b, size_t pb_0, size_t pb_1); - -int main(void) { - unsigned cb_abc, cb_ab_c; - struct fsm *f_all = build_and_combine(&cb_abc, &cb_ab_c); - unsigned captures; - const unsigned exp_0 = 1U << 0; - const unsigned exp_1 = 1U << 1; - - captures = fsm_countcaptures(f_all); - if (captures != 2) { - fprintf(stderr, "expected 2 captures, got %u\n", captures); - exit(EXIT_FAILURE); - } - - #define NO_POS FSM_CAPTURE_NO_POS - check(f_all, "abc", /* captures 0 and 1 */ - exp_0 | exp_1, - cb_abc, 0, 3, - cb_ab_c, 0, 3); - check(f_all, "ac", /* only capture 1 */ - exp_1, - cb_abc, NO_POS, NO_POS, - cb_ab_c, 0, 2); - check(f_all, "abbc", /* only capture 1 */ - exp_1, - cb_abc, NO_POS, NO_POS, - cb_ab_c, 0, 4); - - fsm_free(f_all); - - return 0; -} - -static struct fsm * -build_and_combine(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *f_abc = captest_fsm_of_string("abc", 0); - struct fsm *f_ab_c = build_ab_c(); - struct fsm *f_all; - struct fsm_combine_info ci; - - assert(f_abc); - assert(f_ab_c); - - if (!fsm_capture_set_path(f_abc, 0, 0, 3)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_ab_c, 0, 0, 3)) { - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==================== abc \n"); - fsm_print_fsm(stderr, f_abc); - fsm_capture_dump(stderr, "abc", f_abc); - - fprintf(stderr, "==================== ab*c \n"); - fsm_print_fsm(stderr, f_ab_c); - fsm_capture_dump(stderr, "ab*c", f_ab_c); -#endif - - det_and_min("abc", f_abc); - det_and_min("ab*c", f_ab_c); - - /* union them */ - f_all = fsm_union(f_abc, f_ab_c, &ci); - assert(f_all != NULL); - - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==================== post-union \n"); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "capture_actions", f_all); - fprintf(stderr, "====================\n"); -#endif - - if (!fsm_determinise(f_all)) { - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==================== post-det \n"); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "capture_actions", f_all); - fprintf(stderr, "====================\n"); -#endif - - return f_all; -} - -static void -det_and_min(const char *tag, struct fsm *fsm) -{ - if (!fsm_determinise(fsm)) { - fprintf(stderr, "Failed to determise '%s'\n", tag); - exit(EXIT_FAILURE); - } - - if (!fsm_minimise(fsm)) { - fprintf(stderr, "Failed to minimise '%s'\n", tag); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, tag, fsm); -#endif - -} - -static struct fsm * -build_ab_c(void) -{ - struct fsm *fsm = captest_fsm_with_options(); - assert(fsm != NULL); - - if (!fsm_addstate_bulk(fsm, 4)) { goto fail; } - - fsm_setstart(fsm, 0); - if (!fsm_addedge_literal(fsm, 0, 1, 'a')) { goto fail; } - - if (!fsm_addedge_literal(fsm, 1, 2, 'b')) { goto fail; } - if (!fsm_addedge_literal(fsm, 1, 3, 'c')) { goto fail; } - - if (!fsm_addedge_literal(fsm, 2, 2, 'b')) { goto fail; } - if (!fsm_addedge_literal(fsm, 2, 3, 'c')) { goto fail; } - - fsm_setend(fsm, 3, 1); - if (!fsm_setendid(fsm, 1)) { - goto fail; - } - - return fsm; - -fail: - exit(EXIT_FAILURE); -} - -static void -check(const struct fsm *fsm, const char *string, - unsigned expected_ends, - unsigned cb_a, size_t pa_0, size_t pa_1, - unsigned cb_b, size_t pb_0, size_t pb_1) -{ - int exec_res; - size_t i; - struct captest_input input; - fsm_state_t end; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - fprintf(stderr, "#### check '%s', exp: ends 0x%u, c%u: (%ld, %ld), c%u: %ld, %ld)\n", - string, expected_ends, - cb_a, pa_0, pa_1, - cb_b, pb_0, pb_1); - - input.string = string; - input.pos = 0; - - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; - } - - exec_res = fsm_exec(fsm, captest_getc, &input, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "fsm_exec: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - /* check captures */ - fprintf(stderr, "captures for '%s': [%ld, %ld], [%ld, %ld]\n", - string, - captures[0].pos[0], captures[0].pos[1], - captures[1].pos[0], captures[1].pos[1]); - assert(captures[cb_a].pos[0] == pa_0); - assert(captures[cb_a].pos[1] == pa_1); - assert(captures[cb_b].pos[0] == pb_0); - assert(captures[cb_b].pos[1] == pb_1); - - { - enum fsm_getendids_res gres; - fsm_end_id_t id_buf[2]; - size_t written; - gres = fsm_getendids(fsm, end, 2, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { - assert(!"fsm_getendids failed"); - } - - if (expected_ends == 0x2) { - assert(written == 1); - assert(id_buf[0] == 1); - } else if (expected_ends == 0x3) { - assert(written == 2); - assert(id_buf[0] == 0); - assert(id_buf[1] == 1); - } else { - assert(!"test not handled"); - } - } -} diff --git a/tests/capture/capture5.c b/tests/capture/capture5.c deleted file mode 100644 index b3a4be3ee..000000000 --- a/tests/capture/capture5.c +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#define LOG_INTERMEDIATE_FSMS 0 -#include "captest.h" - -/* Check that self edges are handled properly in the - * capture action analysis. - * - * The DFA corresponds to /a(b*)(c)/. */ - -static struct fsm * -build(void); - -static void -check(struct fsm *f, const char *input, - unsigned pa_0, unsigned pa_1, - unsigned pb_0, unsigned pb_1); - -int main(void) { - struct fsm *f = build(); - unsigned captures; - assert(f != NULL); - - captures = fsm_countcaptures(f); - if (captures != 2) { - fprintf(stderr, "expected 2 captures, got %u\n", captures); - exit(EXIT_FAILURE); - } - - check(f, "ac", - 1, 1, - 1, 2); - check(f, "abc", - 1, 2, - 2, 3); - check(f, "abbc", - 1, 3, - 3, 4); - - fsm_free(f); - - return 0; -} - -static struct fsm * -build(void) -{ - struct fsm *fsm = captest_fsm_with_options(); - - if (!fsm_addstate_bulk(fsm, 4)) { goto fail; } - - fsm_setstart(fsm, 0); - if (!fsm_addedge_literal(fsm, 0, 1, 'a')) { goto fail; } - - if (!fsm_addedge_literal(fsm, 1, 1, 'b')) { goto fail; } - if (!fsm_addedge_literal(fsm, 1, 2, 'c')) { goto fail; } - - fsm_setend(fsm, 2, 1); - - if (!fsm_capture_set_path(fsm, 0, 1, 1)) { goto fail; } - if (!fsm_capture_set_path(fsm, 1, 1, 2)) { goto fail; } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== built\n"); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "built", fsm); -#endif - - if (!fsm_determinise(fsm)) { - fprintf(stderr, "Failed to determise\n"); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after det\n"); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "after det", fsm); -#endif - - if (!fsm_minimise(fsm)) { - fprintf(stderr, "Failed to minimise\n"); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after min\n"); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "after min", fsm); -#endif - return fsm; - -fail: - exit(EXIT_FAILURE); -} - -static void -check(struct fsm *fsm, const char *string, - unsigned pa_0, unsigned pa_1, - unsigned pb_0, unsigned pb_1) -{ - int exec_res; - size_t i; - struct captest_input input; - fsm_state_t end; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - fprintf(stderr, "#### check '%s', exp: c%u: (%u, %u), c%u: %u, %u)\n", - string, - 0, pa_0, pa_1, - 1, pb_0, pb_1); - - input.string = string; - input.pos = 0; - - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; - } - - exec_res = fsm_exec(fsm, captest_getc, &input, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "fsm_exec: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - /* check captures */ - fprintf(stderr, "captures for '%s': [%ld, %ld], [%ld, %ld]\n", - string, - captures[0].pos[0], captures[0].pos[1], - captures[1].pos[0], captures[1].pos[1]); - assert(captures[0].pos[0] == pa_0); - assert(captures[0].pos[1] == pa_1); - assert(captures[1].pos[0] == pb_0); - assert(captures[1].pos[1] == pb_1); -} diff --git a/tests/capture/capture_concat1.c b/tests/capture/capture_concat1.c deleted file mode 100644 index ee9c8aaab..000000000 --- a/tests/capture/capture_concat1.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include "captest.h" - -/* concat /(ab)/ and /(cde)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_ab, size_t exp_start_ab, size_t exp_end_ab, - unsigned cb_cde, size_t exp_start_cde, size_t exp_end_cde); - -int main(void) { - unsigned cb_ab, cb_cde; /* capture base */ - struct fsm *abcde = build(&cb_ab, &cb_cde); - - check(abcde, "abcde", 1, - cb_ab, 0, 2, - cb_cde, 2, 5); - - fsm_free(abcde); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *ab = captest_fsm_of_string("ab", 0); - struct fsm *cde = captest_fsm_of_string("cde", 1); - struct fsm *abcde; - struct fsm_combine_info ci; - size_t cc_ab, cc_cde, cc_abcde; - - assert(ab); - assert(cde); - - if (!fsm_capture_set_path(ab, 0, 0, 2)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(cde, 0, 0, 3)) { - assert(!"path 1"); - } - - cc_ab = fsm_countcaptures(ab); - assert(cc_ab == 1); - - cc_cde = fsm_countcaptures(cde); - assert(cc_cde == 1); - - abcde = fsm_concat(ab, cde, &ci); - assert(abcde); - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - - cc_abcde = fsm_countcaptures(abcde); - assert(cc_abcde == cc_ab + cc_cde); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after concat: cb_ab %u, cb_cde %u\n", - *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); - - fsm_capture_dump(stderr, "#### after concat", abcde); - - fprintf(stderr, "==== determinise\n"); -#endif - - if (!fsm_determinise(abcde)) { - assert(!"determinise"); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); - - assert(fsm_countcaptures(abcde) == cc_abcde); - - fsm_capture_dump(stderr, "#### after det", abcde); -#endif - - assert(fsm_countcaptures(abcde) == cc_abcde); - return abcde; -} - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_ab, size_t exp_start_ab, size_t exp_end_ab, - unsigned cb_cde, size_t exp_start_cde, size_t exp_end_cde) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - assert(captures[cb_ab].pos[0] == exp_start_ab); - assert(captures[cb_ab].pos[1] == exp_end_ab); - - assert(captures[cb_cde].pos[0] == exp_start_cde); - assert(captures[cb_cde].pos[1] == exp_end_cde); -} diff --git a/tests/capture/capture_concat2.c b/tests/capture/capture_concat2.c deleted file mode 100644 index a8f070c7e..000000000 --- a/tests/capture/capture_concat2.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include "captest.h" - -/* concat /(abc)/ and /(de)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_ab, size_t exp_start_ab, size_t exp_end_ab, - unsigned cb_cde, size_t exp_start_cde, size_t exp_end_cde); - -int main(void) { - unsigned cb_abc, cb_de; /* capture base */ - struct fsm *abcde = build(&cb_abc, &cb_de); - - check(abcde, "abcde", 1, - cb_abc, 0, 3, - cb_de, 3, 5); - - fsm_free(abcde); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *abc = captest_fsm_of_string("abc", 0); - struct fsm *de = captest_fsm_of_string("de", 1); - struct fsm *abcde; - struct fsm_combine_info ci; - size_t cc_abc, cc_de, cc_abcde; - - assert(abc); - assert(de); - - if (!fsm_capture_set_path(abc, 0, 0, 3)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(de, 0, 0, 2)) { - assert(!"path 1"); - } - - cc_abc = fsm_countcaptures(abc); - assert(cc_abc == 1); - - cc_de = fsm_countcaptures(de); - assert(cc_de == 1); - - abcde = fsm_concat(abc, de, &ci); - assert(abcde); - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - - cc_abcde = fsm_countcaptures(abcde); - assert(cc_abcde == cc_abc + cc_de); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after concat: cb_abc %u, cb_de %u\n", - *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); - - fsm_capture_dump(stderr, "#### after concat", abcde); - - fprintf(stderr, "==== determinise\n"); -#endif - - if (!fsm_determinise(abcde)) { - assert(!"determinise"); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); - - assert(fsm_countcaptures(abcde) == cc_abcde); - - fsm_capture_dump(stderr, "#### after det", abcde); -#endif - - assert(fsm_countcaptures(abcde) == cc_abcde); - return abcde; -} - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_abc, size_t exp_start_abc, size_t exp_end_abc, - unsigned cb_de, size_t exp_start_de, size_t exp_end_de) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - assert(captures[cb_abc].pos[0] == exp_start_abc); - assert(captures[cb_abc].pos[1] == exp_end_abc); - - assert(captures[cb_de].pos[0] == exp_start_de); - assert(captures[cb_de].pos[1] == exp_end_de); -} diff --git a/tests/capture/capture_long_trail.c b/tests/capture/capture_long_trail.c deleted file mode 100644 index 349717b0f..000000000 --- a/tests/capture/capture_long_trail.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" -/* a(bcdefghijklmnopqrstuvwxy)z - * This is long enough to exercise growing the trail for - * capture action analysis. */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abcdefghijklmnopqrstuvwxyz", - { - { 1, 25 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture_union1.c b/tests/capture/capture_union1.c deleted file mode 100644 index 5d9bd2920..000000000 --- a/tests/capture/capture_union1.c +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "captest.h" - -/* union /(ab)/ and /(cde)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end); - -int main(void) { - unsigned cb_ab, cb_cde; /* capture base */ - struct fsm *abcde = build(&cb_ab, &cb_cde); - - check(abcde, "ab", 0, cb_ab, 0, 2); - check(abcde, "cde", 1, cb_cde, 0, 3); - - fsm_free(abcde); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *ab = captest_fsm_of_string("ab", 0); - struct fsm *cde = captest_fsm_of_string("cde", 1); - struct fsm *abcde; - struct fsm_combine_info ci; - size_t cc_ab, cc_cde, cc_abcde; - - assert(ab); - assert(cde); - - if (!fsm_capture_set_path(ab, 0, 0, 2)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(cde, 0, 0, 3)) { - assert(!"path 1"); - } - - cc_ab = fsm_countcaptures(ab); - assert(cc_ab == 1); - - cc_cde = fsm_countcaptures(cde); - assert(cc_cde == 1); - - abcde = fsm_union(ab, cde, &ci); - assert(abcde); - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - - cc_abcde = fsm_countcaptures(abcde); - assert(cc_abcde == cc_ab + cc_cde); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after union: cb_ab %u, cb_cde %u\n", - *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); - - fsm_capture_dump(stderr, "#### after union", abcde); - - fprintf(stderr, "==== determinise\n"); -#endif - - if (!fsm_determinise(abcde)) { - assert(!"determinise"); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); - - assert(fsm_countcaptures(abcde) == cc_abcde); - - fsm_capture_dump(stderr, "#### after det", abcde); -#endif - - assert(fsm_countcaptures(abcde) == cc_abcde); - return abcde; -} - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture got_captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, got_captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - if (got_captures[exp_capture_id].pos[0] != exp_start) { - fprintf(stderr, "capture[%u].pos[0]: exp %lu, got %lu\n", - exp_capture_id, exp_start, - got_captures[exp_capture_id].pos[0]); - exit(EXIT_FAILURE); - } - if (got_captures[exp_capture_id].pos[1] != exp_end) { - fprintf(stderr, "capture[%u].pos[1]: exp %lu, got %lu\n", - exp_capture_id, exp_end, - got_captures[exp_capture_id].pos[1]); - exit(EXIT_FAILURE); - } -} diff --git a/tests/capture/capture_union2.c b/tests/capture/capture_union2.c deleted file mode 100644 index 7fab2f18d..000000000 --- a/tests/capture/capture_union2.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "captest.h" - -/* union /(abcd)/ and /(abed)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end); - -int main(void) { - unsigned cb_abcd, cb_abed; - struct fsm *fsm = build(&cb_abcd, &cb_abed); - - check(fsm, "abcd", 0, cb_abcd, 0, 4); - check(fsm, "abed", 1, cb_abed, 0, 4); - - fsm_free(fsm); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *abcd = captest_fsm_of_string("abcd", 0); - struct fsm *abed = captest_fsm_of_string("abed", 1); - struct fsm *res; - - assert(abcd); - assert(abed); - - if (!fsm_capture_set_path(abcd, 0, 0, 4)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(abed, 0, 0, 4)) { - assert(!"path 1"); - } - - { - struct fsm *fsms[2]; - struct fsm_combined_base_pair bases[2]; - fsms[0] = abcd; - fsms[1] = abed; - res = fsm_union_array(2, fsms, bases); - assert(res); - *cb_a = bases[0].capture; - *cb_b = bases[1].capture; - } - - if (!fsm_determinise(res)) { - assert(!"determinise"); - } - - assert(fsm_countcaptures(res) == 2); - - return res; -} - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture got_captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, got_captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - if (got_captures[exp_capture_id].pos[0] != exp_start) { - fprintf(stderr, "capture[%u].pos[0]: exp %lu, got %lu\n", - exp_capture_id, exp_start, - got_captures[exp_capture_id].pos[0]); - exit(EXIT_FAILURE); - } - if (got_captures[exp_capture_id].pos[1] != exp_end) { - fprintf(stderr, "capture[%u].pos[1]: exp %lu, got %lu\n", - exp_capture_id, exp_end, - got_captures[exp_capture_id].pos[1]); - exit(EXIT_FAILURE); - } -} From ffea22e3164142e3944c1a458108af1a83c14590 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 31 May 2023 17:14:35 -0400 Subject: [PATCH 006/373] fuzz/run_fuzzer: Run single seed file when given as argument. --- fuzz/run_fuzzer | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fuzz/run_fuzzer b/fuzz/run_fuzzer index be8ba1d95..429ffa961 100755 --- a/fuzz/run_fuzzer +++ b/fuzz/run_fuzzer @@ -4,6 +4,8 @@ BUILD=../build FUZZER=${BUILD}/fuzz/fuzzer SEEDS=${BUILD}/fuzz/fuzzer_seeds +ARG=$1 + SECONDS=${SECONDS:-60} WORKERS=${WORKERS:-4} SEEDS=${SEEDS:-seeds} @@ -25,5 +27,9 @@ if [ ! -d "${SEEDS}" ]; then mkdir -p "${SEEDS}" fi -echo "\n==== ${FUZZER}" -${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +if [ -z "${ARG}" ]; then + echo "\n==== ${FUZZER}" + exec ${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +else + exec ${FUZZER} ${ARG} +fi \ No newline at end of file From 2beeff42ac594353906862a6636f11d69e43d987 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 31 May 2023 17:13:38 -0400 Subject: [PATCH 007/373] Complemely rework capture resolution. This is a big commit, unfortunately difficult to break apart further due to interface changes, metadata being passed through whole-FSM transformations, and so on. Sorry about that. - Delete code related to capture action metadata on edges. That approach made FSM transformations (determinisation, minimisation, etc.) considerably more expensive, and there were some corner cases that I wasn't able to get working correctly. - Switch to a somewhat simpler method, adapted from Russ Cox's "Regular Expression Matching: the Virtual Machine Approach". Since the capture resolution metadata (an opcode program for a virtual machine) is associated with individual end states, this combines cleanly when multiple regexes are unioned into a single large DFA that matches them all at once. - Add lots of capture regression tests, mostly from using libfsm's `fsm_generate_matches` and a fuzzer to compare behavior against PCRE. This brought many, many obscure cases to light. The updated fuzzer harness will appear in a later commit. --- include/adt/idmap.h | 5 +- include/fsm/capture.h | 43 +- include/fsm/fsm.h | 32 +- include/re/re.h | 24 +- src/adt/Makefile | 8 +- src/fsm/main.c | 4 +- src/libfsm/Makefile | 2 + src/libfsm/capture.c | 1126 +++++------- src/libfsm/capture.h | 139 +- src/libfsm/capture_internal.h | 115 -- src/libfsm/capture_log.h | 27 + src/libfsm/capture_vm.c | 193 ++ src/libfsm/capture_vm.h | 69 + src/libfsm/capture_vm_exec.c | 2063 ++++++++++++++++++++++ src/libfsm/capture_vm_log.h | 21 + src/libfsm/capture_vm_program.h | 74 + src/libfsm/clone.c | 86 +- src/libfsm/consolidate.c | 173 +- src/libfsm/determinise.c | 249 +-- src/libfsm/determinise_internal.h | 30 +- src/libfsm/endids.c | 136 +- src/libfsm/endids.h | 7 +- src/libfsm/endids_internal.h | 43 - src/libfsm/epsilons.c | 412 ++--- src/libfsm/exec.c | 96 +- src/libfsm/internal.h | 4 - src/libfsm/libfsm.syms | 15 +- src/libfsm/merge.c | 224 ++- src/libfsm/minimise.c | 579 +++++- src/libfsm/print/Makefile | 3 + src/libfsm/state.c | 24 +- src/libfsm/trim.c | 81 +- src/libfsm/union.c | 12 + src/libre/Makefile | 6 +- src/libre/ast.c | 3 + src/libre/ast.h | 19 +- src/libre/ast_analysis.c | 742 ++++++-- src/libre/ast_analysis.h | 4 +- src/libre/ast_compile.c | 1260 +++++++++---- src/libre/ast_rewrite.c | 2 +- src/libre/print/tree.c | 23 +- src/libre/re.c | 18 +- src/libre/re_capvm_compile.c | 1689 ++++++++++++++++++ src/libre/re_capvm_compile.h | 32 + src/libre/strerror.c | 2 + src/lx/parser.act | 2 +- src/re/main.c | 71 +- tests/capture/Makefile | 8 +- tests/capture/captest.c | 560 ++++-- tests/capture/captest.h | 121 +- tests/capture/capture_test_case_list.c | 1867 ++++++++++++++++++++ tests/idmap/idmap_basic.c | 3 +- tests/minimise/minimise_test_case_list.c | 3 +- theft/fuzz_capture_string_set.c | 12 +- 54 files changed, 10356 insertions(+), 2210 deletions(-) delete mode 100644 src/libfsm/capture_internal.h create mode 100644 src/libfsm/capture_log.h create mode 100644 src/libfsm/capture_vm.c create mode 100644 src/libfsm/capture_vm.h create mode 100644 src/libfsm/capture_vm_exec.c create mode 100644 src/libfsm/capture_vm_log.h create mode 100644 src/libfsm/capture_vm_program.h delete mode 100644 src/libfsm/endids_internal.h create mode 100644 src/libre/re_capvm_compile.c create mode 100644 src/libre/re_capvm_compile.h create mode 100644 tests/capture/capture_test_case_list.c diff --git a/include/adt/idmap.h b/include/adt/idmap.h index 064fd15d1..a13c9d115 100644 --- a/include/adt/idmap.h +++ b/include/adt/idmap.h @@ -39,8 +39,9 @@ int idmap_get(const struct idmap *m, fsm_state_t state_id, size_t buf_size, unsigned *buf, size_t *written); -/* Iterator callback. */ -typedef void +/* Iterator callback. + * The return value indicates whether iteration should continue. */ +typedef int idmap_iter_fun(fsm_state_t state_id, unsigned value, void *opaque); /* Iterate over the ID map. State IDs may be yielded out of order, diff --git a/include/fsm/capture.h b/include/fsm/capture.h index d3c1aaa54..0fd418925 100644 --- a/include/fsm/capture.h +++ b/include/fsm/capture.h @@ -23,41 +23,32 @@ struct fsm_capture { size_t pos[2]; }; -/* How many captures does the FSM use? */ +/* What is the max capture ID an FSM uses? */ unsigned -fsm_countcaptures(const struct fsm *fsm); +fsm_capture_ceiling(const struct fsm *fsm); /* Does a specific state have any capture actions? */ int fsm_capture_has_capture_actions(const struct fsm *fsm, fsm_state_t state); -/* Set a capture path on an FSM. This means that during matching, the - * portion of a match between the path's START and END states will be - * captured. As the FSM is transformed (determinisation, minimisation, - * unioning, etc.), the path will be converted to refer to the pair(s) - * of new states instead. If the path's END state is no longer reachable - * from its START state, then the capture path will be ignored. - * Multiple instances of the same capture_id and path are ignored. */ -int -fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, - fsm_state_t start, fsm_state_t end); - -/* Increase the base capture ID for all captures in an fsm. - * This could be used before combining multiple FSMs -- for - * example, before unioning a and b, where a has 3 captures - * and b has 2, b may be rebase'd to 3 -- so a has captures - * 0-2 and b has 3-4. */ -void -fsm_capture_rebase_capture_id(struct fsm *fsm, unsigned base); +/* Allocate a capture buffer with enough space for + * the current FSM's captures. + * + * This is provided for convenience -- the necessary array + * count can be checked with fsm_capture_ceiling, and then + * the buffer can be allocated directly. */ +struct fsm_capture * +fsm_capture_alloc_capture_buffer(const struct fsm *fsm); -/* Same, but for capture action states. */ +/* Free a capture buffer. */ void -fsm_capture_rebase_capture_action_states(struct fsm *fsm, fsm_state_t base); +fsm_capture_free_capture_buffer(const struct fsm *fsm, struct fsm_capture *capture_buffer); -/* Allocate a capture buffer with enough space for - * the current FSM's captures. */ -struct fsm_capture * -fsm_capture_alloc(const struct fsm *fsm); +/* Note that a capture is active for a particular end state. + * Using this for a non-end state is an unchecked error. */ +int +fsm_capture_set_active_for_end(struct fsm *fsm, + unsigned capture_id, fsm_state_t end_state); #ifndef NDEBUG #include diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index d8e99d9a3..30e2f3f91 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -190,6 +190,10 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq); void fsm_setend(struct fsm *fsm, fsm_state_t state, int end); +/* Associate a numeric ID with a single end state in an FSM. */ +int +fsm_setendid_state(struct fsm *fsm, fsm_state_t s, fsm_end_id_t id); + /* Associate a numeric ID with the end states in an fsm. * This can be used to track which of the original fsms matched * input when multiple fsms are combined. @@ -225,6 +229,17 @@ fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, size_t fsm_getendidcount(const struct fsm *fsm, fsm_state_t end_state); +/* Callback for iterating over end IDs. + * Returns whether iteration should continue. */ +typedef int +fsm_iterendids_cb(const struct fsm *fsm, fsm_state_t end_state, + size_t nth, fsm_end_id_t id, void *opaque); + +/* Iterate over the end IDs associated with a state, if any. */ +void +fsm_iterendids(const struct fsm *fsm, fsm_state_t state, + fsm_iterendids_cb *cb, void *opaque); + /* * Find the state (if there is just one), or add epsilon edges from all states, * for which the given predicate is true. @@ -415,8 +430,21 @@ fsm_shortest(const struct fsm *fsm, * The given FSM is expected to be a DFA. */ int -fsm_exec(const struct fsm *fsm, int (*fsm_getc)(void *opaque), void *opaque, - fsm_state_t *end, struct fsm_capture *captures); +fsm_exec(const struct fsm *fsm, + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end); + +/* Same as fsm_exec, but also populate information about captures if + * *captures is non-NULL and capture metadata is available for the DFA. + * Captures is expected to be large enough to fit captures from the FSM. + * To check, use `fsm_capture_ceiling`. + * + * The current implementation requires all input to be buffered ahead of + * time, so this takes a pointer to an input array rather than a + * character iterator. */ +int +fsm_exec_with_captures(const struct fsm *fsm, const unsigned char *input, + size_t input_length, fsm_state_t *end, + struct fsm_capture *captures, size_t capture_buf_length); /* * Callbacks which may be passed to fsm_exec(). These are conveniences for diff --git a/include/re/re.h b/include/re/re.h index deab6caed..ab5f09b39 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -20,16 +20,18 @@ enum re_dialect { }; enum re_flags { - RE_ICASE = 1 << 0, - RE_TEXT = 1 << 1, - RE_MULTI = 1 << 2, - RE_REVERSE = 1 << 3, - RE_SINGLE = 1 << 4, /* aka PCRE_DOTALL */ - RE_ZONE = 1 << 5, - RE_ANCHORED = 1 << 6, - RE_EXTENDED = 1 << 7, /* PCRE extended mode */ - RE_END_NL = 1 << 8, /* end anchor matches '\n' */ - RE_FLAGS_NONE = 0 + RE_ICASE = 1 << 0, + RE_TEXT = 1 << 1, + RE_MULTI = 1 << 2, + RE_REVERSE = 1 << 3, + RE_SINGLE = 1 << 4, /* aka PCRE_DOTALL */ + RE_ZONE = 1 << 5, + RE_ANCHORED = 1 << 6, + RE_EXTENDED = 1 << 7, /* PCRE extended mode */ + RE_NOCAPTURE = 1 << 8, /* disable captures */ + RE_END_NL = 1 << 9, /* end anchor matches '\n' */ + RE_END_NL_DISABLE = 1 << 10, /* disable end anchor matching '\n' */ + RE_FLAGS_NONE = 0 }; #define RE_ANCHOR (RE_TEXT | RE_MULTI | RE_ZONE) @@ -46,6 +48,8 @@ enum re_errno { RE_EERRNO = 1 | RE_MISC, RE_EBADDIALECT = 2 | RE_MISC, RE_EBADGROUP = 3 | RE_MISC, + RE_EUNSUPCAPTUR = 4 | RE_MISC, + RE_EUNSUPPPCRE = 5 | RE_MISC, RE_ENEGRANGE = 0 | RE_MARK | RE_GROUP, RE_ENEGCOUNT = 1 | RE_MARK | RE_GROUP, diff --git a/src/adt/Makefile b/src/adt/Makefile index 64ad7429f..6fae4e7ca 100644 --- a/src/adt/Makefile +++ b/src/adt/Makefile @@ -20,12 +20,10 @@ CFLAGS.${src} += -I src # XXX: for internal.h DFLAGS.${src} += -I src # XXX: for internal.h .endfor -# not all concrete set interfaces use all static functions from set.inc -.if ${CC:T:Mgcc*} || ${CC:T:Mclang*} -.for src in ${SRC:Msrc/adt/stateset.c} ${SRC:Msrc/adt/tupleset.c} ${SRC:Msrc/adt/edgeset.c} -CFLAGS.${src} += -Wno-unused-function +.for src in ${SRC:Msrc/adt/siphash.c} ${SRC:Msrc/adt/edgeset.c} ${SRC:Msrc/adt/idmap.c} ${SRC:Msrc/adt/ipriq.c} ${SRC:Msrc/adt/internedstateset.c} +CFLAGS.${src} += -std=c99 # XXX: for internal.h +DFLAGS.${src} += -std=c99 # XXX: for internal.h .endfor -.endif # I want to assert on things which are currently true for this platform, # but not true in general. diff --git a/src/fsm/main.c b/src/fsm/main.c index 0f97a9a81..0873b241f 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -646,7 +646,7 @@ main(int argc, char *argv[]) f = xopen(argv[0]); - e = fsm_exec(fsm, fsm_fgetc, f, &state, NULL); + e = fsm_exec(fsm, fsm_fgetc, f, &state); fclose(f); } else { @@ -654,7 +654,7 @@ main(int argc, char *argv[]) s = argv[i]; - e = fsm_exec(fsm, fsm_sgetc, &s, &state, NULL); + e = fsm_exec(fsm, fsm_sgetc, &s, &state); } if (e != 1) { diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 9af51a5a4..bfa8e67db 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -1,6 +1,8 @@ .include "../../share/mk/top.mk" SRC += src/libfsm/capture.c +SRC += src/libfsm/capture_vm.c +SRC += src/libfsm/capture_vm_exec.c SRC += src/libfsm/collate.c SRC += src/libfsm/complete.c SRC += src/libfsm/consolidate.c diff --git a/src/libfsm/capture.c b/src/libfsm/capture.c index 806bb3b12..21f32d06e 100644 --- a/src/libfsm/capture.c +++ b/src/libfsm/capture.c @@ -6,31 +6,82 @@ #include -#include "capture_internal.h" +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "capture.h" +#include "capture_vm_program.h" +#include "capture_log.h" +#include "capture_vm.h" +#include "endids.h" + +#define DEF_PROGRAMS_CEIL 4 + +struct fsm_capture_info { + unsigned max_capture_id; + + /* For particular end states, which captures are active? */ + struct idmap *end_capture_map; + + /* Set of capture resolution programs associated with specific + * end states. */ + struct capvm_program_set { + uint32_t ceil; + uint32_t used; + struct capvm_program **set; + } programs; + + /* For particular end states, which capture programs are + * associtaed with them? */ + struct idmap *end_capvm_program_map; +}; int fsm_capture_init(struct fsm *fsm) { struct fsm_capture_info *ci = NULL; - size_t i; + struct idmap *end_capture_map = NULL; + struct idmap *end_capvm_program_map = NULL; ci = f_calloc(fsm->opt->alloc, 1, sizeof(*ci)); if (ci == NULL) { goto cleanup; } - fsm->capture_info = ci; + end_capture_map = idmap_new(fsm->opt->alloc); + if (end_capture_map == NULL) { + goto cleanup; + } + ci->end_capture_map = end_capture_map; - for (i = 0; i < fsm->statealloc; i++) { - fsm->states[i].has_capture_actions = 0; + end_capvm_program_map = idmap_new(fsm->opt->alloc); + if (end_capvm_program_map == NULL) { + goto cleanup; } + ci->end_capvm_program_map = end_capvm_program_map; + + fsm->capture_info = ci; return 1; cleanup: - if (ci != NULL) { - f_free(fsm->opt->alloc, ci); - } + f_free(fsm->opt->alloc, ci); + idmap_free(end_capture_map); + idmap_free(end_capvm_program_map); return 0; } @@ -41,799 +92,570 @@ fsm_capture_free(struct fsm *fsm) if (ci == NULL) { return; } - f_free(fsm->opt->alloc, ci->buckets); + + idmap_free(ci->end_capture_map); + idmap_free(ci->end_capvm_program_map); + + for (size_t p_i = 0; p_i < ci->programs.used; p_i++) { + fsm_capvm_program_free(fsm->opt->alloc, ci->programs.set[p_i]); + } + f_free(fsm->opt->alloc, ci->programs.set); + f_free(fsm->opt->alloc, ci); fsm->capture_info = NULL; } unsigned -fsm_countcaptures(const struct fsm *fsm) +fsm_capture_ceiling(const struct fsm *fsm) { - (void)fsm; if (fsm->capture_info == NULL) { return 0; } - if (fsm->capture_info->buckets_used == 0) { - return 0; - } - /* check actual */ #if EXPENSIVE_CHECKS - { - struct fsm_capture_info *ci = fsm->capture_info; - size_t i; - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { /* empty */ - continue; - } - assert(ci->max_capture_id >= b->action.id); + /* check actual */ + unsigned res = 0; + for (size_t i = 0; i < fsm->capture_info->programs.used; i++) { + const unsigned id = fsm_capvm_program_get_max_capture_id(fsm->capture_info->programs.set[i]); + if (id > res) { + res = id; } } + assert(res == fsm->capture_info->max_capture_id); #endif return fsm->capture_info->max_capture_id + 1; } +struct fsm_capture * +fsm_capture_alloc_capture_buffer(const struct fsm *fsm) +{ + assert(fsm != NULL); + const size_t len = fsm_capture_ceiling(fsm); + struct fsm_capture *res = f_malloc(fsm->opt->alloc, + len * sizeof(res[0])); + return res; +} + +void +fsm_capture_free_capture_buffer(const struct fsm *fsm, + struct fsm_capture *capture_buffer) +{ + assert(fsm != NULL); + f_free(fsm->opt->alloc, capture_buffer); +} + + int fsm_capture_has_captures(const struct fsm *fsm) { return fsm->capture_info - ? fsm->capture_info->buckets_used > 0 + ? fsm->capture_info->programs.used > 0 : 0; } -int -fsm_capture_has_capture_actions(const struct fsm *fsm, fsm_state_t state) +void +fsm_capture_dump_programs(FILE *f, const struct fsm *fsm) { - assert(state < fsm->statecount); - return fsm->states[state].has_capture_actions; + fprintf(f, "\n==== %s:\n", __func__); + struct fsm_capture_info *ci = fsm->capture_info; + for (uint32_t i = 0; i < ci->programs.used; i++) { + const struct capvm_program *p = ci->programs.set[i]; + fprintf(f, "# program %u, capture_count %u, base %u\n", + i, p->capture_count, p->capture_base); + fsm_capvm_program_dump(f, p); + fprintf(f, "\n"); + } } int -fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, - fsm_state_t start, fsm_state_t end) +fsm_capture_set_active_for_end(struct fsm *fsm, + unsigned capture_id, fsm_state_t end_state) { - struct fsm_capture_info *ci; - struct capture_set_path_env env; - size_t seen_words; - int res = 0; - - assert(fsm != NULL); - assert(start < fsm->statecount); - assert(end < fsm->statecount); - - ci = fsm->capture_info; + struct fsm_capture_info *ci = fsm->capture_info; assert(ci != NULL); + struct idmap *m = ci->end_capture_map; + assert(m != NULL); - /* captures should no longer be stored as paths -- instead, set - * the info on the states _here_, and convert it as necessary. */ - -#if LOG_CAPTURE > 0 - fprintf(stderr, "fsm_capture_set_path: capture %u: <%u, %u>\n", - capture_id, start, end); -#endif - - if (capture_id > FSM_CAPTURE_MAX) { - return 0; /* ID out of range */ - } - - if (!init_capture_action_htab(fsm, ci)) { - return 0; - } - - /* This will create a trail and do a depth-first search from the - * start state, marking every unique path to the end state. */ - env.fsm = fsm; - env.capture_id = capture_id; - env.start = start; - env.end = end; - - env.trail_ceil = 0; - env.trail = NULL; - env.seen = NULL; + #if EXPENSIVE_CHECKS + assert(fsm_isend(fsm, end_state)); + #endif - env.trail = f_malloc(fsm->opt->alloc, - DEF_TRAIL_CEIL * sizeof(env.trail[0])); - if (env.trail == NULL) { - goto cleanup; - } - env.trail_ceil = DEF_TRAIL_CEIL; - - seen_words = fsm->statecount/64 + 1; - env.seen = f_malloc(fsm->opt->alloc, - seen_words * sizeof(env.seen[0])); + return idmap_set(m, end_state, capture_id); +} - if (!mark_capture_path(&env)) { - goto cleanup; - } +void +fsm_capture_iter_active_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_active_for_end_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter_for_state(fsm->capture_info->end_capture_map, state, + idmap_cb, opaque); +} - if (capture_id >= ci->max_capture_id) { - ci->max_capture_id = capture_id; - } +void +fsm_capture_iter_active_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_active_for_end_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter(fsm->capture_info->end_capture_map, + idmap_cb, opaque); +} - res = 1; - /* fall through */ +void +fsm_capture_iter_program_ids_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter_for_state(fsm->capture_info->end_capvm_program_map, state, + idmap_cb, opaque); +} -cleanup: - f_free(fsm->opt->alloc, env.trail); - f_free(fsm->opt->alloc, env.seen); - return res; +void +fsm_capture_iter_program_ids_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter(fsm->capture_info->end_capvm_program_map, + idmap_cb, opaque); } static int -init_capture_action_htab(struct fsm *fsm, struct fsm_capture_info *ci) +dump_active_for_ends_cb(fsm_state_t state_id, unsigned value, void *opaque) { - size_t count, i; - assert(fsm != NULL); - assert(ci != NULL); + FILE *f = opaque; + fprintf(f, " -- state %d: value %u\n", state_id, value); + return 1; +} - if (ci->bucket_count > 0) { - assert(ci->buckets != NULL); - return 1; /* done */ - } +void +fsm_capture_dump_active_for_ends(FILE *f, const struct fsm *fsm) +{ + fprintf(f, "%s:\n", __func__); + idmap_iter(fsm->capture_info->end_capture_map, dump_active_for_ends_cb, f); +} - assert(ci->buckets == NULL); - assert(ci->buckets_used == 0); +void +fsm_capture_dump_program_end_mapping(FILE *f, const struct fsm *fsm) +{ + fprintf(f, "%s:\n", __func__); + idmap_iter(fsm->capture_info->end_capvm_program_map, dump_active_for_ends_cb, f); +} - count = DEF_CAPTURE_ACTION_BUCKET_COUNT; - ci->buckets = f_malloc(fsm->opt->alloc, - count * sizeof(ci->buckets[0])); - if (ci->buckets == NULL) { - return 0; - } +/* Dump capture metadata about an FSM. */ +void +fsm_capture_dump(FILE *f, const char *tag, const struct fsm *fsm) +{ + struct fsm_capture_info *ci; - /* Init buckets to CAPTURE_NO_STATE -> empty. */ - for (i = 0; i < count; i++) { - ci->buckets[i].state = CAPTURE_NO_STATE; + assert(fsm != NULL); + ci = fsm->capture_info; + if (ci == NULL) { + fprintf(f, "==== %s -- no captures\n", tag); + return; } - ci->bucket_count = count; - return 1; + fsm_endid_dump(f, fsm); + fsm_capture_dump_active_for_ends(f, fsm); + fsm_capture_dump_programs(f, fsm); + fsm_capture_dump_program_end_mapping(f, fsm); } +struct carry_active_captures_env { + fsm_state_t dst; + struct idmap *dst_m; + int ok; +}; + static int -mark_capture_path(struct capture_set_path_env *env) +copy_active_captures_cb(fsm_state_t state_id, unsigned value, void *opaque) { - const size_t seen_words = env->fsm->statecount/64 + 1; - -#if LOG_CAPTURE > 0 - fprintf(stderr, "mark_capture_path: path [id %u, %u - %u]\n", - env->capture_id, env->start, env->end); -#endif + (void)state_id; - if (env->start == env->end) { - struct fsm_capture_action action; - action.type = CAPTURE_ACTION_COMMIT_ZERO_STEP; - action.id = env->capture_id; - action.to = CAPTURE_NO_STATE; - if (!add_capture_action(env->fsm, env->fsm->capture_info, - env->start, &action)) { - return 0; - } - return 1; - } - - memset(env->seen, 0x00, - seen_words * sizeof(env->seen[0])); - - /* initialize to starting node */ - env->trail_i = 1; - env->trail[0].state = env->start; - env->trail[0].step = TRAIL_STEP_START; - env->trail[0].has_self_edge = 0; - - while (env->trail_i > 0) { - const enum trail_step step = env->trail[env->trail_i - 1].step; -#if LOG_CAPTURE > 0 - fprintf(stderr, "mark_capture_path: trail %u/%u, cur %u, step %d\n", - env->trail_i, env->trail_ceil, - env->trail[env->trail_i - 1].state, - step); -#endif - - switch (step) { - case TRAIL_STEP_START: - if (!step_trail_start(env)) { - return 0; - } - break; - case TRAIL_STEP_ITER_EDGES: - if (!step_trail_iter_edges(env)) { - return 0; - } - break; - case TRAIL_STEP_ITER_EPSILONS: - if (!step_trail_iter_epsilons(env)) { - return 0; - } - break; - case TRAIL_STEP_DONE: - if (!step_trail_done(env)) { - return 0; - } - break; - default: - assert(!"match fail"); - } + struct carry_active_captures_env *env = opaque; + if (!idmap_set(env->dst_m, env->dst, value)) { + env->ok = false; + return 0; } - return 1; } static int -cmp_action(const struct fsm_capture_action *a, - const struct fsm_capture_action *b) { - /* could use memcmp here, provided padding is always zeroed. */ - return a->id < b->id ? -1 - : a->id > b->id ? 1 - : a->type < b->type ? -1 - : a->type > b->type ? 1 - : a->to < b->to ? -1 - : a->to > b->to ? 1 - : 0; -} - -int -fsm_capture_add_action(struct fsm *fsm, - fsm_state_t state, enum capture_action_type type, - unsigned id, fsm_state_t to) +copy_program_associations_cb(fsm_state_t state_id, unsigned value, void *opaque) { - struct fsm_capture_action action; - assert(fsm->capture_info != NULL); + (void)state_id; - action.type = type; - action.id = id; - action.to = to; - return add_capture_action(fsm, fsm->capture_info, - state, &action); + struct carry_active_captures_env *env = opaque; + if (!idmap_set(env->dst_m, env->dst, value)) { + env->ok = false; + return 0; + } + return 1; } -static int -add_capture_action(struct fsm *fsm, struct fsm_capture_info *ci, - fsm_state_t state, const struct fsm_capture_action *action) +int +fsm_capture_copy_active_for_ends(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state) { - uint64_t h; - size_t b_i, mask; - - assert(state < fsm->statecount); - assert(action->to == CAPTURE_NO_STATE || action->to < fsm->statecount); - -#if LOG_CAPTURE > 0 - fprintf(stderr, "add_capture_action: state %u, type %s, ID %u, TO %d\n", - state, fsm_capture_action_type_name[action->type], - action->id, action->to); -#endif - - if (ci->bucket_count == 0) { - if (!init_capture_action_htab(fsm, ci)) { - return 0; - } - } else if (ci->buckets_used >= ci->bucket_count/2) { /* grow */ - if (!grow_capture_action_buckets(fsm->opt->alloc, ci)) { - return 0; + struct state_iter it; + fsm_state_t s; + + assert(src_fsm != NULL); + assert(src_fsm->capture_info != NULL); + assert(src_fsm->capture_info->end_capture_map != NULL); + assert(dst_fsm != NULL); + assert(dst_fsm->capture_info != NULL); + assert(dst_fsm->capture_info->end_capture_map != NULL); + struct idmap *src_m = src_fsm->capture_info->end_capture_map; + struct idmap *dst_m = dst_fsm->capture_info->end_capture_map; + + struct carry_active_captures_env env = { + .dst_m = dst_m, + .dst = dst_state, + .ok = true, + }; + + state_set_reset(states, &it); + while (state_set_next(&it, &s)) { + if (!fsm_isend(src_fsm, s)) { + continue; } - } - h = hash_id(state); - mask = ci->bucket_count - 1; - - for (b_i = 0; b_i < ci->bucket_count; b_i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[(h + b_i) & mask]; - if (b->state == CAPTURE_NO_STATE) { /* empty */ - b->state = state; - memcpy(&b->action, action, sizeof(*action)); - ci->buckets_used++; - fsm->states[state].has_capture_actions = 1; - if (action->id > ci->max_capture_id) { - ci->max_capture_id = action->id; - } - return 1; - } else if (b->state == state && - 0 == cmp_action(action, &b->action)) { - /* already present, ignore duplicate */ - assert(fsm->states[state].has_capture_actions); - assert(ci->max_capture_id >= action->id); - return 1; - } else { - continue; /* skip past collision */ + idmap_iter_for_state(src_m, s, copy_active_captures_cb, &env); + if (!env.ok) { + goto cleanup; } } - assert(!"unreachable"); - return 0; +cleanup: + return env.ok; } -static int -grow_capture_action_buckets(const struct fsm_alloc *alloc, - struct fsm_capture_info *ci) +int +fsm_capture_copy_program_end_state_associations(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state) { - const size_t ncount = 2 * ci->bucket_count; - struct fsm_capture_action_bucket *nbuckets; - size_t nused = 0; - size_t i; + struct state_iter it; + fsm_state_t s; + + assert(src_fsm != NULL); + assert(src_fsm->capture_info != NULL); + assert(src_fsm->capture_info->end_capvm_program_map != NULL); + assert(dst_fsm != NULL); + assert(dst_fsm->capture_info != NULL); + assert(dst_fsm->capture_info->end_capvm_program_map != NULL); + struct idmap *src_m = src_fsm->capture_info->end_capvm_program_map; + struct idmap *dst_m = dst_fsm->capture_info->end_capvm_program_map; + + struct carry_active_captures_env env = { + .dst_m = dst_m, + .dst = dst_state, + .ok = true, + }; + + state_set_reset(states, &it); + while (state_set_next(&it, &s)) { + if (!fsm_isend(src_fsm, s)) { + continue; + } - assert(ncount != 0); - nbuckets = f_malloc(alloc, ncount * sizeof(nbuckets[0])); - if (nbuckets == NULL) { - return 0; - } + LOG(5 - LOG_CAPTURE_COMBINING_ANALYSIS, + "%s: dst_state %d, state_set_next => %d\n", + __func__, dst_state, s); - for (i = 0; i < ncount; i++) { - nbuckets[i].state = CAPTURE_NO_STATE; + idmap_iter_for_state(src_m, s, copy_program_associations_cb, &env); + if (!env.ok) { + goto cleanup; + } } - for (i = 0; i < ci->bucket_count; i++) { - const struct fsm_capture_action_bucket *src_b = &ci->buckets[i]; - uint64_t h; - const size_t mask = ncount - 1; - size_t b_i; +cleanup: + return env.ok; +} - if (src_b->state == CAPTURE_NO_STATE) { - continue; +int +fsm_capture_copy_programs(const struct fsm *src_fsm, + struct fsm *dst_fsm) +{ + const struct fsm_alloc *alloc = src_fsm->opt->alloc; + assert(alloc == dst_fsm->opt->alloc); + const struct fsm_capture_info *src_ci = src_fsm->capture_info; + + for (uint32_t p_i = 0; p_i < src_ci->programs.used; p_i++) { + const struct capvm_program *p = src_ci->programs.set[p_i]; + struct capvm_program *cp = capvm_program_copy(alloc, p); + if (cp == NULL) { + return 0; } - h = hash_id(src_b->state); - for (b_i = 0; b_i < ncount; b_i++) { - struct fsm_capture_action_bucket *dst_b; - dst_b = &nbuckets[(h + b_i) & mask]; - if (dst_b->state == CAPTURE_NO_STATE) { - memcpy(dst_b, src_b, sizeof(*src_b)); - nused++; - break; - } else { - continue; - } + /* unused: because this is an in-order copy, it's assumed + * the programs will retain their order. */ + uint32_t prog_id; + if (!fsm_capture_add_program(dst_fsm, cp, &prog_id)) { + return 0; } } - - assert(nused == ci->buckets_used); - f_free(alloc, ci->buckets); - ci->buckets = nbuckets; - ci->bucket_count = ncount; return 1; } -static int -grow_trail(struct capture_set_path_env *env) +size_t +fsm_capture_program_count(const struct fsm *fsm) { - struct trail_cell *ntrail; - unsigned nceil; - assert(env != NULL); - - nceil = 2 * env->trail_ceil; - assert(nceil > env->trail_ceil); + return fsm->capture_info->programs.used; +} - ntrail = f_realloc(env->fsm->opt->alloc, env->trail, - nceil * sizeof(env->trail[0])); - if (ntrail == NULL) { - return 0; - } +struct check_program_mappings_env { + const struct fsm *fsm; +}; - env->trail = ntrail; - env->trail_ceil = nceil; +static int +check_program_mappings_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + const uint32_t prog_id = (uint32_t)value; + struct check_program_mappings_env *env = opaque; + assert(state_id < env->fsm->statecount); + assert(prog_id < env->fsm->capture_info->programs.used); return 1; } -static int -step_trail_start(struct capture_set_path_env *env) -{ - struct trail_cell *tc = &env->trail[env->trail_i - 1]; - const fsm_state_t cur = tc->state; - size_t i; - struct edge_set *edge_set = NULL; - - /* check if node is endpoint, if so mark trail, - * then pop trail and continue */ - if (cur == env->end) { - struct fsm_capture_action action; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- GOT END at %u\n", cur); -#endif - action.id = env->capture_id; +void +fsm_capture_integrity_check(const struct fsm *fsm) +{ + if (!EXPENSIVE_CHECKS) { return; } - for (i = 0; i < env->trail_i; i++) { - fsm_state_t state = env->trail[i].state; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- %lu: %d\n", - i, state); -#endif + /* check that all program mappings are in range */ + struct check_program_mappings_env env = { + .fsm = fsm, + }; + idmap_iter(fsm->capture_info->end_capvm_program_map, check_program_mappings_cb, &env); +} - /* Special case: if this is marked as having - * a self-edge on the path, then also add an - * extend for that. */ - if (env->trail[i].has_self_edge) { - struct fsm_capture_action self_action; - self_action.type = CAPTURE_ACTION_EXTEND; - self_action.id = env->capture_id; - self_action.to = state; - - if (!add_capture_action(env->fsm, - env->fsm->capture_info, - state, &self_action)) { - return 0; - } - } - - - if (i == 0) { - action.type = CAPTURE_ACTION_START; - } else { - action.type = (i < env->trail_i - 1 - ? CAPTURE_ACTION_EXTEND - : CAPTURE_ACTION_COMMIT); - } - - if (i < env->trail_i - 1) { - action.to = env->trail[i + 1].state; - } else { - action.to = CAPTURE_NO_STATE; - } - - if (!add_capture_action(env->fsm, - env->fsm->capture_info, - state, &action)) { - return 0; - } - } +struct capture_idmap_compact_env { + int ok; + struct idmap *dst; + const fsm_state_t *mapping; + size_t orig_statecount; +}; - tc->step = TRAIL_STEP_DONE; - return 1; - } +static int +copy_with_mapping_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + fsm_state_t dst_id; + struct capture_idmap_compact_env *env = opaque; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- resetting edge iterator\n"); -#endif - edge_set = env->fsm->states[cur].edges; + assert(state_id < env->orig_statecount); + dst_id = env->mapping[state_id]; - MARK_SEEN(env, cur); -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- marking %u as seen\n", cur); -#endif + if (dst_id == FSM_STATE_REMAP_NO_STATE) { + return 1; /* discard */ + } + + if (!idmap_set(env->dst, dst_id, value)) { + env->ok = 0; + return 0; + } - edge_set_reset(edge_set, &tc->iter); - tc->step = TRAIL_STEP_ITER_EDGES; return 1; } -static int -step_trail_iter_edges(struct capture_set_path_env *env) +int +fsm_capture_id_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount) { - struct trail_cell *tc = &env->trail[env->trail_i - 1]; - struct trail_cell *next_tc = NULL; - - struct fsm_edge e; + struct capture_idmap_compact_env env; + struct idmap *old_idmap = fsm->capture_info->end_capture_map; + struct idmap *new_idmap = idmap_new(fsm->opt->alloc); - if (!edge_set_next(&tc->iter, &e)) { -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- ITER_EDGE_NEXT: DONE %u\n", tc->state); -#endif - tc->step = TRAIL_STEP_ITER_EPSILONS; - return 1; + if (new_idmap == NULL) { + return 0; } -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- ITER_EDGE_NEXT: %u -- NEXT %u\n", - tc->state, e.state); -#endif + env.ok = 1; + env.dst = new_idmap; + env.mapping = mapping; + env.orig_statecount = orig_statecount; - if (tc->state == e.state) { -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- special case, self-edge\n"); -#endif - /* Mark this state as having a self-edge, then continue - * the iterator. An EXTEND action will be added for the - * self-edge later, if necessary. */ - tc->has_self_edge = 1; - return 1; - } else if (CHECK_SEEN(env, e.state)) { -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- seen, skipping\n"); -#endif - return 1; /* continue */ - } - - if (env->trail_i == env->trail_ceil) { - if (!grow_trail(env)) { - return 0; - } + idmap_iter(old_idmap, copy_with_mapping_cb, &env); + if (!env.ok) { + idmap_free(new_idmap); + return 0; } -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- marking %u as seen\n", e.state); -#endif - MARK_SEEN(env, e.state); + idmap_free(old_idmap); + fsm->capture_info->end_capture_map = new_idmap; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- not seen (%u), exploring\n", e.state); -#endif - env->trail_i++; - next_tc = &env->trail[env->trail_i - 1]; - next_tc->state = e.state; - next_tc->step = TRAIL_STEP_START; - next_tc->has_self_edge = 0; return 1; } -static int -step_trail_iter_epsilons(struct capture_set_path_env *env) +int +fsm_capture_program_association_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount) { - struct trail_cell *tc = &env->trail[env->trail_i - 1]; + struct capture_idmap_compact_env env; + struct idmap *old_idmap = fsm->capture_info->end_capvm_program_map; + struct idmap *new_idmap = idmap_new(fsm->opt->alloc); - /* skipping this for now */ - -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- ITER_EPSILONS: %u\n", tc->state); -#endif - - tc->step = TRAIL_STEP_DONE; - return 1; -} + if (new_idmap == NULL) { + return 0; + } -static int -step_trail_done(struct capture_set_path_env *env) -{ - struct trail_cell *tc; + env.ok = 1; + env.dst = new_idmap; + env.mapping = mapping; + env.orig_statecount = orig_statecount; - /* 0-step paths already handled outside loop */ - assert(env->trail_i > 0); + idmap_iter(old_idmap, copy_with_mapping_cb, &env); + if (!env.ok) { + idmap_free(new_idmap); + return 0; + } - tc = &env->trail[env->trail_i - 1]; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- DONE: %u\n", tc->state); -#endif - CLEAR_SEEN(env, tc->state); + idmap_free(old_idmap); + fsm->capture_info->end_capvm_program_map = new_idmap; - env->trail_i--; return 1; } void -fsm_capture_rebase_capture_id(struct fsm *fsm, unsigned base) +fsm_capture_update_max_capture_id(struct fsm_capture_info *ci, + unsigned capture_id) { - size_t i; - struct fsm_capture_info *ci = fsm->capture_info; assert(ci != NULL); - - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { - continue; - } - - b->action.id += base; - if (b->action.id > ci->max_capture_id) { - ci->max_capture_id = b->action.id; - } + if (capture_id >= ci->max_capture_id) { + ci->max_capture_id = capture_id; } } -void -fsm_capture_rebase_capture_action_states(struct fsm *fsm, fsm_state_t base) +int +fsm_capture_add_program(struct fsm *fsm, + struct capvm_program *program, uint32_t *prog_id) { - size_t i; + assert(program != NULL); + assert(prog_id != NULL); + struct fsm_capture_info *ci = fsm->capture_info; - assert(ci != NULL); - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { - continue; + if (ci->programs.used == ci->programs.ceil) { + const size_t nceil = (ci->programs.ceil == 0 + ? DEF_PROGRAMS_CEIL + : 2*ci->programs.ceil); + assert(nceil > ci->programs.ceil); + struct capvm_program **nset = f_realloc(fsm->opt->alloc, + ci->programs.set, nceil * sizeof(nset[0])); + if (nset == NULL) { + return 0; } - b->state += base; - if (b->action.to != CAPTURE_NO_STATE) { - b->action.to += base; - } + ci->programs.ceil = nceil; + ci->programs.set = nset; } -} + assert(ci->programs.used < ci->programs.ceil); -struct fsm_capture * -fsm_capture_alloc(const struct fsm *fsm) -{ - (void)fsm; - assert(!"todo"); - return NULL; -} - -void -fsm_capture_update_captures(const struct fsm *fsm, - fsm_state_t cur_state, fsm_state_t next_state, size_t offset, - struct fsm_capture *captures) -{ - const struct fsm_capture_info *ci; - uint64_t h; - size_t b_i, mask; - - assert(cur_state < fsm->statecount); - assert(fsm->states[cur_state].has_capture_actions); - - ci = fsm->capture_info; - assert(ci != NULL); - - h = hash_id(cur_state); - mask = ci->bucket_count - 1; - -#if LOG_CAPTURE > 0 - fprintf(stderr, "-- updating captures at state %u, to %d, offset %lu\n", - cur_state, next_state, offset); -#endif - - for (b_i = 0; b_i < ci->bucket_count; b_i++) { - const size_t b_id = (h + b_i) & mask; - struct fsm_capture_action_bucket *b = &ci->buckets[b_id]; - unsigned capture_id; - -#if LOG_CAPTURE > 3 - fprintf(stderr, " -- update_captures: bucket %lu, state %d\n", b_id, b->state); -#endif - - - if (b->state == CAPTURE_NO_STATE) { -#if LOG_CAPTURE > 3 - fprintf(stderr, " -- no more actions for this state\n"); -#endif - break; /* no more for this state */ - } else if (b->state != cur_state) { - continue; /* skip collision */ - } - - assert(b->state == cur_state); - capture_id = b->action.id; - - switch (b->action.type) { - case CAPTURE_ACTION_START: -#if LOG_CAPTURE > 0 - fprintf(stderr, "START [%u, %u]\n", - b->action.id, b->action.to); -#endif - if (next_state == b->action.to && captures[capture_id].pos[0] == FSM_CAPTURE_NO_POS) { - captures[capture_id].pos[0] = offset; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[0] to %lu\n", b->action.id, offset); -#endif - } else { - /* filtered, ignore */ - } - break; - case CAPTURE_ACTION_EXTEND: -#if LOG_CAPTURE > 0 - fprintf(stderr, "EXTEND [%u, %u]\n", - b->action.id, b->action.to); -#endif - if (captures[capture_id].pos[0] != FSM_CAPTURE_NO_POS - && (0 == (captures[capture_id].pos[1] & COMMITTED_CAPTURE_FLAG))) { - if (next_state == b->action.to) { - captures[capture_id].pos[1] = offset; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[1] to %lu\n", b->action.id, offset); -#endif - } else { - /* filtered, ignore */ - } - } - break; - case CAPTURE_ACTION_COMMIT_ZERO_STEP: -#if LOG_CAPTURE > 0 - fprintf(stderr, "COMMIT_ZERO_STEP [%u]\n", - b->action.id); -#endif - - if (captures[capture_id].pos[0] == FSM_CAPTURE_NO_POS) { - captures[capture_id].pos[0] = offset; - captures[capture_id].pos[1] = offset | COMMITTED_CAPTURE_FLAG; - } else { /* extend */ - captures[capture_id].pos[1] = offset | COMMITTED_CAPTURE_FLAG; - } - -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[0] and [1] to %lu (with COMMIT flag)\n", b->action.id, offset); -#endif - break; - case CAPTURE_ACTION_COMMIT: -#if LOG_CAPTURE > 0 - fprintf(stderr, "COMMIT [%u]\n", - b->action.id); -#endif - captures[capture_id].pos[1] = offset | COMMITTED_CAPTURE_FLAG; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[1] to %lu (with COMMIT flag)\n", b->action.id, offset); -#endif - break; - default: - assert(!"matchfail"); - } + const unsigned max_prog_capture_id = fsm_capvm_program_get_max_capture_id(program); + if (max_prog_capture_id > ci->max_capture_id) { + fsm_capture_update_max_capture_id(ci, max_prog_capture_id); } -} -void -fsm_capture_finalize_captures(const struct fsm *fsm, - size_t capture_count, struct fsm_capture *captures) -{ - size_t i; - - /* If either pos[] is FSM_CAPTURE_NO_POS or the - * COMMITTED_CAPTURE_FLAG isn't set on pos[1], then the capture - * wasn't finalized; clear it. Otherwise, clear that bit so the - * pos[1] offset is meaningful. */ - - /* FIXME: this should also take the end state(s) associated - * with a capture into account, when that information is available; - * otherwise there will be false positives for zero-width captures - * where the paths have a common prefix. */ - (void)fsm; - - for (i = 0; i < capture_count; i++) { -#if LOG_CAPTURE > 1 - fprintf(stderr, "finalize[%lu]: pos[0]: %ld, pos[1]: %ld\n", - i, captures[i].pos[0], captures[i].pos[1]); -#endif + *prog_id = ci->programs.used; + ci->programs.set[ci->programs.used] = program; + ci->programs.used++; + return 1; +} - if (captures[i].pos[0] == FSM_CAPTURE_NO_POS - || captures[i].pos[1] == FSM_CAPTURE_NO_POS - || (0 == (captures[i].pos[1] & COMMITTED_CAPTURE_FLAG))) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; -#if LOG_CAPTURE > 1 - fprintf(stderr, "finalize: discard %lu\n", i); -#endif - } else if (captures[i].pos[1] & COMMITTED_CAPTURE_FLAG) { - captures[i].pos[1] &=~ COMMITTED_CAPTURE_FLAG; - } +const struct capvm_program * +fsm_capture_get_program_by_id(const struct fsm *fsm, uint32_t prog_id) +{ + struct fsm_capture_info *ci = fsm->capture_info; + if (prog_id >= ci->programs.used) { + return NULL; } + return ci->programs.set[prog_id]; } -void -fsm_capture_action_iter(const struct fsm *fsm, - fsm_capture_action_iter_cb *cb, void *opaque) +int +fsm_capture_associate_program_with_end_state(struct fsm *fsm, + uint32_t prog_id, fsm_state_t end_state) { - size_t i; struct fsm_capture_info *ci = fsm->capture_info; - assert(ci != NULL); - - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { - continue; - } + assert(end_state < fsm->statecount); + assert(prog_id < ci->programs.used); - if (!cb(b->state, b->action.type, - b->action.id, b->action.to, opaque)) { - break; - } + if (!idmap_set(ci->end_capvm_program_map, end_state, prog_id)) { + return 0; } + return 1; } -const char *fsm_capture_action_type_name[] = { - "START", "EXTEND", - "COMMIT_ZERO_STEP", "COMMIT" +struct capture_resolve_env { + const struct fsm_capture_info *ci; + const unsigned char *input; + const size_t length; + + int res; + struct fsm_capture *captures; + size_t captures_len; }; static int -dump_iter_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +exec_capvm_program_cb(fsm_state_t state_id, unsigned prog_id, void *opaque) { - FILE *f = opaque; - fprintf(f, " - state %u, %s [capture_id: %u, to: %d]\n", - state, fsm_capture_action_type_name[type], capture_id, to); + struct capture_resolve_env *env = opaque; + (void)state_id; + + /* TODO: idmap_iter could take a halt return value */ + if (env->res != 1) { return 0; } + + assert(prog_id < env->ci->programs.used); + struct capvm_program *p = env->ci->programs.set[prog_id]; + + LOG(5 - LOG_EVAL, "%s: evaluating prog_id %u for state %d\n", + __func__, prog_id, state_id); + +#define EXEC_COUNT 1 /* can be increased for benchmarking */ + + for (size_t i = 0; i < EXEC_COUNT; i++) { + const enum fsm_capvm_program_exec_res exec_res = + fsm_capvm_program_exec(p, + (const uint8_t *)env->input, env->length, + env->captures, env->captures_len); + if (exec_res != FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN) { + env->res = 0; + return 0; + } + } return 1; } -/* Dump capture metadata about an FSM. */ -void -fsm_capture_dump(FILE *f, const char *tag, const struct fsm *fsm) +int +fsm_capture_resolve_during_exec(const struct fsm *fsm, + fsm_state_t end_state, const unsigned char *input, size_t input_offset, + struct fsm_capture *captures, size_t captures_len) { - struct fsm_capture_info *ci; - assert(fsm != NULL); - ci = fsm->capture_info; - if (ci == NULL || ci->bucket_count == 0) { - fprintf(f, "==== %s -- no captures\n", tag); - return; - } - - fprintf(f, "==== %s -- capture action hash table (%u buckets)\n", - tag, ci->bucket_count); - fsm_capture_action_iter(fsm, dump_iter_cb, f); + assert(input != NULL); + assert(captures != NULL); + + const struct fsm_capture_info *ci = fsm->capture_info; + + struct capture_resolve_env capture_env = { + .res = 1, + .ci = ci, + .input = input, + .length = input_offset, + .captures = captures, + .captures_len = captures_len, + }; + + LOG(5 - LOG_EVAL, "%s: ended on state %d\n", + __func__, end_state); + idmap_iter_for_state(ci->end_capvm_program_map, + end_state, exec_capvm_program_cb, &capture_env); + + return capture_env.res; } diff --git a/src/libfsm/capture.h b/src/libfsm/capture.h index 4c0ba4722..16588060c 100644 --- a/src/libfsm/capture.h +++ b/src/libfsm/capture.h @@ -2,28 +2,26 @@ #define LIBFSM_CAPTURE_H #include +#include #include #include +#include -#define NEXT_STATE_END ((fsm_state_t)-1) - +/* Internal state IDs that are out of range for valid state IDs. + * + * CAPTURE_NO_STATE is used to represent the absence of a state, such as + * when remapping a state to a dead state (removing it) or empty + * hash table buckets. + * + * NEXT_STATE_END is used as a destination for capture actions that + * trigger when ending on a state. */ #define CAPTURE_NO_STATE ((fsm_state_t)-1) /* Capture interface -- functions internal to libfsm. * The public interface should not depend on any of these details. */ -enum capture_action_type { - /* Start an active capture if transitioning to TO. */ - CAPTURE_ACTION_START, - /* Continue an active capture if transitioning to TO, - * otherwise deactivate it. */ - CAPTURE_ACTION_EXTEND, - /* Write a zero-step capture (i.e., the start and - * end state are the same). */ - CAPTURE_ACTION_COMMIT_ZERO_STEP, - /* Write an active capture's endpoints. */ - CAPTURE_ACTION_COMMIT -}; +struct fsm_capture_info; +struct capvm_program; int fsm_capture_init(struct fsm *fsm); @@ -31,41 +29,110 @@ fsm_capture_init(struct fsm *fsm); void fsm_capture_free(struct fsm *fsm); +void +fsm_capture_dump_active_for_ends(FILE *f, const struct fsm *fsm); + +void +fsm_capture_dump_program_end_mapping(FILE *f, const struct fsm *fsm); + /* Does the FSM have captures? */ int fsm_capture_has_captures(const struct fsm *fsm); -/* Update captures, called when exiting or ending on a state. - * If ending on a state, use NEXT_STATE_END for next_state. */ void -fsm_capture_update_captures(const struct fsm *fsm, - fsm_state_t cur_state, fsm_state_t next_state, size_t offset, - struct fsm_capture *captures); +fsm_capture_dump(FILE *f, const char *tag, const struct fsm *fsm); + +void +fsm_capture_dump_programs(FILE *f, const struct fsm *fsm); +/* If EXPENSIVE_CHECKS is non-zero, assert that all capture metadata on + * an FSM is internally consistent. */ void -fsm_capture_finalize_captures(const struct fsm *fsm, - size_t capture_count, struct fsm_capture *captures); +fsm_capture_integrity_check(const struct fsm *fsm); + +int +fsm_capture_id_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount); -/* Add a capture action. This is used to update capture actions - * in the destination FSM when combining/transforming other FSMs. */ int -fsm_capture_add_action(struct fsm *fsm, - fsm_state_t state, enum capture_action_type type, - unsigned id, fsm_state_t to); - -/* Callback for iterating over capture actions. - * Return 1 to continue, return 0 to halt. - * If TO is not meaningful for a particular type, it will be - * set to NEXT_STATE_END. */ +fsm_capture_program_association_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount); + +/* Iterator callback for capture IDs that are active for a particular + * end state. Returns whether iteration should continue. */ typedef int -fsm_capture_action_iter_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, +fsm_capture_iter_active_for_end_cb(fsm_state_t state, unsigned capture_id, void *opaque); void -fsm_capture_action_iter(const struct fsm *fsm, - fsm_capture_action_iter_cb *cb, void *opaque); +fsm_capture_iter_active_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_active_for_end_cb *cb, void *opaque); + +void +fsm_capture_iter_active_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_active_for_end_cb *cb, void *opaque); -extern const char *fsm_capture_action_type_name[]; +/* Iterator callback for program IDs that are active for a particular + * end state. Returns whether iteration should continue. */ +typedef int +fsm_capture_iter_program_ids_for_end_state_cb(fsm_state_t state, unsigned prog_id, + void *opaque); + +void +fsm_capture_iter_program_ids_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque); +void +fsm_capture_iter_program_ids_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque); + +/* TODO: combine/rename */ +int +fsm_capture_copy_active_for_ends(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state); +int +fsm_capture_copy_program_end_state_associations(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state); + +int +fsm_capture_copy_programs(const struct fsm *src_fsm, + struct fsm *dst_fsm); + +size_t +fsm_capture_program_count(const struct fsm *fsm); + +void +fsm_capture_update_max_capture_id(struct fsm_capture_info *ci, + unsigned capture_id); + +int +fsm_capture_add_program(struct fsm *fsm, + struct capvm_program *program, uint32_t *prog_id); + +const struct capvm_program * +fsm_capture_get_program_by_id(const struct fsm *fsm, uint32_t prog_id); + +int +fsm_capture_associate_program_with_end_state(struct fsm *fsm, + uint32_t prog_id, fsm_state_t end_state); + +/* Resolve captures. + * + * FIXME: With the current implementation, if enough memory + * was passed in then it couldn't fail, but it may be worth + * changing the interface so that it doesn't assume there was + * already a successful match in order to support one-pass + * matching & capture resolution attempts from a stream. + * + * TODO: This should pass in a size for captures[]. + * TODO: An alternate interface that allows passing in + * preallocated buffers for working memory. + * + * TODO: describe return value. */ +int +fsm_capture_resolve_during_exec(const struct fsm *fsm, + fsm_state_t end_state, const unsigned char *input, size_t input_offset, + struct fsm_capture *captures, size_t captures_len); #endif diff --git a/src/libfsm/capture_internal.h b/src/libfsm/capture_internal.h deleted file mode 100644 index 70418b988..000000000 --- a/src/libfsm/capture_internal.h +++ /dev/null @@ -1,115 +0,0 @@ -#ifndef CAPTURE_INTERNAL_H -#define CAPTURE_INTERNAL_H - -#include -#include - -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include "internal.h" -#include "capture.h" - -/* Bucket count for capture action hash table. - * Must be a power of 2. */ - -#define DEF_CAPTURE_ACTION_BUCKET_COUNT 32 -#define DEF_TRAIL_CEIL 8 - -#define LOG_CAPTURE 0 - -/* Most significant bit of a size_t. */ -#define COMMITTED_CAPTURE_FLAG ((SIZE_MAX) ^ (SIZE_MAX >> 1)) - -struct fsm_capture_info { - unsigned max_capture_id; - - /* Add-only hash table. */ - unsigned bucket_count; - unsigned buckets_used; /* grow if >= 1/2 used */ - - /* Hash buckets. If state is CAPTURE_NO_STATE, - * the bucket is empty. */ - struct fsm_capture_action_bucket { - fsm_state_t state; /* key */ - struct fsm_capture_action { - enum capture_action_type type; - unsigned id; - /* only used by START and EXTEND */ - fsm_state_t to; - } action; - } *buckets; -}; - -enum trail_step { - TRAIL_STEP_START, - TRAIL_STEP_ITER_EDGES, - TRAIL_STEP_ITER_EPSILONS, - TRAIL_STEP_DONE -}; - -/* env->seen is used as a bit set for tracking which states have already - * been processed. These macros set/check/clear the bits. */ -#define SEEN_BITOP(ENV, STATE, OP) ENV->seen[STATE/64] OP ((uint64_t)1 << (STATE&63)) -#define MARK_SEEN(ENV, STATE) SEEN_BITOP(ENV, STATE, |=) -#define CHECK_SEEN(ENV, STATE) SEEN_BITOP(ENV, STATE, &) -#define CLEAR_SEEN(ENV, STATE) SEEN_BITOP(ENV, STATE, &=~) - -struct capture_set_path_env { - struct fsm *fsm; - unsigned capture_id; - fsm_state_t start; - fsm_state_t end; - - unsigned trail_i; - unsigned trail_ceil; - struct trail_cell { - fsm_state_t state; - enum trail_step step; - char has_self_edge; - struct edge_iter iter; - } *trail; - - /* bitset for which states have already been seen. */ - uint64_t *seen; -}; - -static int -init_capture_action_htab(struct fsm *fsm, struct fsm_capture_info *ci); - -static int -mark_capture_path(struct capture_set_path_env *env); - -static int -add_capture_action(struct fsm *fsm, struct fsm_capture_info *ci, - fsm_state_t state, const struct fsm_capture_action *action); - -static int -grow_capture_action_buckets(const struct fsm_alloc *alloc, - struct fsm_capture_info *ci); - -static int -grow_trail(struct capture_set_path_env *env); - -static int -step_trail_start(struct capture_set_path_env *env); -static int -step_trail_iter_edges(struct capture_set_path_env *env); -static int -step_trail_iter_epsilons(struct capture_set_path_env *env); -static int -step_trail_done(struct capture_set_path_env *env); - -static int -cmp_action(const struct fsm_capture_action *a, - const struct fsm_capture_action *b); - -#endif diff --git a/src/libfsm/capture_log.h b/src/libfsm/capture_log.h new file mode 100644 index 000000000..c850460bb --- /dev/null +++ b/src/libfsm/capture_log.h @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_LOG_H +#define CAPTURE_LOG_H + +/* Log levels */ +#define LOG_CAPTURE 0 +#define LOG_CAPTURE_COMBINING_ANALYSIS 0 +#define LOG_EVAL 0 +#define LOG_APPEND_ACTION 0 +#define LOG_PRINT_FSM 0 +#define LOG_MARK_PATH 0 + +#include + +#define LOG(LEVEL, ...) \ + do { \ + if ((LEVEL) <= LOG_CAPTURE) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while(0) + +#endif diff --git a/src/libfsm/capture_vm.c b/src/libfsm/capture_vm.c new file mode 100644 index 000000000..aacae7f9c --- /dev/null +++ b/src/libfsm/capture_vm.c @@ -0,0 +1,193 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +/* Virtual machine for resolving captures while executing regular + * expressions from a subset of PCRE. This is based on the approach + * described in Russ Cox's "Regular Expression Matching: the Virtual + * Machine Approach" (https://swtch.com/~rsc/regexp/regexp2.html), but + * has a couple major modifications, mainly to keep memory usage low and + * predictable, and to be more consistent (arguably, bug-compatible...) + * with PCRE's behavior for libfsm's supported subset of PCRE. + * + * Instead of giving each green thread its own copy of the capture + * buffers, which uses a prohibitive amount of memory when matching DFAs + * that combine several regexes with several captures each, operate in + * two passes. + * + * In the first pass, each thread keeps track of its execution path, + * appending a bit for each branch: 1 for the greedy option, 0 for the + * non-greedy. Since there can be at most one live thread per program + * instruction, and all of them are either on the current or next input + * character, there's a bounded window for diverging paths during execution. + * After a certain distance back all paths either have a common prefix + * or consist entirely of 0 bits (for continually looping at an unanchored + * start). The path bits are stored in chunks in a backwards linked list, + * so nodes for common path prefixes can be shared by multiple threads, + * and the prefix of all 0 bits is instead stored as a counter. This + * keeps memory usage substantially lower. This search runs threads in + * parallel, breadth-first, halting any threads that duplicate work of + * a greedier search path (since PCRE's results match the greediest). + * + * In the second pass, replay the execution path for just the single + * greediest thread, which represents the "best" match, and write + * capture offsets into buffers passed in by the caller. + * + * Most of the other differences have to do with matching PCRE's quirky + * behaviors, particularly interactions between newlines and start/end + * anchors. + * */ + +#include "capture_vm.h" +#include "capture_vm_program.h" + +#include + +#include +#include +#include + +void +fsm_capvm_program_free(const struct fsm_alloc *alloc, + struct capvm_program *program) +{ + if (program == NULL) { return; } + f_free(alloc, program->ops); + f_free(alloc, program->char_classes.sets); + f_free(alloc, program); +} + +struct capvm_program * +capvm_program_copy(const struct fsm_alloc *alloc, + const struct capvm_program *src) +{ + assert(src != NULL); + struct capvm_program *p = NULL; + struct capvm_opcode *ops = NULL; + struct capvm_char_class *sets = NULL; + + p = f_calloc(alloc, 1, sizeof(*p)); + if (p == NULL) { goto cleanup; } + + /* This allocates exactly as many instructions and char_classes + * as necessary, rather than a power-of-2 buffer, because + * they are only added during compilation in libre. */ + + ops = f_calloc(alloc, src->used, sizeof(ops[0])); + if (ops == NULL) { goto cleanup; } + + sets = f_calloc(alloc, src->char_classes.count, + sizeof(src->char_classes.sets[0])); + if (sets == NULL) { goto cleanup; } + + memcpy(ops, src->ops, src->used * sizeof(src->ops[0])); + + assert(src->char_classes.sets != NULL || src->char_classes.count == 0); + if (src->char_classes.count > 0) { + memcpy(sets, src->char_classes.sets, + src->char_classes.count * sizeof(src->char_classes.sets[0])); + } + + struct capvm_program np = { + .capture_count = src->capture_count, + .capture_base = src->capture_base, + + .used = src->used, + .ceil = src->used, + .ops = ops, + + .char_classes = { + .count = src->char_classes.count, + .ceil = src->char_classes.count, + .sets = sets, + }, + }; + memcpy(p, &np, sizeof(np)); + return p; + +cleanup: + f_free(alloc, p); + f_free(alloc, ops); + f_free(alloc, sets); + return NULL; +} + +void +capvm_program_rebase(struct capvm_program *program, unsigned capture_offset) +{ + assert(program->capture_base + capture_offset > program->capture_base); + program->capture_base += capture_offset; +} + +void +fsm_capvm_program_dump(FILE *f, + const struct capvm_program *p) +{ + for (size_t i = 0; i < p->used; i++) { + const struct capvm_opcode *op = &p->ops[i]; + switch (op->t) { + case CAPVM_OP_CHAR: + fprintf(f, "%zu: char 0x%02x (%c)\n", + i, op->u.chr, isprint(op->u.chr) ? op->u.chr : '.'); + break; + case CAPVM_OP_CHARCLASS: + { + const uint32_t id = op->u.charclass_id; + assert(id < p->char_classes.count); + const struct capvm_char_class *cc = &p->char_classes.sets[id]; + fprintf(f, "%zu: charclass %u -> [", i, id); + for (size_t i = 0; i < 4; i++) { + fprintf(f, "%016lx", cc->octets[i]); + } + fprintf(f, "]\n"); + break; + } + case CAPVM_OP_MATCH: + fprintf(f, "%zu: match\n", i); + break; + case CAPVM_OP_JMP: + fprintf(f, "%zu: jmp %u\n", i, op->u.jmp); + break; + case CAPVM_OP_JMP_ONCE: + fprintf(f, "%zu: jmp_once %u\n", i, op->u.jmp_once); + break; + case CAPVM_OP_SPLIT: + fprintf(f, "%zu: split cont %u new %u\n", i, op->u.split.cont, op->u.split.new); + break; + case CAPVM_OP_SAVE: + fprintf(f, "%zu: save %u (cap %u, %s)\n", + i, op->u.save, + op->u.save / 2, (op->u.save & (uint32_t)0x01) ? "end" : "start"); + break; + case CAPVM_OP_ANCHOR: + fprintf(f, "%zu: anchor %s\n", i, + op->u.anchor == CAPVM_ANCHOR_START ? "start" : "end"); + break; + default: + assert(!"matchfail"); + } + } + for (size_t i = 0; i < p->char_classes.count; i++) { + const uint64_t *octets = p->char_classes.sets[i].octets; + fprintf(f, "char_class %zu: 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n", + i, octets[0], octets[1], octets[2], octets[3]); + } +} + +unsigned +fsm_capvm_program_get_capture_count(const struct capvm_program *program) +{ + assert(program != NULL); + return program->capture_count; +} + +unsigned +fsm_capvm_program_get_max_capture_id(const struct capvm_program *program) +{ + assert(program != NULL); + return (program->capture_count == 0 + ? 0 + : program->capture_base + program->capture_count - 1); +} diff --git a/src/libfsm/capture_vm.h b/src/libfsm/capture_vm.h new file mode 100644 index 000000000..74e958eb8 --- /dev/null +++ b/src/libfsm/capture_vm.h @@ -0,0 +1,69 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_VM_H +#define CAPTURE_VM_H + +#include +#include + +#include +#include + +/* Interface the virtual machine used to resolve captures. + * These interfaces are exposed to libre but should not be + * used directly. */ + +/* Opaque struct, details in capture_vm_program.h. */ +struct capvm_program; + +void +fsm_capvm_program_free(const struct fsm_alloc *alloc, + struct capvm_program *program); + +struct capvm_program * +capvm_program_copy(const struct fsm_alloc *alloc, + const struct capvm_program *program); + +/* Add an offset to the capture ID base for a program. + * Used when FSMs are merged, one of the source FSMs' capture IDs + * will be shifted to appear after the others. */ +void +capvm_program_rebase(struct capvm_program *program, unsigned capture_offset); + +void +fsm_capvm_program_dump(FILE *f, + const struct capvm_program *program); + +enum fsm_capvm_program_exec_res { + FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN, + FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND, + + FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC = -1, + /* FIXME: what other ways can this fail? */ +}; + +/* Execute a capture program with the given input and populate + * the capture buffer. + * + * This asserts that the capture buffer is at least as large as + * necessary. This is an internal interface, and the buffer size + * should have already been checked by the caller. */ +enum fsm_capvm_program_exec_res +fsm_capvm_program_exec(const struct capvm_program *program, + const uint8_t *input, size_t length, + struct fsm_capture *capture_buf, size_t capture_buf_length); + +/* Get the capture count from the program. */ +unsigned +fsm_capvm_program_get_capture_count(const struct capvm_program *program); + +/* Get the max capture ID from the program. + * If there are no captures (which is pointless) it will return 0. */ +unsigned +fsm_capvm_program_get_max_capture_id(const struct capvm_program *program); + +#endif diff --git a/src/libfsm/capture_vm_exec.c b/src/libfsm/capture_vm_exec.c new file mode 100644 index 000000000..5e33f159c --- /dev/null +++ b/src/libfsm/capture_vm_exec.c @@ -0,0 +1,2063 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "capture_vm.h" +#include "capture_vm_program.h" +#include "capture_vm_log.h" + +#include +#include +#include +#include + +#include + +/* for EXPENSIVE_CHECKS and TRACK_TIMES */ +#include "internal.h" + +#if EXPENSIVE_CHECKS +#include +#endif + +/* Special handling for a path node that has a long prefix of all 0 + * bits, as is common when the regex is unanchored at the start. */ +#define USE_COLLAPSED_ZERO_PREFIX 1 + +/* Special out-of-range NONE values. */ +#define NO_POS ((uint32_t)-1) +#define NO_ID ((uint32_t)-1) +#define COLLAPSED_ZERO_PREFIX_ID ((uint32_t)-2) +#define NO_POS_SIZE_T ((size_t)-1) + +/* Max number of bits each path link can store. + * This value cannot be changed without reworking the data structures. */ +#define PATH_LINK_BITS 32 + +/* This enables extra debugging/testing output */ +#ifndef TESTING_OPTIONS +#define TESTING_OPTIONS 0 +#endif + +/* Write the solution to stdout (used for testing). */ +#define LOG_SOLUTION_TO_STDOUT (0 || TESTING_OPTIONS) + +/* Enable extra fields for debugging/performance tuning, most notably + * a 'uniq_id' field that helps to see the various execution paths. */ +#define CAPVM_STATS (0 || TESTING_OPTIONS) +#define CAPVM_PATH_STATS (0 && CAPVM_STATS) + +/* This may no longer be necessary after further work on path handling. */ +#define ALLOW_TABLE_RESIZING 1 +#define ALLOW_PATH_TABLE_RESIZING (1 || ALLOW_TABLE_RESIZING) +#define ALLOW_THREAD_TABLE_RESIZING (0 || ALLOW_TABLE_RESIZING) + +/* Set to non-zero to trap runaway path table growth */ +#define PATH_TABLE_CEIL_LIMIT 0 + +/* Specialized logging that can be scraped to reconstruct non-interleaved + * execution paths per thread. */ +#define LOG_EXECUTION 0 +#define LOG_EXECUTION_FILE stderr +#if LOG_EXECUTION + +#if CAPVM_STATS == 0 +#error CAPVM_STATS must be 1 for uniq_id +#endif + +/* Various execution log messages, in an easily scraped format */ +#define LOG_EXEC_OP(UNIQ_ID, INPUT_POS, OP_ID, OP_NAME) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC OP %u %u %u %s\n", \ + UNIQ_ID, INPUT_POS, OP_ID, OP_NAME) + +#define LOG_EXEC_CHAR(UNIQ_ID, CHAR) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC CHAR %u %c 0x%02x\n", UNIQ_ID, isprint(CHAR) ? CHAR : '.', CHAR) + +#define LOG_EXEC_HALT(UNIQ_ID) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC HALT %u\n", UNIQ_ID) + +#define LOG_EXEC_PATH_FIND_SOLUTION(UNIQ_ID, BIT) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC PATH_FIND_SOLUTION %u %u\n", UNIQ_ID, BIT) + +#define LOG_EXEC_PATH_SAVE_CAPTURES(UNIQ_ID, BIT) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC PATH_SAVE_CAPTURES %u %u\n", UNIQ_ID, BIT) + +#define LOG_EXEC_SPLIT(PARENT_UNIQ_ID, CHILD_UNIQ_ID) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC SPLIT %u %u\n", PARENT_UNIQ_ID, CHILD_UNIQ_ID) +#else +#define LOG_EXEC_OP(UNIQ_ID, INPUT_POS, OP_ID, OP_NAME) /* no-op */ +#define LOG_EXEC_CHAR(UNIQ_ID, CHAR) /* no-op */ +#define LOG_EXEC_HALT(UNIQ_ID) /* no-op */ +#define LOG_EXEC_PATH_FIND_SOLUTION(UNIQ_ID, BIT) /* no-op */ +#define LOG_EXEC_PATH_SAVE_CAPTURES(UNIQ_ID, BIT) /* no-op */ +#define LOG_EXEC_SPLIT(PARENT_UNIQ_ID, CHILD_UNIQ_ID) /* no-op */ +#endif + +/* Bitset backed by an array of 32-bit words */ +#define GET_BIT32(BITARRAY, BIT) (BITARRAY[BIT/32] & ((uint32_t)1 << (BIT & 31))) +#define SET_BIT32(BITARRAY, BIT) (BITARRAY[BIT/32] |= ((uint32_t)1 << (BIT & 31))) + +static const char * +op_name[] = { + [CAPVM_OP_CHAR] = "CHAR", + [CAPVM_OP_CHARCLASS] = "CHARCLASS", + [CAPVM_OP_MATCH] = "MATCH", + [CAPVM_OP_JMP] = "JMP", + [CAPVM_OP_JMP_ONCE] = "JMP_ONCE", + [CAPVM_OP_SPLIT] = "SPLIT", + [CAPVM_OP_SAVE] = "SAVE", + [CAPVM_OP_ANCHOR] = "ANCHOR", +}; + +enum pair_id { PAIR_ID_CURRENT = 0, PAIR_ID_NEXT = 1 }; + +struct capvm { + const struct capvm_program *p; + const uint8_t *input; + const uint32_t input_len; + struct fsm_capture *capture_buf; + const size_t capture_buf_length; + size_t step_limit; + +#if CAPVM_STATS + uint32_t uniq_id_counter; +#endif + + /* Two stacks, used to track which execution instruction should + * be advanced next. The current stack is + * run_stacks[PAIR_ID_CURRENT], run_stacks[PAIR_ID_NEXT] is the + * stack for the next input position, and when the current stack + * is completed the next stack is copied over (and reversed). + * Same with run_stacks_h, the height for each stack, and the + * other fields with [2] below. */ + uint32_t *run_stacks[2]; + uint32_t run_stacks_h[2]; + + /* Similarly, two columns of bits and two arrays of path_info + * node IDs and uniq_ids for the execution at a particular + * opcode. */ + uint32_t *evaluated[2]; + uint32_t *path_info_heads[2]; +#if CAPVM_STATS + uint32_t *uniq_ids[2]; +#endif + + struct capvm_thread_stats { + uint32_t live; + uint32_t max_live; + } threads; + + /* Pool of nodes for linked lists of path segments. */ + struct capvm_path_info_pool { + uint32_t ceil; + uint32_t live; + uint32_t max_live; + uint32_t freelist_head; + struct capvm_path_info { + union { + struct capvm_path_freelist_link { + uint16_t refcount; /* == 0 */ + uint32_t freelist; + } freelist_node; + struct capvm_path_info_link { + uint16_t refcount; /* > 0, sticky at UINT16_MAX? */ + uint8_t used; /* .bits used, <= PATH_LINK_BITS */ + uint32_t bits; + uint32_t offset; + /* Linked list to earlier path nodes, with common + * nodes shared until paths diverge. + * + * This can be either a valid path node ID, NO_ID + * for end of list, or COLLAPSED_ZERO_PREFIX_ID + * to indicate that the node is preceded by + * (offset) zero bits. */ + uint32_t backlink; +#if CAPVM_PATH_STATS + uint32_t bits_added_per_input_character; +#endif + } path; + } u; + } *pool; + } paths; + + struct capvm_solution_info { + uint32_t best_path_id; +#if CAPVM_STATS + uint32_t best_path_uniq_id; +#endif + uint32_t zeros_evaluated_up_to; + } solution; + + struct { + size_t steps; +#if CAPVM_STATS + uint32_t matches; + uint32_t path_prefixes_shared; + uint32_t collapsed_zero_prefixes; +#endif +#if CAPVM_PATH_STATS + uint32_t max_bits_added_per_input_character; + uint32_t max_path_length_memory; +#endif + } stats; + + enum fsm_capvm_program_exec_res res; +}; + +/* Type identifier macros */ +#define IS_THREAD_FREELIST(T) (T->u.thread.path_info_head == NO_ID) +#define IS_PATH_FREELIST(P) (P->u.path.refcount == 0) +#define IS_PATH_NODE(P) (P->u.path.refcount > 0 && P->u.path.used <= PATH_LINK_BITS) + +static void +release_path_info_link(struct capvm *vm, uint32_t *pi_id); + +static void +dump_path_table(FILE *f, const struct capvm *vm); + +static void +set_max_threads_live(struct capvm *vm, uint32_t new_max_live) +{ + vm->threads.max_live = new_max_live; + if (LOG_CAPVM >= 6) { + LOG(0, "==== new vm->threads.max_live: %u\n", vm->threads.max_live); + dump_path_table(stderr, vm); + } +} + + +/*********************** + * path_info functions * + ***********************/ + +static void +set_max_paths_live(struct capvm *vm) +{ + vm->paths.max_live = vm->paths.live; + if (LOG_CAPVM >= 6) { + LOG(0, "==== new vm->paths.max_live: %u\n", vm->paths.max_live); + dump_path_table(stderr, vm); + } +} + +static uint32_t +get_path_node_refcount(const struct capvm *vm, uint32_t p_id) +{ + assert(p_id < vm->paths.ceil); + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { + return pi->u.freelist_node.refcount; + } else { + assert(IS_PATH_NODE(pi)); + return pi->u.path.refcount; + } +} + +static void +inc_path_node_refcount(struct capvm *vm, uint32_t p_id) +{ + /* TODO: sticky refcount handling is not currently implemented */ + if (p_id == COLLAPSED_ZERO_PREFIX_ID) { return; } + assert(p_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(IS_PATH_NODE(pi)); + LOG(5, "%s: p_id %u: refcnt %u -> %u\n", + __func__, p_id, pi->u.path.refcount, pi->u.path.refcount + 1); + pi->u.path.refcount++; +} + +static uint32_t +get_path_node_offset(const struct capvm *vm, uint32_t p_id) +{ + assert(p_id < vm->paths.ceil); + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(IS_PATH_NODE(pi)); + return pi->u.path.offset; +} + +static uint32_t +get_path_node_backlink(const struct capvm *vm, uint32_t p_id) +{ + assert(p_id < vm->paths.ceil); + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { + return pi->u.freelist_node.freelist; + } else { + assert(IS_PATH_NODE(pi)); + return pi->u.path.backlink; + } +} + +static void +set_path_node_backlink(struct capvm *vm, uint32_t p_id, uint32_t backlink) +{ + assert(p_id < vm->paths.ceil); + assert(backlink < vm->paths.ceil || (backlink == NO_ID || backlink == COLLAPSED_ZERO_PREFIX_ID)); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(IS_PATH_NODE(pi)); + pi->u.path.backlink = backlink; +} + +static void +dump_path_table(FILE *f, const struct capvm *vm) +{ + fprintf(f, "=== path table, %u/%u live\n", + vm->paths.live, vm->paths.ceil); + for (uint32_t i = 0; i < vm->paths.ceil; i++) { + struct capvm_path_info *pi = &vm->paths.pool[i]; + if (IS_PATH_FREELIST(pi)) { + if (LOG_CAPVM >= 5) { + fprintf(f, "paths[%u]: freelist -> %d\n", + i, (int)pi->u.freelist_node.freelist); + } + } else { + assert(IS_PATH_NODE(pi)); + fprintf(f, "paths[%u]: refcount %u, used %u, bits 0x%08x, offset %u, backlink %d%s\n", + i, pi->u.path.refcount, pi->u.path.used, pi->u.path.bits, + pi->u.path.offset, (int)pi->u.path.backlink, + pi->u.path.backlink == COLLAPSED_ZERO_PREFIX_ID + ? " (collapsed zero prefix)" + : pi->u.path.backlink == NO_ID + ? " (none)" + : " (link)"); + } + } +} + +static void +check_path_table(const struct capvm *vm) +{ +#if EXPENSIVE_CHECKS + uint32_t *refcounts = calloc(vm->paths.ceil, sizeof(refcounts[0])); + assert(refcounts); + + if (LOG_CAPVM >= 4) { + dump_path_table(stderr, vm); + } + + LOG(4, "%s: stack heights %u, %u\n", __func__, + vm->run_stacks_h[PAIR_ID_CURRENT], vm->run_stacks_h[PAIR_ID_NEXT]); + + for (uint32_t pair_id = 0; pair_id < 2; pair_id++) { + for (uint32_t h = 0; h < vm->run_stacks_h[pair_id]; h++) { + const uint32_t op_id = vm->run_stacks[pair_id][h]; + if (op_id == NO_ID) { continue; } +#if CAPVM_STATS + const uint32_t uniq_id = vm->uniq_ids[pair_id][op_id]; +#else + const uint32_t uniq_id = 0; +#endif + + LOG(4, "%s: run_stacks[%u][%u/%u]: op_id %u (uniq_id %u) -> path_info_head %u\n", + __func__, pair_id, h, vm->run_stacks_h[pair_id], op_id, + uniq_id, vm->path_info_heads[pair_id][op_id]); + if (op_id == NO_ID) { continue; } + const uint32_t p_id = vm->path_info_heads[pair_id][op_id]; + if (p_id != NO_ID) { + refcounts[p_id]++; + } + } + } + + for (uint32_t p_id = 0; p_id < vm->paths.ceil; p_id++) { + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { + continue; + } + const uint32_t backlink = get_path_node_backlink(vm, p_id); + if (backlink != NO_ID && backlink != COLLAPSED_ZERO_PREFIX_ID) { + assert(backlink < vm->paths.ceil); + refcounts[backlink]++; + } + } + + if (vm->solution.best_path_id != NO_ID) { + assert(vm->solution.best_path_id < vm->paths.ceil); + refcounts[vm->solution.best_path_id]++; + } + + for (uint32_t p_id = 0; p_id < vm->paths.ceil; p_id++) { + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { continue; } + bool ok; + const uint32_t refcount = get_path_node_refcount(vm, p_id); + ok = refcounts[p_id] == refcount; + + if (!ok) { + dump_path_table(stderr, vm); + + fprintf(stderr, "BAD REFCOUNT: pi[%u], expected %u, got %u\n", + p_id, refcounts[p_id], refcount); + assert(ok); + } + } + + free(refcounts); + LOG(6, "%s: passed\n", __func__); +#else + (void)vm; +#endif +} + +static bool +reserve_path_info_link(struct capvm *vm, uint32_t *pi_id) +{ + if (vm->paths.live == vm->paths.ceil) { +#if ALLOW_PATH_TABLE_RESIZING + if (LOG_CAPVM >= 4) { + fprintf(stderr, "\n"); + dump_path_table(stderr, vm); + check_path_table(vm); + fprintf(stderr, "\n"); + } + + const uint32_t nceil = 2*vm->paths.ceil; + LOG(1, "%s: growing path table %u -> %u\n", + __func__, vm->paths.ceil, nceil); + + /* This can legitimitely be reached with very long inputs, but + * if PATH_TABLE_CEIL_LIMIT is non-zero and this is hit then + * it's most likely a sign of an infinite loop. */ + if (PATH_TABLE_CEIL_LIMIT != 0 && nceil > PATH_TABLE_CEIL_LIMIT) { + assert(!"reached PATH_TABLE_CEIL_LIMIT"); + } + + assert(nceil > vm->paths.ceil); + struct capvm_path_info *npool = realloc(vm->paths.pool, + nceil * sizeof(npool[0])); + if (npool == NULL) { + return false; + } + + for (size_t i = vm->paths.ceil; i < nceil; i++) { + npool[i].u.freelist_node.refcount = 0; + npool[i].u.freelist_node.freelist = i + 1; + } + npool[nceil - 1].u.freelist_node.refcount = 0; + npool[nceil - 1].u.freelist_node.freelist = NO_POS; + vm->paths.freelist_head = vm->paths.ceil; + vm->paths.ceil = nceil; + vm->paths.pool = npool; +#else + assert(!"shouldn't need to grow path pool"); +#endif + } + + assert(vm->paths.live < vm->paths.ceil); + assert(vm->paths.freelist_head != NO_POS); + + *pi_id = vm->paths.freelist_head; + LOG(3, "%s: returning %u\n", __func__, *pi_id); + return true; +} + +/* Release a reference to a path_info_link. Consume the argument. + * If the reference count reaches 0, repool the node and release + * its backlink. */ +static void +release_path_info_link(struct capvm *vm, uint32_t *pi_id) +{ +#define LOG_RELEASE_PI 0 + size_t count = 0; + assert(pi_id != NULL); + uint32_t cur_id = *pi_id; + LOG(4 - LOG_RELEASE_PI, "%s: pi_id %u\n", __func__, cur_id); + *pi_id = NO_ID; + + while (cur_id != NO_ID) { + struct capvm_path_info *pi = &vm->paths.pool[cur_id]; + uint32_t refcount = get_path_node_refcount(vm, cur_id); + LOG(4 - LOG_RELEASE_PI, "-- checking path_info[%u]: refcount %u\n", + cur_id, refcount); + assert(refcount > 0); + LOG(4 - LOG_RELEASE_PI, "release: pi[%u] refcount %u -> %u\n", + cur_id, refcount, refcount - 1); + + const uint32_t backlink = get_path_node_backlink(vm, cur_id); + assert(IS_PATH_NODE(pi)); + pi->u.path.refcount--; + refcount = pi->u.path.refcount; + + if (refcount > 0) { + break; + } + + count++; + LOG(3 - LOG_RELEASE_PI, "-- repooling path_info %u, now %u live\n", + cur_id, vm->paths.live - 1); + LOG(3 - LOG_RELEASE_PI, "-- backlink: %d\n", backlink); + + pi->u.freelist_node.freelist = vm->paths.freelist_head; + vm->paths.freelist_head = cur_id; + assert(vm->paths.live > 0); + vm->paths.live--; + + cur_id = backlink; + if (cur_id == COLLAPSED_ZERO_PREFIX_ID) { + break; + } + } +} + +static void +print_path(FILE *f, const struct capvm *vm, uint32_t p_id) +{ + if (p_id == NO_ID) { + fprintf(f, "/0"); + return; + } + + /* reverse links to the root node */ + uint32_t zero_prefix = 0; + uint32_t next = NO_ID; + uint32_t first = NO_ID; + uint32_t prev; + + while (p_id != NO_ID) { + assert(p_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(!IS_PATH_FREELIST(pi)); + + uint32_t bl; + assert(IS_PATH_NODE(pi)); + bl = pi->u.path.backlink; + pi->u.path.backlink = next; + + if (bl == NO_ID) { + prev = bl; + first = p_id; + break; + } else if (bl == COLLAPSED_ZERO_PREFIX_ID) { + prev = bl; + first = p_id; + zero_prefix = pi->u.path.offset; + break; + } + + next = p_id; + p_id = bl; + } + + if (zero_prefix > 0) { + fprintf(f, "0/%u", zero_prefix); + } + + /* iterate forward, printing and restoring link order */ + p_id = first; + while (p_id != NO_ID) { + assert(p_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(!IS_PATH_FREELIST(pi)); + + uint32_t fl; /* now a forward link */ + assert(IS_PATH_NODE(pi)); + fl = pi->u.path.backlink; + pi->u.path.backlink = prev; + prev = p_id; + fprintf(f, "%s%08x/%u", prev == NO_ID ? "" : " ", + pi->u.path.bits, pi->u.path.used); + + p_id = fl; + } +} + +#if EXPENSIVE_CHECKS +SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() +#endif +static int +cmp_paths(struct capvm *vm, uint32_t p_a, uint32_t p_b) +{ +#if EXPENSIVE_CHECKS + /* When EXPENSIVE_CHECKS is set, walk the chains + * before and after and compare incremental hashing of node IDs, + * to ensure the chains are restored properly. */ + uint64_t hash_a_before = 0; + uint64_t hash_b_before = 0; +#endif + +#define LOG_CMP_PATHS 0 + LOG(3 - LOG_CMP_PATHS, "%s: p_a %d, p_b %d\n", __func__, p_a, p_b); + + if (p_a == NO_ID) { + return p_b == NO_ID ? 0 : -1; + } else if (p_b == NO_ID) { + return 1; + } + + assert(p_a != p_b); + + if (LOG_CAPVM >= 5) { + fprintf(stderr, "A: "); + print_path(stderr, vm, p_a); + fprintf(stderr, "\n"); + + fprintf(stderr, "B: "); + print_path(stderr, vm, p_b); + fprintf(stderr, "\n"); + } + + /* walk both paths backward until they reach a beginning + * or the common prefix node, reversing links along the + * way, then compare forward and restore link order. */ + uint32_t link_a = p_a; + uint32_t link_b = p_b; + + uint32_t fwd_a = NO_ID; + uint32_t fwd_b = NO_ID; + + /* Walk both paths backward, individually until reaching a + * common offset, then back until reaching a common prefix + * (including the start). While iterating backward, replace + * the .backlink field with a forward link, which will be + * reverted when iterating forward and comparing from the + * common prefix. */ + bool common_prefix_found = false; + uint32_t first_a = NO_ID; + uint32_t first_b = NO_ID; + uint32_t common_prefix_link; /* can be NO_ID */ + +#if EXPENSIVE_CHECKS + uint32_t hash_step = 0; /* added so ordering matters */ + while (link_a != NO_ID) { + assert(link_a < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_a); + hash_a_before += hash_id(link_a + hash_step); + link_a = prev; + hash_step++; + } + hash_step = 0; + while (link_b != NO_ID) { + assert(link_b < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_b); + hash_b_before += hash_id(link_b + hash_step); + link_b = prev; + hash_step++; + } + + link_a = p_a; + link_b = p_b; +#endif + + while (!common_prefix_found) { + assert(link_a != NO_ID); + assert(link_b != NO_ID); + assert(link_a < vm->paths.ceil); + assert(link_b < vm->paths.ceil); + + const uint32_t prev_a = get_path_node_backlink(vm, link_a); + const uint32_t prev_b = get_path_node_backlink(vm, link_b); + const uint32_t offset_a = get_path_node_offset(vm, link_a); + const uint32_t offset_b = get_path_node_offset(vm, link_b); + const uint32_t backlink_a = get_path_node_backlink(vm, link_a); + const uint32_t backlink_b = get_path_node_backlink(vm, link_b); + + LOG(3 - LOG_CMP_PATHS, + "%s: backward loop: link_a %d (offset %u, prev %d), link_b %d (offset %u, prev %d)\n", + __func__, link_a, offset_a, prev_a, link_b, offset_b, prev_b); + + assert((offset_a & 31) == 0); /* multiple of 32 */ + assert((offset_b & 31) == 0); /* multiple of 32 */ + if (offset_a > offset_b) { + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: a longer than b\n", __func__); + set_path_node_backlink(vm, link_a, fwd_a); + fwd_a = link_a; + link_a = prev_a; + } else if (offset_b > offset_a) { + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: b longer than a\n", __func__); + set_path_node_backlink(vm, link_b, fwd_b); + fwd_b = link_b; + link_b = prev_b; + } else { + assert(offset_b == offset_a); + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: comparing backlinks: a: %d, b: %d\n", + __func__, backlink_a, backlink_b); + assert(fwd_a != link_a); + set_path_node_backlink(vm, link_a, fwd_a); + assert(fwd_b != link_b); + set_path_node_backlink(vm, link_b, fwd_b); + + if (prev_a == prev_b) { + /* if == NO_ID, empty prefix */ + common_prefix_found = true; + common_prefix_link = prev_a; + first_a = link_a; + first_b = link_b; + + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: common_prefix_found: %d\n", + __func__, common_prefix_link); + } else { + fwd_a = link_a; + fwd_b = link_b; + + link_a = prev_a; + link_b = prev_b; + } + } + } + + assert(first_a != NO_ID); + assert(first_b != NO_ID); + link_a = first_a; + link_b = first_b; + + bool cmp_done = false; + int res; + bool done_restoring_link_order = false; + uint32_t prev_a = common_prefix_link; + uint32_t prev_b = common_prefix_link; + while (!done_restoring_link_order) { + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, link_a %d, link_b %d, cmp_done %d\n", + __func__, link_a, link_b, cmp_done); + if (!cmp_done) { + if (link_a == NO_ID) { /* b is longer */ + cmp_done = true; + if (link_b == NO_ID) { + res = 0; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, equal length, res %d\n", __func__, res); + } else { + res = -1; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, b is longer, res %d\n", __func__, res); + } + } else if (link_b == NO_ID) { /* a is longer */ + cmp_done = true; + res = 1; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, a is longer, res %d\n", __func__, res); + } else { + assert(link_a < vm->paths.ceil); + assert(link_b < vm->paths.ceil); + struct capvm_path_info *pi_a = &vm->paths.pool[link_a]; + struct capvm_path_info *pi_b = &vm->paths.pool[link_b]; + + const uint32_t offset_a = get_path_node_offset(vm, link_a); + const uint32_t offset_b = get_path_node_offset(vm, link_b); + + assert(offset_a == offset_b); + + if (pi_a->u.path.bits != pi_b->u.path.bits) { + res = pi_a->u.path.bits < pi_b->u.path.bits ? -1 : 1; + cmp_done = true; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, different bits (0x%08x, 0x%08x) => res %d\n", + __func__, pi_a->u.path.bits, pi_b->u.path.bits, res); + } + } + } + + /* Check if both have reached the original head node. */ + if (link_a == NO_ID && link_b == NO_ID) { + done_restoring_link_order = true; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop: reached end of both paths, prev_a %d (p_a %d), prev_b %d (p_b %d)\n", + __func__, prev_a, p_a, prev_b, p_b); + assert(prev_a == p_a); + assert(prev_b == p_b); + } + + /* Whether or not comparison has finished, iterate forward, + * restoring forward links. */ + if (link_a != NO_ID) { + assert(link_a < vm->paths.ceil); + const uint32_t fwd_a = get_path_node_backlink(vm, link_a); + LOG(3 - LOG_CMP_PATHS, "%s: fwd loop: link_a %d, fwd_a %d\n", + __func__, link_a, fwd_a); + assert(fwd_a != link_a); + + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, restoring a's backlink: pi[%u].backlink <- %d\n", + __func__, link_a, prev_a); + set_path_node_backlink(vm, link_a, prev_a); + prev_a = link_a; + link_a = fwd_a; + } + + if (link_b != NO_ID) { + assert(link_b < vm->paths.ceil); + const uint32_t fwd_b = get_path_node_backlink(vm, link_b); + LOG(3 - LOG_CMP_PATHS, "%s: fwd loop: link_b %d, fwd_b %d\n", + __func__, link_b, fwd_b); + assert(fwd_b != link_b); + + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, restoring b's backlink: pi[%u].backlink <- %d\n", + __func__, link_b, prev_b); + set_path_node_backlink(vm, link_b, prev_b); + prev_b = link_b; + link_b = fwd_b; + } + } + + LOG(3 - LOG_CMP_PATHS, "%s: res %d\n", __func__, res); + +#if EXPENSIVE_CHECKS + uint64_t hash_a_after = 0; + uint64_t hash_b_after = 0; + hash_step = 0; + link_a = p_a; + while (link_a != NO_ID) { + assert(link_a < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_a); + hash_a_after += hash_id(link_a + hash_step); + link_a = prev; + hash_step++; + } + link_b = p_b; + hash_step = 0; + while (link_b != NO_ID) { + assert(link_b < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_b); + hash_b_after += hash_id(link_b + hash_step); + link_b = prev; + hash_step++; + } + + assert(hash_a_after == hash_a_before); + assert(hash_b_after == hash_b_before); +#endif + + return res; +#undef LOG_CMP_PATHS +} + +static void +handle_possible_matching_path(struct capvm *vm, uint32_t path_info_head, uint32_t uniq_id); + +static bool +copy_path_info(struct capvm *vm, uint32_t path_info_head, + uint32_t *new_path_info_head); + +static bool +extend_path_info(struct capvm *vm, uint32_t path_info_head, bool greedy, uint32_t uniq_id, + uint32_t *new_path_info_head); + +/* Push the next execution step onto the stack, if it hasn't already + * been run by a greedier path. Calling this hands off ownership of the + * path_info_head, so it is released if execution will not be resumed + * later. */ +static void +schedule_possible_next_step(struct capvm *vm, enum pair_id pair_id, + uint32_t input_pos, uint32_t op_id, + uint32_t path_info_head, uint32_t uniq_id) +{ + assert(path_info_head != NO_ID); + uint32_t *stack = vm->run_stacks[pair_id]; + uint32_t *stack_h = &vm->run_stacks_h[pair_id]; + assert((*stack_h) < vm->p->used); + + /* If that instruction has already been evaluated, skip the + * redundant execution by a less greedy path. */ + const uint32_t *evaluated = vm->evaluated[pair_id]; + const bool already_evaluated = GET_BIT32(evaluated, op_id) != 0; + LOG(3, "%s: pair_id %u, input_pos %u, op_id %u, path_info_head %u, uniq_id %u, already_evaluated %d, stack_h %u\n", + __func__, pair_id, input_pos, op_id, path_info_head, uniq_id, already_evaluated, *stack_h); + + if (already_evaluated) { + LOG_EXEC_HALT(uniq_id); + release_path_info_link(vm, &path_info_head); + assert(vm->threads.live > 0); + vm->threads.live--; + } else { + + /* If the work being scheduled by the current greediest + * thread pre-empts work scheduled by a less greedy + * thread, release that thread's path link and clear its + * op ID on the run stack. + * + * TODO: Ideally, avoid the linear scan here, but the + * total stack height is bounded by the generated program size + * and should be fairly small in practice. Wait to change this + * untill there are benchmarks in place showing it's necessary. + * + * An extra two bits per opcode (one for each stack) could + * be used to track whether the stack already contains + * op_id, so the linear scan could be avoided except when + * actually necessary. */ + uint32_t cur_pih = vm->path_info_heads[pair_id][op_id]; + if (cur_pih != NO_ID) { + release_path_info_link(vm, &cur_pih); + vm->path_info_heads[pair_id][op_id] = NO_ID; + const size_t h = *stack_h; + for (size_t i = 0; i < h; i++) { + if (stack[i] == op_id) { + stack[i] = NO_ID; + vm->threads.live--; + } + } + } + stack[(*stack_h)++] = op_id; + vm->path_info_heads[pair_id][op_id] = path_info_head; +#if CAPVM_STATS + vm->uniq_ids[pair_id][op_id] = uniq_id; +#endif + + if (*stack_h > vm->threads.max_live) { + vm->threads.max_live = *stack_h; + if (LOG_CAPVM >= 6) { + LOG(0, "==== new vm->threads.max_live: %u\n", vm->threads.max_live); + dump_path_table(stderr, vm); + } + } + } +} + +static void +eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, + uint32_t path_info_head, uint32_t uniq_id, uint32_t op_id) +{ + LOG(5, "%s: input_pos %u, input_len %u, op_id %u, threads_live %u\n", + __func__, input_pos, vm->input_len, op_id, vm->threads.live); + + if (vm->stats.steps == vm->step_limit) { + LOG(1, "%s: halting, steps == step_limit %zu\n", + __func__, vm->step_limit); + return; + } + + assert(op_id < vm->p->used); + + const struct capvm_opcode *op = &vm->p->ops[op_id]; + + LOG(2, "%s: op_id[%u]: input_pos %u, path_info_head %u, uniq_id %u, op %s\n", + __func__, op_id, input_pos, path_info_head, uniq_id, op_name[op->t]); + LOG_EXEC_OP(uniq_id, input_pos, op_id, op_name[op->t]); + + vm->stats.steps++; + if (vm->stats.steps == vm->step_limit) { + /* TODO: Set some sort of STEP_LIMIT_REACHED error. */ + return; + } + + switch (op->t) { + case CAPVM_OP_CHAR: + if (input_pos == vm->input_len) { + goto halt_thread; /* past end of input */ + } + + LOG(3, "OP_CHAR: input_pos %u, exp char '%c', got '%c'\n", + input_pos, op->u.chr, vm->input[input_pos]); + + if (vm->input[input_pos] != op->u.chr) { + goto halt_thread; /* character mismatch */ + } + LOG_EXEC_CHAR(uniq_id, vm->input[input_pos]); + + schedule_possible_next_step(vm, PAIR_ID_NEXT, input_pos + 1, op_id + 1, + path_info_head, uniq_id); + break; + + case CAPVM_OP_CHARCLASS: + { + if (input_pos == vm->input_len) { + goto halt_thread; /* past end of input */ + } + + const uint8_t c = vm->input[input_pos]; + const uint32_t cc_id = op->u.charclass_id; + assert(cc_id < vm->p->char_classes.count); + const struct capvm_char_class *class = &vm->p->char_classes.sets[cc_id]; + + if (!(class->octets[c/64] & ((uint64_t)1 << (c&63)))) { + goto halt_thread; /* character not in class */ + } + LOG_EXEC_CHAR(uniq_id, vm->input[input_pos]); + + schedule_possible_next_step(vm, PAIR_ID_NEXT, input_pos + 1, op_id + 1, + path_info_head, uniq_id); + break; + } + + case CAPVM_OP_MATCH: + if (input_pos == vm->input_len) { + handle_possible_matching_path(vm, path_info_head, uniq_id); + } else if (vm->input_len > 0 && input_pos == vm->input_len - 1 + && vm->input[input_pos] == '\n') { + LOG(3, "OP_MATCH: special case for trailing newline\n"); + handle_possible_matching_path(vm, path_info_head, uniq_id); + } + goto halt_thread; + + case CAPVM_OP_JMP: + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op->u.jmp, + path_info_head, uniq_id); + break; + + case CAPVM_OP_JMP_ONCE: + { + /* If the destination for this jump has already been visited + * without advancing input, then skip the jump. This is necessary + * for edge cases like the first branch in `^(^|.$)*`, which would + * otherwise have a backward jump to before the first case, due to + * the repetition, and would effectively be treated as an infinite + * loop and ignored, leading to incorrect match bounds for "x". + * + * Replaying the capture path does not track what has been evaluated, + * so this needs to record the branch in the path. This will make + * repetition more expensive in some cases, but compilation could + * emit a JMP when it's safe to do so. */ + const bool greedy = GET_BIT32(vm->evaluated[PAIR_ID_CURRENT], op->u.jmp_once); + if (greedy) { + /* non-greedy branch -- fall through */ + uint32_t new_path_info_head = NO_ID; + if (!extend_path_info(vm, path_info_head, 0, uniq_id, &new_path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + new_path_info_head, uniq_id); + } else { + /* greedy branch -- loop back and potentially match more */ + uint32_t new_path_info_head = NO_ID; + if (!extend_path_info(vm, path_info_head, 1, uniq_id, &new_path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op->u.jmp_once, + new_path_info_head, uniq_id); + } + break; + } + + case CAPVM_OP_SPLIT: + { + const uint32_t dst_cont = op->u.split.cont; + const uint32_t dst_new = op->u.split.new; + + /* destinations must be in range and not self-referential */ + assert(dst_cont < vm->p->used); + assert(dst_new < vm->p->used); + assert(dst_cont != op_id); + assert(dst_new != op_id); + + uint32_t new_path_info_head; + if (!copy_path_info(vm, path_info_head, &new_path_info_head)) { + goto alloc_error; + } + + /* cont is the greedy branch */ + if (!extend_path_info(vm, path_info_head, 1, uniq_id, &path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + + /* new is the non-greedy branch */ + if (!extend_path_info(vm, new_path_info_head, 0, uniq_id, &new_path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + +#if CAPVM_STATS + const uint32_t new_uniq_id = ++vm->uniq_id_counter; +#else + const uint32_t new_uniq_id = 0; +#endif + + vm->threads.live++; + if (vm->threads.live > vm->threads.max_live) { + set_max_threads_live(vm, vm->threads.live); + } + + /* Push the split.new destination, and then the + * split.cont destination on top of it, so that the + * greedier .cont branch will be fully evaluated + * first. */ + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_new, + new_path_info_head, new_uniq_id); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_cont, + path_info_head, uniq_id); + LOG_EXEC_SPLIT(uniq_id, new_uniq_id); + + break; + } + + case CAPVM_OP_SAVE: + /* no-op, during this stage */ + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + + case CAPVM_OP_ANCHOR: + if (op->u.anchor == CAPVM_ANCHOR_START) { + LOG(3, "%s: ^ anchor\n", __func__); + /* ignore a trailing newline, because PCRE does, + * even after a $ anchor. */ + if (input_pos == 0 + && vm->input_len == 1 + && vm->input[0] == '\n') { + /* allowed */ + LOG(3, "%s: special case: ^ ignoring trailing newline\n", __func__); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + } else if (input_pos == 1 + && vm->input_len == 1 + && vm->input[0] == '\n') { + /* allowed */ + } else if (input_pos != 0) { goto halt_thread; } + } else { + assert(op->u.anchor == CAPVM_ANCHOR_END); + LOG(3, "%s: $ anchor: input_len %u, input_pos %u\n", + __func__, vm->input_len, input_pos); + + /* ignore a trailing newline, because PCRE does */ + if (vm->input_len > 0 && input_pos == vm->input_len - 1) { + if (vm->input[input_pos] != '\n') { + goto halt_thread; + } + LOG(3, "%s: special case: $ allowing trailing newline\n", __func__); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + } else if (input_pos != vm->input_len) { + goto halt_thread; + } + } + + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + + default: + assert(!"unreachable"); + return; + } + + if (EXPENSIVE_CHECKS) { /* postcondition */ + check_path_table(vm); + } + + /* FIXME: Check the cleanup logic here. */ + return; + +halt_thread: + /* do not push further execution on the run stack */ + LOG_EXEC_HALT(uniq_id); + + release_path_info_link(vm, &path_info_head); + assert(vm->threads.live > 0); + vm->threads.live--; + return; + +alloc_error: + release_path_info_link(vm, &path_info_head); + vm->res = FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC; +} + +static void +handle_possible_matching_path(struct capvm *vm, uint32_t pi_id, uint32_t uniq_id) +{ + LOG(3, "\n%s: HIT, pi_id %u, uniq_id %u\n", __func__, pi_id, uniq_id); + + if (LOG_CAPVM >= 3) { + LOG(3, "--- current_live: %u, max_live: %u\n", + vm->threads.live, vm->threads.max_live); + dump_path_table(stderr, vm); + LOG(3, "=====\n"); + } + +#if CAPVM_STATS + vm->stats.matches++; +#endif + + assert(pi_id < vm->paths.ceil); + + if (vm->solution.best_path_id == NO_ID) { + struct capvm_path_info *pi = &vm->paths.pool[pi_id]; + assert(!IS_PATH_FREELIST(pi)); + if (LOG_CAPVM >= 5) { + const uint32_t refcount = get_path_node_refcount(vm, pi_id); + LOG(5, "MATCH: pi_id %u refcount %u -> %u\n", + pi_id, refcount, refcount + 1); + } + inc_path_node_refcount(vm, pi_id); + vm->solution.best_path_id = pi_id; +#if CAPVM_STATS + vm->solution.best_path_uniq_id = uniq_id; +#endif + LOG(3, "MATCH: saved current best solution path (pi_id %u)\n", pi_id); + } else { + /* Compare path info and only keep the path associated + * with the greediest match so far. */ + const int res = cmp_paths(vm, pi_id, vm->solution.best_path_id); + if (res > 0) { + /* replace current best solution */ + struct capvm_path_info *pi = &vm->paths.pool[pi_id]; + assert(!IS_PATH_FREELIST(pi)); + if (LOG_CAPVM >= 5) { + const uint32_t refcount = get_path_node_refcount(vm, pi_id); + LOG(5, "MATCH: pi_id %u refcount %u -> %u\n", + pi_id, refcount, refcount + 1); + } + inc_path_node_refcount(vm, pi_id); + + LOG(3, "MATCH: replacing current best solution path %u with %u\n", + vm->solution.best_path_id, pi_id); + + release_path_info_link(vm, &vm->solution.best_path_id); + vm->solution.best_path_id = pi_id; +#if CAPVM_STATS + vm->solution.best_path_uniq_id = uniq_id; +#endif + } else { + /* keep the current best solution */ + LOG(3, "MATCH: ignoring new solution path %u, keeping %u\n", + pi_id, vm->solution.best_path_id); + } + } +} + +static bool +eval_vm(struct capvm *vm) +{ + uint32_t i_i; + + /* init the path_info_heads tables to NO_ID, except for cell 0 + * in next, which contains the starting point. */ + for (size_t op_i = 0; op_i < vm->p->used; op_i++) { + vm->path_info_heads[PAIR_ID_CURRENT][op_i] = NO_ID; +#if CAPVM_STATS + vm->uniq_ids[PAIR_ID_CURRENT][op_i] = NO_ID; +#endif + } + for (size_t op_i = 1; op_i < vm->p->used; op_i++) { + vm->path_info_heads[PAIR_ID_NEXT][op_i] = NO_ID; +#if CAPVM_STATS + vm->uniq_ids[PAIR_ID_NEXT][op_i] = NO_ID; +#endif + } + + for (i_i = 0; i_i <= vm->input_len; i_i++) { + if (vm->threads.live == 0 + || vm->stats.steps == vm->step_limit) { + LOG(3, "%s: breaking, live %u, steps %zu/%zd\n", + __func__, vm->threads.live, vm->stats.steps, vm->step_limit); + break; + } + LOG(3, "\n###### i_i %u\n", i_i); + + LOG(4, "-- clearing evaluated\n"); + const size_t evaluated_bit_words = vm->p->used/32 + 1; + for (size_t i = 0; i < evaluated_bit_words; i++) { + vm->evaluated[PAIR_ID_CURRENT][i] = 0; + vm->evaluated[PAIR_ID_NEXT][i] = 0; + } + + uint32_t *stack_h = &vm->run_stacks_h[PAIR_ID_CURRENT]; + uint32_t *run_stack = vm->run_stacks[PAIR_ID_CURRENT]; + + /* Copy everything from the next run stack to the + * current. Copy in reverse, so items that were pushed + * earlier by greedier paths end up on the top of the + * stack and evalated first, preserving greedy + * ordering. */ + { + const uint32_t next_stack_h = vm->run_stacks_h[PAIR_ID_NEXT]; + const uint32_t *next_stack = vm->run_stacks[PAIR_ID_NEXT]; + uint32_t *next_path_info_heads = vm->path_info_heads[PAIR_ID_NEXT]; + uint32_t *cur_path_info_heads = vm->path_info_heads[PAIR_ID_CURRENT]; + + uint32_t discarded = 0; + for (size_t i = 0; i < next_stack_h; i++) { + const uint32_t op_id = next_stack[i]; + if (op_id == NO_ID) { + assert(!"unreachable"); + discarded++; + continue; + } + + cur_path_info_heads[op_id] = next_path_info_heads[op_id]; + LOG(3, "%s: run_stack[%zd] <- %u, path_info_head %u\n", + __func__, i, op_id, cur_path_info_heads[op_id]); + assert(next_path_info_heads[op_id] < vm->paths.ceil); + next_path_info_heads[op_id] = NO_ID; /* move reference */ +#if CAPVM_STATS + vm->uniq_ids[PAIR_ID_CURRENT][op_id] = + vm->uniq_ids[PAIR_ID_NEXT][op_id]; +#endif + run_stack[next_stack_h - i - 1 - discarded] = op_id; + } + *stack_h = next_stack_h - discarded; + vm->run_stacks_h[PAIR_ID_NEXT] = 0; + +#if CAPVM_PATH_STATS + /* reset counters */ + for (size_t i = 0; i < vm->paths.ceil; i++) { + struct capvm_path_info *pi = &vm->paths.pool[i]; + if (IS_PATH_NODE(pi)) { + pi->u.path.bits_added_per_input_character = 0; + } + } +#endif + } + + uint32_t *path_info_heads = vm->path_info_heads[PAIR_ID_CURRENT]; + while (vm->run_stacks_h[PAIR_ID_CURRENT] > 0) { + /* Do this here, before popping, so that the reference + * on the stack can be counted properly. */ + if (EXPENSIVE_CHECKS) { + check_path_table(vm); + } + + const uint32_t h = --(*stack_h); + assert(h < vm->p->used); + const uint32_t op_id = run_stack[h]; + LOG(4, "%s: popped op_id %d off stack\n", __func__, op_id); + if (op_id == NO_ID) { + LOG(4, "%s: ignoring halted pending execution\n", __func__); + continue; + } + assert(op_id < vm->p->used); + + if (GET_BIT32(vm->evaluated[PAIR_ID_CURRENT], op_id)) { + LOG(2, "%s: evaluated[current] already set for op_id %u (popped off stack), skipping\n", + __func__, op_id); + assert(!"unreachable"); + continue; + } + + LOG(4, "%s: setting evaluated[current] for op_id %u (popped off stack)\n", __func__, op_id); + SET_BIT32(vm->evaluated[PAIR_ID_CURRENT], op_id); + + const uint32_t path_info_head = path_info_heads[op_id]; + LOG(4, "%s: op_id %d's path_info_head: %d\n", __func__, op_id, path_info_head); + path_info_heads[op_id] = NO_ID; + + +#if CAPVM_STATS + const uint32_t uniq_id = vm->uniq_ids[PAIR_ID_CURRENT][op_id]; + assert(uniq_id != NO_ID); +#else + const uint32_t uniq_id = 0; +#endif + eval_vm_advance_greediest(vm, i_i, path_info_head, uniq_id, op_id); + } + + +#if CAPVM_PATH_STATS + uint32_t max_path_bits_added = 0; + for (size_t i = 0; i < vm->paths.ceil; i++) { + const struct capvm_path_info *pi = &vm->paths.pool[i]; + if (IS_PATH_NODE(pi)) { + if (pi->u.path.bits_added_per_input_character > max_path_bits_added) { + max_path_bits_added = pi->u.path.bits_added_per_input_character; + } + } + } + LOG(2, "%s: input_i %u: max_path_bits_added: %u\n", + __func__, i_i, max_path_bits_added); + if (max_path_bits_added > vm->stats.max_bits_added_per_input_character) { + vm->stats.max_bits_added_per_input_character = max_path_bits_added; + } + + if (CAPVM_PATH_STATS > 1) { + dump_path_table(stderr, vm); + } +#endif + } + + return vm->solution.best_path_id != NO_ID; +} + +static bool +copy_path_info(struct capvm *vm, uint32_t path_info_head, + uint32_t *new_path_info_head) +{ + if (!reserve_path_info_link(vm, new_path_info_head)) { + return false; + } + + assert(path_info_head != NO_ID); + assert(path_info_head < vm->paths.ceil); + assert(*new_path_info_head < vm->paths.ceil); + assert(*new_path_info_head != path_info_head); + + /* Since this is the path head, it can never be a collapsed + * zero prefix node. */ + const struct capvm_path_info *pi = &vm->paths.pool[path_info_head]; + assert(IS_PATH_NODE(pi)); + + struct capvm_path_info *npi = &vm->paths.pool[*new_path_info_head]; + assert(IS_PATH_FREELIST(npi)); + + /* unlink from freelist */ + vm->paths.freelist_head = npi->u.freelist_node.freelist; + vm->paths.live++; + if (vm->paths.live > vm->paths.max_live) { + set_max_paths_live(vm); + } + + *npi = (struct capvm_path_info){ + .u.path = { + .refcount = 1, + .used = pi->u.path.used, + .bits = pi->u.path.bits, + .offset = pi->u.path.offset, + .backlink = pi->u.path.backlink, + } + }; + if (pi->u.path.backlink != NO_ID) { + inc_path_node_refcount(vm, pi->u.path.backlink); + } + return true; +} + +#if CAPVM_PATH_STATS +static void +update_max_path_length_memory(struct capvm *vm, const struct capvm_path_info *pi) +{ + const uint32_t len = pi->u.path.used + + (pi->u.path.backlink == COLLAPSED_ZERO_PREFIX_ID + ? 0 /* not actually stored, so don't count it */ + : pi->u.path.offset); + + if (len > vm->stats.max_path_length_memory) { + vm->stats.max_path_length_memory = len; + } +} +#endif + +static bool +extend_path_info(struct capvm *vm, uint32_t pi_id, bool greedy, uint32_t uniq_id, + uint32_t *new_path_info_head) +{ + assert(pi_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[pi_id]; + assert(IS_PATH_NODE(pi)); + + (void)uniq_id; + LOG_EXEC_PATH_FIND_SOLUTION(uniq_id, greedy); + +#define LOG_EPI 0 + LOG(5 - LOG_EPI, "%s: pi_id %u, greedy %d\n", + __func__, pi_id, greedy); + + + if (pi->u.path.used == PATH_LINK_BITS) { /* full */ + uint32_t npi_id; + if (!reserve_path_info_link(vm, &npi_id)) { + assert(!"alloc fail"); + } + pi = &vm->paths.pool[pi_id]; /* refresh stale pointer */ + LOG(5 - LOG_EPI, "%s: new head at %u (%u is full)\n", __func__, npi_id, pi_id); + assert(npi_id < vm->paths.ceil); + struct capvm_path_info *npi = &vm->paths.pool[npi_id]; + vm->paths.freelist_head = npi->u.freelist_node.freelist; + vm->paths.live++; + if (vm->paths.live > vm->paths.max_live) { + set_max_paths_live(vm); + } + + LOG(5 - LOG_EPI, "%s: npi_id %u refcount 1 (new link)\n", + __func__, npi_id); + npi->u.path.refcount = 1; + npi->u.path.offset = pi->u.path.offset + pi->u.path.used; + npi->u.path.bits = (greedy ? ((uint32_t)1 << 31) : 0); + LOG(5 - LOG_EPI, "%s: bits after: 0x%08x\n", __func__, npi->u.path.bits); + npi->u.path.used = 1; + +#if CAPVM_PATH_STATS + npi->u.path.bits_added_per_input_character = pi->u.path.bits_added_per_input_character + 1; +#endif + + /* If the path node is full of zero bits and it's either at the start, + * or its backlink is a COLLAPSED_ZERO_PREFIX_ID, then extend the + * backlink to a collapsed run of zeroes. The node's offset field + * indicates the prefix length. Long prefixes of zero bits tend to + * occur with an unanchored start loop. */ + if (pi->u.path.bits == (uint32_t)0 && USE_COLLAPSED_ZERO_PREFIX + && (pi->u.path.offset == 0 || pi->u.path.backlink == COLLAPSED_ZERO_PREFIX_ID)) { + release_path_info_link(vm, &pi_id); + pi_id = COLLAPSED_ZERO_PREFIX_ID; + +#if CAPVM_STATS + vm->stats.collapsed_zero_prefixes++; +#endif + } else { + /* Check if there's an existing full path node with + * exactly the same bits. If so, link backward to that + * and free the old full one, rather than saving it as + * a duplicate. */ + const uint32_t old_path_bits = pi->u.path.bits; + const uint32_t old_path_offset = pi->u.path.offset; + const uint32_t old_path_backlink = pi->u.path.backlink; + + for (uint32_t epi_id = 0; epi_id < vm->paths.ceil; epi_id++) { + if (epi_id == pi_id) { continue; } + struct capvm_path_info *epi = &vm->paths.pool[epi_id]; + if (IS_PATH_FREELIST(epi)) { + continue; + } + + assert(IS_PATH_NODE(epi)); + if (epi->u.path.used == PATH_LINK_BITS + && epi->u.path.bits == old_path_bits + && epi->u.path.offset == old_path_offset + && epi->u.path.backlink == old_path_backlink) { + + if (LOG_CAPVM >= 4 || 1) { + const uint32_t refcount = get_path_node_refcount(vm, epi_id); + LOG(4 - LOG_EPI, "%s: pi[%u] refcount %u -> %u (reusing identical path backlink %u instead of %u)\n", + __func__, epi_id, refcount, refcount + 1, + epi_id, pi_id); + } + inc_path_node_refcount(vm, epi_id); + release_path_info_link(vm, &pi_id); + pi_id = epi_id; +#if CAPVM_STATS + vm->stats.path_prefixes_shared++; +#endif + break; + } + } + } + + assert(IS_PATH_NODE(npi)); + npi->u.path.backlink = pi_id; + /* transfer pi_id's reference to npi_id */ + *new_path_info_head = npi_id; + +#if CAPVM_PATH_STATS + update_max_path_length_memory(vm, npi); +#endif + + return true; + } else { + assert(IS_PATH_NODE(pi)); + assert(pi->u.path.used < PATH_LINK_BITS); + + LOG(5 - LOG_EPI, "%s: appending to head node %u, %u -> %u used\n", + __func__, pi_id, pi->u.path.used, pi->u.path.used + 1); + assert(pi->u.path.used < PATH_LINK_BITS); + if (greedy) { + LOG(5 - LOG_EPI, "%s: bits before: 0x%08x (greedy: %d)\n", + __func__, pi->u.path.bits, greedy); + pi->u.path.bits |= (uint32_t)1 << (31 - pi->u.path.used); + LOG(5 - LOG_EPI, "%s: bits after: 0x%08x\n", + __func__, pi->u.path.bits); + } + pi->u.path.used++; +#if CAPVM_PATH_STATS + pi->u.path.bits_added_per_input_character++; +#endif + +#if CAPVM_PATH_STATS + update_max_path_length_memory(vm, pi); +#endif + + *new_path_info_head = pi_id; + return true; + } +#undef LOG_EPI +} + +static void +populate_solution(struct capvm *vm) +{ + if (LOG_CAPVM >= 3) { + fsm_capvm_program_dump(stderr, vm->p); + LOG(0, "%s: best_path_id %d, tables:\n", __func__, vm->solution.best_path_id); + dump_path_table(stderr, vm); + check_path_table(vm); + fprintf(stderr, "SOLUTION_PATH: "); + print_path(stderr, vm, vm->solution.best_path_id); + fprintf(stderr, "\n"); + } + +#if CAPVM_PATH_STATS + LOG(1, "%s: prog_size %u, max_path_length_memory %u (bits)\n", + __func__, vm->p->used, vm->stats.max_path_length_memory); + const uint32_t uniq_id = vm->solution.best_path_uniq_id; +#else + const uint32_t uniq_id = NO_ID; +#endif + (void)uniq_id; + + /* Interpret the program again, but rather than using the input to + * drive execution, use the saved path for the primary solution. */ + + /* Walk the solution path, reversing the edges temporarily so it + * can be executed start to finish, and truncate any bits appended + * after branches on the path. */ + assert(vm->solution.best_path_id != NO_ID); + assert(vm->solution.best_path_id < vm->paths.ceil); + + uint32_t path_link = vm->solution.best_path_id; + uint32_t next_link = NO_ID; + uint32_t next_offset = NO_POS; + uint32_t first_link = NO_ID; + + size_t split_count = 0; + uint32_t zero_prefix_length = 0; + + if (LOG_CAPVM >= 3) { + const struct capvm_path_info *pi = &vm->paths.pool[path_link]; + assert(!IS_PATH_FREELIST(pi)); + LOG(3, "%s: best_path %d, path_length %u\n", + __func__, vm->solution.best_path_id, pi->u.path.offset + pi->u.path.used); + if (LOG_CAPVM > 4) { + dump_path_table(stderr, vm); + } + } + + uint32_t prev; + do { + struct capvm_path_info *pi = &vm->paths.pool[path_link]; + assert(!IS_PATH_FREELIST(pi)); + const uint32_t prev_link = get_path_node_backlink(vm, path_link); + + if (LOG_CAPVM >= 3) { + if (IS_PATH_NODE(pi)) { + LOG(3, "%s (moving back), node %u: refcount %u, used %u, offset %u, backlink %d, bits '", + __func__, path_link, pi->u.path.refcount, pi->u.path.used, + pi->u.path.offset, pi->u.path.backlink); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", bit ? '1' : '0'); + } + LOG(3, "'\n"); + } + } + + split_count += pi->u.path.used; + + if (next_link != NO_ID) { + LOG(3, "-- setting backlink to %d\n", next_link); + set_path_node_backlink(vm, path_link, next_link); /* point fwd */ + } else { + LOG(3, "-- setting backlink to %d\n", NO_ID); + set_path_node_backlink(vm, path_link, NO_ID); /* now EOL */ + } + + if (prev_link == NO_ID) { + first_link = path_link; + prev = prev_link; + } else if (prev_link == COLLAPSED_ZERO_PREFIX_ID) { + first_link = path_link; + split_count += pi->u.path.offset; + zero_prefix_length = pi->u.path.offset; + prev = prev_link; + } + + next_offset = get_path_node_offset(vm, path_link); + next_link = path_link; + assert(path_link != prev_link); + path_link = prev_link; + } while (path_link != NO_ID && path_link != COLLAPSED_ZERO_PREFIX_ID); + + /* iter forward */ + uint32_t cur = first_link; + if (LOG_CAPVM >= 3) do { + struct capvm_path_info *pi = &vm->paths.pool[cur]; + + assert(IS_PATH_NODE(pi)); + LOG(3, "%s (moving fwd): node %u: refcount %u, used %u, offset %u, fwdlink %d, bits '", + __func__, cur, get_path_node_refcount(vm, cur), + pi->u.path.used, + get_path_node_offset(vm, cur), + get_path_node_backlink(vm, cur)); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", bit ? '1' : '0'); + } + LOG(3, "'\n"); + + const uint32_t next_cur = get_path_node_backlink(vm, cur); + assert(cur != next_cur); + cur = next_cur; /* fwd link */ + } while (cur != NO_ID); + + /* evaluate program with forward path */ + LOG(3, "%s: split_count %zu\n", __func__, split_count); + size_t split_i = 0; + uint32_t prog_i = 0; + uint32_t input_i = 0; + size_t capture_lookup_steps = 0; + bool done = false; + + /* This flag tracks whether an explicit newline was matched at + * the end of input. Normally a trailing newline is implicitly + * ignored in the bounds for captures, but when the regex + * matches a newline at the end, it must still be included. An + * example case where this matters is `^[^x]$` for "\n", because + * the character class matches the newline this should capture + * as (0,1). */ + bool explicitly_matched_nl_at_end = false; + + cur = first_link; + while (split_i < split_count || !done) { + assert(prog_i < vm->p->used); + const uint32_t cur_prog_i = prog_i; + const struct capvm_opcode *op = &vm->p->ops[cur_prog_i]; + LOG(3, "%s: i_i %u, p_i %u, s_i %zu/%zu, op %s\n", + __func__, input_i, cur_prog_i, split_i, split_count, op_name[op->t]); + + prog_i++; + capture_lookup_steps++; + switch (op->t) { + case CAPVM_OP_CHAR: + assert(input_i < vm->input_len); + LOG(3, "OP_CHAR: input_i %u, exp char '%c', got '%c'\n", + input_i, op->u.chr, vm->input[input_i]); + assert(vm->input[input_i] == op->u.chr); + if (vm->input_len > 0 + && input_i == vm->input_len - 1 + && vm->input[input_i] == '\n') { + explicitly_matched_nl_at_end = true; + } + input_i++; + break; + case CAPVM_OP_CHARCLASS: + assert(input_i < vm->input_len); + if (vm->input_len > 0 + && input_i == vm->input_len - 1 + && vm->input[input_i] == '\n') { + explicitly_matched_nl_at_end = true; + } + input_i++; + break; + case CAPVM_OP_MATCH: + LOG(2, "split_i %zu, split_count %zu\n", split_i, split_count); + assert(split_i == split_count); + done = true; + break; + case CAPVM_OP_JMP: + prog_i = op->u.jmp; + break; + case CAPVM_OP_JMP_ONCE: + { + /* look at next bit of path and jmp or fall through */ + const uint32_t offset = get_path_node_offset(vm, cur); + const struct capvm_path_info *pi = &vm->paths.pool[cur]; + + assert(IS_PATH_NODE(pi)); + bool next_bit; + LOG(3, "%s: OP_JMP_ONCE: split_i %zu, zpl %u, offset %u, pi->u.path.used %u\n", + __func__, split_i, zero_prefix_length, offset, pi->u.path.used); + if (split_i < zero_prefix_length) { + next_bit = 0; + } else { + assert(split_i >= offset && + split_i <= offset + pi->u.path.used); + const uint32_t shift = 31 - (split_i & 31); + assert(shift < PATH_LINK_BITS); + next_bit = (pi->u.path.bits & ((uint32_t)1 << shift)) != 0; + } + LOG(3, "jmp_once: next_bit %d\n", next_bit); + LOG_EXEC_PATH_SAVE_CAPTURES(uniq_id, next_bit); + if (next_bit) { /* greedy edge */ + prog_i = op->u.jmp_once; + } else { /* non-greedy edge */ + /* fall through */ + } + split_i++; + if (split_i >= offset && + split_i - offset == pi->u.path.used && split_i < split_count) { + const uint32_t backlink = get_path_node_backlink(vm, cur); + assert(backlink != NO_ID); + cur = backlink; + } + LOG(3, "%s: prog_i now %u, split_i %zu/%zu\n", + __func__, prog_i, split_i, split_count); + assert(split_i <= split_count); + break; + } + case CAPVM_OP_SPLIT: + { + /* look at next bit of path and act accordingly */ + const uint32_t offset = get_path_node_offset(vm, cur); + const struct capvm_path_info *pi = &vm->paths.pool[cur]; + + const uint32_t dst_cont = op->u.split.cont; + const uint32_t dst_new = op->u.split.new; + + assert(IS_PATH_NODE(pi)); + bool next_bit; + LOG(3, "%s: OP_SPLIT_CONT: split_i %zu, zpl %u, offset %u, pi->u.path.used %u\n", + __func__, split_i, zero_prefix_length, offset, pi->u.path.used); + if (split_i < zero_prefix_length) { + next_bit = 0; + } else { + assert(split_i >= offset && + split_i <= offset + pi->u.path.used); + const uint32_t shift = 31 - (split_i & 31); + assert(shift < PATH_LINK_BITS); + next_bit = (pi->u.path.bits & ((uint32_t)1 << shift)) != 0; + } + LOG(3, "split: next_bit %d\n", next_bit); + LOG_EXEC_PATH_SAVE_CAPTURES(uniq_id, next_bit); + if (next_bit) { /* greedy edge */ + prog_i = dst_cont; + } else { /* non-greedy edge */ + prog_i = dst_new; + } + split_i++; + if (split_i >= offset && + split_i - offset == pi->u.path.used && split_i < split_count) { + const uint32_t backlink = get_path_node_backlink(vm, cur); + assert(backlink != NO_ID); + cur = backlink; + } + LOG(3, "%s: prog_i now %u, split_i %zu/%zu\n", + __func__, prog_i, split_i, split_count); + assert(split_i <= split_count); + + break; + } + case CAPVM_OP_SAVE: + { + const unsigned capture_id = op->u.save/2; + const bool is_end = (op->u.save & 1) == 1; + + LOG(5, "%s: input_i %u, save %d -> capture %d pos %d, cur value %zd, prev char 0x%02x\n", + __func__, + input_i, op->u.save, + capture_id, is_end, + vm->capture_buf[op->u.save/2].pos[op->u.save & 1], + input_i > 0 ? vm->input[input_i - 1] : 0xff); + + /* Special case to ignore a trailing + * newline when capturing, unless the + * newline was explicitly matched as the + * last character of input. */ + if (input_i > 0 + && !explicitly_matched_nl_at_end + && input_i == vm->input_len + && vm->input[input_i - 1] == '\n') { + LOG(3, "%s: updating capture[%u].pos[1] to ignore trailing '\\n' at %u\n", + __func__, capture_id, input_i); + vm->capture_buf[capture_id].pos[is_end] = input_i - 1; + } else { + /* Save current position to appropriate capture buffer endpoint */ + vm->capture_buf[op->u.save/2].pos[op->u.save & 1] = input_i; + LOG(3, "%s: saved capture[%d].pos[%d] <- %u\n", + __func__, op->u.save/2, op->u.save&1, input_i); + } + break; + } + case CAPVM_OP_ANCHOR: + if (op->u.anchor == CAPVM_ANCHOR_START) { + assert(input_i == 0 + || (input_i == 1 + && vm->input_len == 1 + && vm->input[0] == '\n')); + } else { + assert(op->u.anchor == CAPVM_ANCHOR_END); + LOG(3, "%s: $ anchor: input_len %u, input_i %u\n", + __func__, vm->input_len, input_i); + + if (vm->input_len > 0 && input_i == vm->input_len - 1) { + /* special hack to not include trailing newline + * in match group zero */ + if (vm->p->capture_count > 0) { + vm->capture_buf[0].pos[1] = input_i; + } + + assert(vm->input[input_i] == '\n'); + input_i++; + } else { + assert(input_i == vm->input_len); + } + } + break; + + default: + assert(!"match fail"); + } + } + + /* write solution into caller's buffers and print */ + if (LOG_SOLUTION_TO_STDOUT) { + /* fprintf(stderr, "capture_count %u\n", vm->p->capture_count); */ + printf("HIT:"); + for (unsigned i = 0; i < vm->p->capture_count; i++) { + printf(" %zd %zd", + vm->capture_buf[i].pos[0], vm->capture_buf[i].pos[1]); + } + printf("\n"); + } + + /* restore original link order */ + cur = first_link; + do { + struct capvm_path_info *pi = &vm->paths.pool[cur]; + assert(!IS_PATH_FREELIST(pi)); + const uint32_t backlink = get_path_node_backlink(vm, cur); + + LOG(3, "%s (moving fwd again): node %u: refcount %u, used %u, offset %u, fwdlink %d, bits '", + __func__, cur, get_path_node_refcount(vm, cur), + pi->u.path.used, + get_path_node_offset(vm, cur), + backlink); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", (pi->u.path.bits & bit) ? '1' : '0'); + } + LOG(3, "'\n"); + + LOG(3, "-- setting node %u's backlink to %d\n", cur, prev); + const uint32_t next = backlink; + set_path_node_backlink(vm, cur, prev); + + prev = cur; + cur = next; /* fwd link */ + } while (cur != NO_ID); +} + +/* TODO: It should be possible to avoid dynamic allocation here + * by calculating the max space needed upfront and passing in a + * uint32_t or uint64_t-aligned array for working space. */ + +enum fsm_capvm_program_exec_res +fsm_capvm_program_exec(const struct capvm_program *program, + const uint8_t *input, size_t length, + struct fsm_capture *capture_buf, size_t capture_buf_length) +{ + assert(program != NULL); + assert(input != NULL || length == 0); + assert(capture_buf != NULL); + + const size_t thread_max = program->used; + + /* FIXME: The path node table can grow beyond this, but in + * practice will usually stay fairly small. The worst case + * should be decidable based on the compiled program and input + * length. */ +#if ALLOW_PATH_TABLE_RESIZING + const size_t path_info_max = thread_max; +#else + const size_t path_info_max = 3 * thread_max; +#endif + + struct capvm_path_info *path_info_pool = malloc(path_info_max + * sizeof(path_info_pool[0])); + if (path_info_pool == NULL) { + return FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC; + } + assert(path_info_pool != NULL); + + /* link path_info freelist */ + for (size_t i = 1; i < path_info_max - 1; i++) { + struct capvm_path_info *pi = &path_info_pool[i]; + pi->u.freelist_node.refcount = 0; + pi->u.freelist_node.freelist = i + 1; + } + struct capvm_path_info *piZ = &path_info_pool[path_info_max - 1]; + piZ->u.freelist_node.refcount = 0; + piZ->u.freelist_node.freelist = NO_ID; + + /* init an empty path descriptor for initial execution */ + struct capvm_path_info *pi0 = &path_info_pool[0]; + pi0->u.path.refcount = 1; + pi0->u.path.used = 0; + pi0->u.path.bits = 0; + pi0->u.path.offset = 0; + pi0->u.path.backlink = NO_ID; + + uint32_t stack_a[thread_max]; + uint32_t stack_b[thread_max]; + + const size_t evaluated_bit_words = program->used/32 + 1; + uint32_t evaluated_a[evaluated_bit_words]; + uint32_t evaluated_b[evaluated_bit_words]; + uint32_t path_info_head_a[thread_max]; + uint32_t path_info_head_b[thread_max]; +#if CAPVM_STATS + uint32_t uniq_ids_a[thread_max]; + uint32_t uniq_ids_b[thread_max]; +#endif + + assert(capture_buf_length >= program->capture_base + program->capture_count); + + struct fsm_capture *offset_capture_buf = &capture_buf[program->capture_base]; + + struct capvm vm = { + .res = FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND, + .p = program, + .input = input, + .input_len = length, + .capture_buf = offset_capture_buf, + .capture_buf_length = capture_buf_length, + .step_limit = SIZE_MAX, +#if CAPVM_STATS + .uniq_id_counter = 0, +#endif + + .run_stacks = { stack_a, stack_b }, + .evaluated = { evaluated_a, evaluated_b }, + .path_info_heads = { path_info_head_a, path_info_head_b }, +#if CAPVM_STATS + .uniq_ids = { uniq_ids_a, uniq_ids_b }, +#endif + + .paths = { + .ceil = path_info_max, + .live = 1, + .max_live = 1, + .freelist_head = 1, + .pool = path_info_pool, + }, + .solution = { + .best_path_id = NO_ID, + }, + }; + + /* enqueue execution at first opcode */ + vm.run_stacks[PAIR_ID_NEXT][0] = 0; + vm.run_stacks_h[PAIR_ID_NEXT] = 1; + vm.threads.live = 1; + vm.threads.max_live = 1; + vm.path_info_heads[PAIR_ID_NEXT][0] = 0; + +#if CAPVM_STATS + vm.uniq_ids[PAIR_ID_NEXT][0] = 0; +#endif + + INIT_TIMERS(); + TIME(&pre); + if (eval_vm(&vm)) { + assert(vm.threads.live == 0); + assert(vm.paths.live > 0); + + populate_solution(&vm); + release_path_info_link(&vm, &vm.solution.best_path_id); + vm.res = FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN; + + /* TODO: This assert will not work if refcounts are + * sticky at the max value, but if the number of paths + * and threads is bounded then it shouldn't be possible + * to overflow the refcount anyway. If sticky refcounts + * are used then reaching one should probably set a + * flag, which would skip this assertion. */ + assert(vm.paths.live == 0); + } else { + assert(vm.res == FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND); + } + + TIME(&post); + DIFF_MSEC(__func__, pre, post, NULL); + +#if CAPVM_STATS + LOG(2, "%s: %zu steps, max_threads %u, max_paths %u, matches %u, path_prefixes_shared %u, collapsed_zero_prefixes %u\n", + __func__, vm.stats.steps, vm.threads.max_live, vm.paths.max_live, vm.stats.matches, + vm.stats.path_prefixes_shared, vm.stats.collapsed_zero_prefixes); +#if CAPVM_PATH_STATS + LOG(2, "%s: prog_size %u, max_path_length_memory %u (bits), input length %zu, max_paths * %zu bytes/path => %zu bytes\n", + __func__, vm.p->used, vm.stats.max_path_length_memory, length, + sizeof(vm.paths.pool[0]), + vm.paths.max_live * sizeof(vm.paths.pool[0])); +#endif +#endif + + free(vm.paths.pool); + return vm.res; +} diff --git a/src/libfsm/capture_vm_log.h b/src/libfsm/capture_vm_log.h new file mode 100644 index 000000000..8ff51d8b4 --- /dev/null +++ b/src/libfsm/capture_vm_log.h @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_VM_LOG_H +#define CAPTURE_VM_LOG_H + +#include + +#define LOG_CAPVM (1+0) +#define LOG(LEVEL, ...) \ + do { \ + if ((LEVEL) <= LOG_CAPVM) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while(0) + + +#endif diff --git a/src/libfsm/capture_vm_program.h b/src/libfsm/capture_vm_program.h new file mode 100644 index 000000000..15e585761 --- /dev/null +++ b/src/libfsm/capture_vm_program.h @@ -0,0 +1,74 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_VM_PROGRAM_H +#define CAPTURE_VM_PROGRAM_H + +#include + +struct capvm_program { + uint32_t capture_count; + uint32_t capture_base; + + uint32_t used; + uint32_t ceil; + struct capvm_opcode { + enum capvm_opcode_type { + /* Next character of input == .u.chr */ + CAPVM_OP_CHAR, + /* Next character of input is in char class */ + CAPVM_OP_CHARCLASS, + /* Input has matched */ + CAPVM_OP_MATCH, + /* Unconditional jump */ + CAPVM_OP_JMP, + /* If destination has already been evaulated + * since advancing the input position, fall + * through to next instruction, otherwise jmp. */ + CAPVM_OP_JMP_ONCE, + /* Split execution to two paths, where .cont + * offset is greedier than .new's offset. */ + CAPVM_OP_SPLIT, + /* Save current input position as capture bound */ + CAPVM_OP_SAVE, + /* Check if current input position is at start/end + * of input, after accounting for PCRE's special + * cases for a trailing newline. */ + CAPVM_OP_ANCHOR, + } t; + union { + uint8_t chr; + uint32_t charclass_id; + uint32_t jmp; /* absolute */ + uint32_t jmp_once; /* absolute */ + struct { + uint32_t cont; /* greedy branch */ + uint32_t new; /* non-greedy branch */ + } split; + /* (save >> 1): capture ID, + * (save & 0x01): save pos to start (0b0) or end (0b1). */ + uint32_t save; + enum capvm_anchor_type { + CAPVM_ANCHOR_START, + CAPVM_ANCHOR_END, + } anchor; + } u; + } *ops; + + /* Most compiled programs only use a few distinct character + * classes (if any), and the data is much larger than the + * other instructions, so they are stored in a separate + * table and referred to by op->u.charclass_id. */ + struct capvm_char_classes { + uint32_t count; + uint32_t ceil; + struct capvm_char_class { + uint64_t octets[4]; /* 256-bitset */ + } *sets; + } char_classes; +}; + +#endif diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index bec16bb0f..8b7b606c8 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -21,10 +21,7 @@ #define LOG_CLONE_ENDIDS 0 static int -copy_capture_actions(struct fsm *dst, const struct fsm *src); - -static int -copy_end_ids(struct fsm *dst, const struct fsm *src); +copy_end_metadata(struct fsm *dst, const struct fsm *src); struct fsm * fsm_clone(const struct fsm *fsm) @@ -70,12 +67,7 @@ fsm_clone(const struct fsm *fsm) } { - if (!copy_capture_actions(new, fsm)) { - fsm_free(new); - return NULL; - } - - if (!copy_end_ids(new, fsm)) { + if (!copy_end_metadata(new, fsm)) { fsm_free(new); return NULL; } @@ -84,75 +76,81 @@ fsm_clone(const struct fsm *fsm) return new; } -struct copy_capture_actions_env { +struct copy_end_ids_env { + char tag; struct fsm *dst; int ok; }; static int -copy_capture_actions_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +copy_end_ids_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque) { - struct copy_capture_actions_env *env = opaque; - assert(env->dst); + struct copy_end_ids_env *env = opaque; + enum fsm_endid_set_res sres; + assert(env->tag == 'c'); + (void)fsm; + (void)nth; - if (!fsm_capture_add_action(env->dst, - state, type, capture_id, to)) { +#if LOG_CLONE_ENDIDS + fprintf(stderr, "clone[%d] <- %d\n", state, id); +#endif + + sres = fsm_endid_set(env->dst, state, id); + if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { env->ok = 0; + return 0; } - return env->ok; + return 1; } static int -copy_capture_actions(struct fsm *dst, const struct fsm *src) +copy_active_capture_ids_cb(fsm_state_t state, unsigned capture_id, void *opaque) { - struct copy_capture_actions_env env = { NULL, 1 }; - env.dst = dst; + struct copy_end_ids_env *env = opaque; - fsm_capture_action_iter(src, - copy_capture_actions_cb, &env); - return env.ok; + if (!fsm_capture_set_active_for_end(env->dst, + capture_id, + state)) { + env->ok = 0; + return 0; + } + return 1; } -struct copy_end_ids_env { - char tag; - struct fsm *dst; - const struct fsm *src; - int ok; -}; - static int -copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) +associate_capture_programs_cb(fsm_state_t state, unsigned prog_id, void *opaque) { struct copy_end_ids_env *env = opaque; - enum fsm_endid_set_res sres; - assert(env->tag == 'c'); -#if LOG_CLONE_ENDIDS - fprintf(stderr, "clone[%d] <- %d\n", state, id); -#endif - - sres = fsm_endid_set(env->dst, state, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_capture_associate_program_with_end_state(env->dst, + prog_id, state)) { env->ok = 0; return 0; } - return 1; } static int -copy_end_ids(struct fsm *dst, const struct fsm *src) +copy_end_metadata(struct fsm *dst, const struct fsm *src) { struct copy_end_ids_env env; env.tag = 'c'; /* for clone */ env.dst = dst; - env.src = src; env.ok = 1; fsm_endid_iter(src, copy_end_ids_cb, &env); + fsm_capture_iter_active_for_all_end_states(src, + copy_active_capture_ids_cb, &env); + + if (!fsm_capture_copy_programs(src, dst)) { + return 0; + } + + fsm_capture_iter_program_ids_for_all_end_states(src, + associate_capture_programs_cb, &env); + return env.ok; } diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 390b2a0c5..b34e69910 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "internal.h" #include "capture.h" @@ -26,27 +27,15 @@ #define LOG_MAPPING 0 #define LOG_CONSOLIDATE_CAPTURES 0 -#define LOG_CONSOLIDATE_ENDIDS 0 +#define LOG_CONSOLIDATE_END_METADATA 0 struct mapping_closure { size_t count; const fsm_state_t *mapping; }; -struct consolidate_copy_capture_actions_env { - char tag; - struct fsm *dst; - size_t mapping_count; - const fsm_state_t *mapping; - int ok; -}; - -static int -consolidate_copy_capture_actions(struct fsm *dst, const struct fsm *src, - const fsm_state_t *mapping, size_t mapping_count); - static int -consolidate_end_ids(struct fsm *dst, const struct fsm *src, +consolidate_end_metadata(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count); static fsm_state_t @@ -68,6 +57,9 @@ fsm_consolidate(const struct fsm *src, size_t max_used = 0; assert(src != NULL); + if (mapping_count == 0) { + return fsm_clone(src); + } assert(src->opt != NULL); dst = fsm_new(src->opt); @@ -76,12 +68,14 @@ fsm_consolidate(const struct fsm *src, } for (src_i = 0; src_i < mapping_count; src_i++) { + const fsm_state_t dst_i = mapping[src_i]; #if LOG_MAPPING fprintf(stderr, "consolidate_mapping[%u]: %u\n", src_i, mapping[src_i]); #endif - if (mapping[src_i] >= max_used) { - max_used = mapping[src_i]; + if (dst_i > max_used) { + assert(dst_i != FSM_STATE_REMAP_NO_STATE); + max_used = dst_i; } } @@ -96,8 +90,8 @@ fsm_consolidate(const struct fsm *src, goto cleanup; } -#define DST_SEEN(I) (seen[I/64] & ((uint64_t)1 << (I&63))) -#define SET_DST_SEEN(I) (seen[I/64] |= ((uint64_t)1 << (I&63))) +#define DST_SEEN(I) u64bitset_get(seen, I) +#define SET_DST_SEEN(I) u64bitset_set(seen, I) /* map N states to M states, where N >= M. * if it's the first time state[M] is seen, @@ -110,6 +104,9 @@ fsm_consolidate(const struct fsm *src, for (src_i = 0; src_i < mapping_count; src_i++) { const fsm_state_t dst_i = mapping[src_i]; + /* fsm_consolidate does not currently support discarding states. */ + assert(dst_i != FSM_STATE_REMAP_NO_STATE); + if (!DST_SEEN(dst_i)) { SET_DST_SEEN(dst_i); @@ -134,11 +131,11 @@ fsm_consolidate(const struct fsm *src, } } - if (!consolidate_copy_capture_actions(dst, src, mapping, mapping_count)) { + if (!fsm_capture_copy_programs(src, dst)) { goto cleanup; } - if (!consolidate_end_ids(dst, src, mapping, mapping_count)) { + if (!consolidate_end_metadata(dst, src, mapping, mapping_count)) { goto cleanup; } @@ -161,31 +158,42 @@ fsm_consolidate(const struct fsm *src, return NULL; } +struct consolidate_end_ids_env { + char tag; + struct fsm *dst; + const struct fsm *src; + const fsm_state_t *mapping; + size_t mapping_count; + int ok; +}; + static int -consolidate_copy_capture_actions_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +consolidate_end_ids_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque) { - struct consolidate_copy_capture_actions_env *env = opaque; - fsm_state_t s, t; - + struct consolidate_end_ids_env *env = opaque; + enum fsm_endid_set_res sres; + fsm_state_t s; assert(env->tag == 'C'); -#if LOG_CONSOLIDATE_CAPTURES - fprintf(stderr, "consolidate_copy_capture_actions_cb: state %u, type %s, ID %u, TO %d\n", - state, - fsm_capture_action_type_name[type], - capture_id, to); + (void)fsm; + (void)nth; + +#if LOG_CONSOLIDATE_END_METADATA > 1 + fprintf(stderr, "consolidate_end_ids_cb: state %u, ID[%zu]: %d\n", + state, nth, id); + fprintf(stderr, " -- mapping_count %zu\n", + env->mapping_count); #endif - assert(state < env->mapping_count); - assert(to == CAPTURE_NO_STATE || to < env->mapping_count); s = env->mapping[state]; - t = to == CAPTURE_NO_STATE - ? CAPTURE_NO_STATE : env->mapping[to]; - if (!fsm_capture_add_action(env->dst, - s, type, capture_id, t)) { +#if LOG_CONSOLIDATE_END_METADATA > 1 + fprintf(stderr, "consolidate[%d] <- %d\n", s, id); +#endif + + sres = fsm_endid_set(env->dst, s, id); + if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { env->ok = 0; return 0; } @@ -194,76 +202,54 @@ consolidate_copy_capture_actions_cb(fsm_state_t state, } static int -consolidate_copy_capture_actions(struct fsm *dst, const struct fsm *src, - const fsm_state_t *mapping, size_t mapping_count) +consolidate_active_captures_cb(fsm_state_t state, unsigned capture_id, + void *opaque) { - size_t i; + struct consolidate_end_ids_env *env = opaque; + fsm_state_t dst_s; + assert(env->tag == 'C'); - struct consolidate_copy_capture_actions_env env; - env.tag = 'C'; - env.dst = dst; - env.mapping_count = mapping_count; - env.mapping = mapping; - env.ok = 1; + assert(state < env->mapping_count); + dst_s = env->mapping[state]; -#if LOG_MAPPING - for (i = 0; i < mapping_count; i++) { - fprintf(stderr, "mapping[%lu]: %u\n", i, mapping[i]); - } -#else - (void)i; +#if LOG_CONSOLIDATE_END_METADATA + fprintf(stderr, "consolidate_active_captures_cb: state %d -> dst_s %d, capture_id %u\n", + state, dst_s, capture_id); #endif - fsm_capture_action_iter(src, - consolidate_copy_capture_actions_cb, &env); - return env.ok; + if (!fsm_capture_set_active_for_end(env->dst, capture_id, dst_s)) { + env->ok = 0; + return 0; + } + return 1; } -struct consolidate_end_ids_env { - char tag; - struct fsm *dst; - const struct fsm *src; - const fsm_state_t *mapping; - size_t mapping_count; - int ok; -}; - static int -consolidate_end_ids_cb(fsm_state_t state, const fsm_end_id_t id, +consolidate_capture_programs_cb(fsm_state_t state, unsigned program_id, void *opaque) { struct consolidate_end_ids_env *env = opaque; - enum fsm_endid_set_res sres; - fsm_state_t s; + fsm_state_t dst_s; assert(env->tag == 'C'); -#if LOG_CONSOLIDATE_ENDIDS > 1 - fprintf(stderr, "consolidate_end_ids_cb: state %u, count %lu, ID %d:", - state, id); - for (i = 0; i < count; i++) { - fprintf(stderr, " %u", ids[i]); - } - fprintf(stderr, "\n"); -#endif - assert(state < env->mapping_count); - s = env->mapping[state]; + dst_s = env->mapping[state]; -#if LOG_CONSOLIDATE_ENDIDS > 1 - fprintf(stderr, "consolidate[%d] <- %d\n", s, id); +#if LOG_CONSOLIDATE_END_METADATA + fprintf(stderr, "%s: state %d -> dst_s %d, capture_id %u\n", + __func__, state, dst_s, program_id); #endif - sres = fsm_endid_set(env->dst, s, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_capture_associate_program_with_end_state(env->dst, + (uint32_t)program_id, dst_s)) { env->ok = 0; return 0; } - return 1; } static int -consolidate_end_ids(struct fsm *dst, const struct fsm *src, +consolidate_end_metadata(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count) { struct consolidate_end_ids_env env; @@ -276,9 +262,28 @@ consolidate_end_ids(struct fsm *dst, const struct fsm *src, fsm_endid_iter(src, consolidate_end_ids_cb, &env); -#if LOG_CONSOLIDATE_ENDIDS > 1 + if (env.ok) { + fsm_state_t s; + const size_t src_state_count = fsm_countstates(src); + for (s = 0; s < src_state_count; s++) { + fsm_capture_iter_active_for_end_state(src, s, + consolidate_active_captures_cb, &env); + if (!env.ok) { + break; + } + + fsm_capture_iter_program_ids_for_end_state(src, s, + consolidate_capture_programs_cb, &env); + if (!env.ok) { + break; + } + } + } + +#if LOG_CONSOLIDATE_END_METADATA > 1 fprintf(stderr, "==== fsm_consolidate -- endid_info after:\n"); fsm_endid_dump(stderr, dst); + fsm_capture_dump_active_for_ends(stderr, dst); #endif return env.ok; diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 56e135afd..817fd335b 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -6,6 +6,8 @@ #include "determinise_internal.h" +#define LOG_DETERMINISATION_COUNTERS 0 + static void dump_labels(FILE *f, const uint64_t labels[4]) { @@ -29,6 +31,8 @@ fsm_determinise(struct fsm *nfa) size_t dfacount = 0; struct analyze_closures_env ac_env = { 0 }; + INIT_TIMERS(); + INIT_TIMERS_NAMED(overall); assert(nfa != NULL); map.alloc = nfa->opt->alloc; @@ -39,9 +43,12 @@ fsm_determinise(struct fsm *nfa) * faster where we can start with an epsilon-free NFA in the first place. */ if (fsm_has(nfa, fsm_hasepsilons)) { + TIME(&pre); if (!fsm_remove_epsilons(nfa)) { return 0; } + TIME(&post); + DIFF_MSEC("det_remove_eps", pre, post, NULL); } #if LOG_DETERMINISE_CAPTURES || LOG_INPUT @@ -49,6 +56,7 @@ fsm_determinise(struct fsm *nfa) fsm_print_fsm(stderr, nfa); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif + TIME(&overall_pre); issp = interned_state_set_pool_alloc(nfa->opt->alloc); if (issp == NULL) { @@ -104,6 +112,17 @@ fsm_determinise(struct fsm *nfa) ac_env.fsm = nfa; ac_env.issp = issp; +#if LOG_DETERMINISATION_STATS + fprintf(stderr, "%s: determinising FSM with %d states\n", __func__, fsm_countstates(nfa)); +#endif + + INIT_TIMERS_NAMED(iss); + size_t iss_accum = 0; + size_t iss_calls = 0; + size_t stack_pushes = 0; + size_t inner_steps = 0; + + TIME(&pre); do { size_t o_i; @@ -114,18 +133,25 @@ fsm_determinise(struct fsm *nfa) assert(curr != NULL); + TIME(&iss_pre); if (!analyze_closures__pairwise_grouping(&ac_env, curr->iss)) { goto cleanup; } + TIME(&iss_post); + DIFF_MSEC("det_iss", iss_pre, iss_post, &iss_accum); + (void)iss_accum; + iss_calls++; if (!edge_set_advise_growth(&curr->edges, nfa->opt->alloc, ac_env.output_count)) { goto cleanup; } + /* each output is an outgoing (label set) -> interned_state_set pair */ for (o_i = 0; o_i < ac_env.output_count; o_i++) { struct mapping *m; struct ac_output *output = &ac_env.outputs[o_i]; interned_state_set_id iss = output->iss; + inner_steps++; #if LOG_DETERMINISE_CLOSURES fprintf(stderr, "fsm_determinise: output %zu/%zu: cur (dfa %zu) label [", @@ -157,6 +183,7 @@ fsm_determinise(struct fsm *nfa) if (!stack_push(stack, m)) { goto cleanup; } + stack_pushes++; } #if LOG_SYMBOL_CLOSURE @@ -174,6 +201,13 @@ fsm_determinise(struct fsm *nfa) /* All elements in sclosures[] are interned, so they will be freed later. */ } while ((curr = stack_pop(stack))); + TIME(&post); + DIFF_MSEC("det_stack_loop", pre, post, NULL); + + if (LOG_DETERMINISATION_COUNTERS) { + fprintf(stderr, "%s: iss_accum total %zu (%zu calls, %g usec avg.), %zu stack pushes, %zu iterations, %zu inner_steps\n", + __func__, iss_accum, iss_calls, iss_accum / (1.0 * iss_calls), stack_pushes, iss_calls, inner_steps); + } { struct map_iter it; @@ -185,6 +219,13 @@ fsm_determinise(struct fsm *nfa) goto cleanup; } + TIME(&pre); + if (!fsm_capture_copy_programs(nfa, dfa)) { + goto cleanup; + } + TIME(&post); + DIFF_MSEC("det_copy_captures", pre, post, NULL); + #if DUMP_MAPPING { fprintf(stderr, "#### fsm_determinise: mapping\n"); @@ -192,10 +233,10 @@ fsm_determinise(struct fsm *nfa) /* build reverse mappings table: for every NFA state X, if X is part * of the new DFA state Y, then add Y to a list for X */ for (m = map_first(&map, &it); m != NULL; m = map_next(&it)) { - struct state_iter si; interned_state_set_id iss_id = m->iss; + struct state_iter si; fsm_state_t state; - struct state_set *ss = interned_state_set_get_state_set(ac_env.issp, iss_id); + struct state_set *ss = interned_state_set_get_state_set(issp, iss_id); fprintf(stderr, "%zu:", m->dfastate); for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { @@ -238,24 +279,41 @@ fsm_determinise(struct fsm *nfa) fsm_setend(dfa, m->dfastate, 1); /* - * Carry through end IDs, if present. This isn't anything to do - * with the DFA conversion; it's meaningful only to the caller. + * Copy over metadata associated with end + * states, if present. This isn't anything to do + * with the DFA conversion; it's meaningful only + * to the caller. * * The closure may contain non-end states, but at least one state is * known to have been an end state. */ - if (!fsm_endid_carry(nfa, ss, dfa, m->dfastate)) { + if (!remap_end_metadata(nfa, ss, dfa, m->dfastate)) { goto cleanup; } } + TIME(&post); + DIFF_MSEC("det_map_loop", pre, post, NULL); - if (!remap_capture_actions(&map, issp, dfa, nfa)) { - goto cleanup; - } + fsm_capture_integrity_check(dfa); fsm_move(nfa, dfa); } +#if LOG_DETERMINISE_CAPTURES + fprintf(stderr, "# post_determinise\n"); + fsm_print_fsm(stderr, nfa); + fsm_capture_dump(stderr, "#### post_determinise", nfa); +#endif + + TIME(&overall_post); + DIFF_MSEC("det_overall", overall_pre, overall_post, NULL); + +#if LOG_DETERMINISATION_STATS + fprintf(stderr, "%s: created DFA with %d states\n", __func__, fsm_countstates(nfa)); + fprintf(stderr, "%s: analyze_closures_env.analyze_usec: %zu\n", + __func__, ac_env.analyze_usec); +#endif + #if EXPENSIVE_CHECKS assert(fsm_all(nfa, fsm_isdfa)); #endif @@ -311,85 +369,6 @@ fsm_determinise(struct fsm *nfa) return res; } -/* Add DFA_state to the list for NFA_state. */ -static int -add_reverse_mapping(const struct fsm_alloc *alloc, - struct reverse_mapping *reverse_mappings, - fsm_state_t dfastate, fsm_state_t nfa_state) -{ - struct reverse_mapping *rm = &reverse_mappings[nfa_state]; - if (rm->count == rm->ceil) { - const unsigned nceil = (rm->ceil ? 2*rm->ceil : 2); - fsm_state_t *nlist = f_realloc(alloc, - rm->list, nceil * sizeof(rm->list)); - if (nlist == NULL) { - return 0; - } - rm->list = nlist; - rm->ceil = nceil; - } - - rm->list[rm->count] = dfastate; - rm->count++; - return 1; -} - -static int -det_copy_capture_actions_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) -{ - struct reverse_mapping *rm_s; - size_t s_i, t_i; - struct det_copy_capture_actions_env *env = opaque; - assert(env->tag == 'D'); - -#if LOG_DETERMINISE_CAPTURES - fprintf(stderr, "det_copy_capture_actions_cb: state %u, type %s, ID %u, TO %d\n", - state, fsm_capture_action_type_name[type], - capture_id, to); -#endif - - rm_s = &env->reverse_mappings[state]; - - for (s_i = 0; s_i < rm_s->count; s_i++) { - const fsm_state_t s = rm_s->list[s_i]; - - if (to == CAPTURE_NO_STATE) { - if (!fsm_capture_add_action(env->dst, - s, type, capture_id, CAPTURE_NO_STATE)) { - env->ok = 0; - return 0; - } - } else { - struct reverse_mapping *rm_t = &env->reverse_mappings[to]; - for (t_i = 0; t_i < rm_t->count; t_i++) { - const fsm_state_t t = rm_t->list[t_i]; - - if (!fsm_capture_add_action(env->dst, - s, type, capture_id, t)) { - env->ok = 0; - return 0; - } - } - } - } - - return 1; -} - -static int -det_copy_capture_actions(struct reverse_mapping *reverse_mappings, - struct fsm *dst, struct fsm *src) -{ - struct det_copy_capture_actions_env env = { 'D', NULL, NULL, 1 }; - env.dst = dst; - env.reverse_mappings = reverse_mappings; - - fsm_capture_action_iter(src, det_copy_capture_actions_cb, &env); - return env.ok; -} - SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static uint64_t hash_iss(interned_state_set_id iss) @@ -636,83 +615,6 @@ stack_pop(struct mappingstack *stack) return item; } -static int -remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, - struct fsm *dst_dfa, struct fsm *src_nfa) -{ - struct map_iter it; - struct state_iter si; - struct mapping *m; - struct reverse_mapping *reverse_mappings; - fsm_state_t state; - const size_t capture_count = fsm_countcaptures(src_nfa); - size_t i, j; - int res = 0; - - if (capture_count == 0) { - return 1; - } - - /* This is not 1 to 1 -- if state X is now represented by multiple - * states Y in the DFA, and state X has action(s) when transitioning - * to state Z, this needs to be added on every Y, for every state - * representing Z in the DFA. - * - * We could probably filter this somehow, at the very least by - * checking reachability from every X, but the actual path - * handling later will also check reachability. */ - reverse_mappings = f_calloc(dst_dfa->opt->alloc, src_nfa->statecount, sizeof(reverse_mappings[0])); - if (reverse_mappings == NULL) { - return 0; - } - - /* build reverse mappings table: for every NFA state X, if X is part - * of the new DFA state Y, then add Y to a list for X */ - for (m = map_first(map, &it); m != NULL; m = map_next(&it)) { - struct state_set *ss; - interned_state_set_id iss_id = m->iss; - assert(m->dfastate < dst_dfa->statecount); - ss = interned_state_set_get_state_set(issp, iss_id); - - for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { - if (!add_reverse_mapping(dst_dfa->opt->alloc, - reverse_mappings, - m->dfastate, state)) { - goto cleanup; - } - } - } - -#if LOG_DETERMINISE_CAPTURES - fprintf(stderr, "#### reverse mapping for %zu states\n", src_nfa->statecount); - for (i = 0; i < src_nfa->statecount; i++) { - struct reverse_mapping *rm = &reverse_mappings[i]; - fprintf(stderr, "%lu:", i); - for (j = 0; j < rm->count; j++) { - fprintf(stderr, " %u", rm->list[j]); - } - fprintf(stderr, "\n"); - } -#else - (void)j; -#endif - - if (!det_copy_capture_actions(reverse_mappings, dst_dfa, src_nfa)) { - goto cleanup; - } - - res = 1; -cleanup: - for (i = 0; i < src_nfa->statecount; i++) { - if (reverse_mappings[i].list != NULL) { - f_free(dst_dfa->opt->alloc, reverse_mappings[i].list); - } - } - f_free(dst_dfa->opt->alloc, reverse_mappings); - - return res; -} - static int group_labels_overlap(const struct ac_group *a, const struct ac_group *b) { @@ -730,6 +632,25 @@ group_labels_overlap(const struct ac_group *a, const struct ac_group *b) return 0; } +static int +remap_end_metadata(const struct fsm *src_fsm, const struct state_set *src_set, + struct fsm *dst_fsm, fsm_state_t dst_state) +{ + if (!fsm_endid_carry(src_fsm, src_set, dst_fsm, dst_state)) { + return 0; + } + + if (!fsm_capture_copy_active_for_ends(src_fsm, src_set, dst_fsm, dst_state)) { + return 0; + } + + if (!fsm_capture_copy_program_end_state_associations(src_fsm, src_set, dst_fsm, dst_state)) { + return 0; + } + + return 1; +} + static void intersect_with(uint64_t *a, const uint64_t *b) { diff --git a/src/libfsm/determinise_internal.h b/src/libfsm/determinise_internal.h index 8fe35fcd8..856f8baf8 100644 --- a/src/libfsm/determinise_internal.h +++ b/src/libfsm/determinise_internal.h @@ -75,19 +75,6 @@ struct map_iter { size_t i; }; -struct reverse_mapping { - unsigned count; - unsigned ceil; - fsm_state_t *list; -}; - -struct det_copy_capture_actions_env { - char tag; - struct fsm *dst; - struct reverse_mapping *reverse_mappings; - int ok; -}; - #define MAPPINGSTACK_DEF_CEIL 16 struct mappingstack { const struct fsm_alloc *alloc; @@ -289,6 +276,10 @@ analyze_closures__grow_dst(struct analyze_closures_env *env); static int analyze_closures__grow_outputs(struct analyze_closures_env *env); +static int +remap_end_metadata(const struct fsm *src_fsm, const struct state_set *src_set, + struct fsm *dst_fsm, fsm_state_t dst_state); + static int map_add(struct map *map, fsm_state_t dfastate, interned_state_set_id iss, struct mapping **new_mapping); @@ -306,22 +297,9 @@ map_first(struct map *map, struct map_iter *iter); static struct mapping * map_next(struct map_iter *iter); -static int -add_reverse_mapping(const struct fsm_alloc *alloc, - struct reverse_mapping *reverse_mappings, - fsm_state_t dfastate, fsm_state_t nfa_state); - -static int -det_copy_capture_actions(struct reverse_mapping *reverse_mappings, - struct fsm *dst, struct fsm *src); - static int grow_map(struct map *map); -static int -remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, - struct fsm *dst_dfa, struct fsm *src_nfa); - static struct mappingstack * stack_init(const struct fsm_alloc *alloc); diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 94cf8f259..1684facc8 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -4,7 +4,44 @@ * See LICENCE for the full copyright terms. */ -#include "endids_internal.h" +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "endids.h" + +#define BUCKET_NO_STATE ((fsm_state_t)-1) +#define DEF_BUCKET_COUNT 4 +#define DEF_BUCKET_ID_COUNT 16 + +struct endid_info { + /* Add-only hash table, with a state ID and an associated + * non-empty ordered array of unique end IDs. The state is the + * key. Grows when the buckets are more than half full. */ + unsigned bucket_count; + unsigned buckets_used; + + struct endid_info_bucket { + fsm_state_t state; + struct end_info_ids { + unsigned count; + unsigned ceil; + fsm_end_id_t ids[1]; + } *ids; + } *buckets; +}; #define LOG_ENDIDS 0 @@ -48,6 +85,14 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) return 1; } +int +fsm_setendid_state(struct fsm *fsm, fsm_state_t s, fsm_end_id_t id) +{ + enum fsm_endid_set_res sres; + sres = fsm_endid_set(fsm, s, id); + return sres != FSM_ENDID_SET_ERROR_ALLOC_FAIL; +} + enum fsm_getendids_res fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, size_t id_buf_count, fsm_end_id_t *id_buf, @@ -420,6 +465,13 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, return FSM_GETENDIDS_NOT_FOUND; } +void +fsm_iterendids(const struct fsm *fsm, fsm_state_t state, + fsm_iterendids_cb *cb, void *opaque) +{ + fsm_endid_iter_state(fsm, state, cb, opaque); +} + struct carry_env { char tag; struct fsm *dst; @@ -428,13 +480,16 @@ struct carry_env { }; static int -carry_iter_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +carry_iter_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, fsm_end_id_t id, void *opaque) { enum fsm_endid_set_res sres; struct carry_env *env = opaque; assert(env->tag == 'C'); + (void)fsm; (void)state; + (void)nth; sres = fsm_endid_set(env->dst, env->dst_state, id); if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { @@ -489,6 +544,72 @@ fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, return 1; } +/* Make a new hash table, copying over converted entries and/or discarding. */ +int +fsm_endid_compact(struct fsm *fsm, + const fsm_state_t *mapping, size_t mapping_count) +{ + struct endid_info *info = fsm->endid_info; + const size_t ocount = info->bucket_count; + const size_t ncount = ocount; /* does not need to grow */ + struct endid_info_bucket *obuckets = info->buckets; + struct endid_info_bucket *nbuckets = f_malloc(fsm->opt->alloc, + ncount * sizeof(nbuckets[0])); + const size_t nmask = ncount - 1; + size_t ob_i, nb_i; + size_t moved = 0; + +#if LOG_ENDIDS > 3 + fprintf(stderr, "fsm_endid_compact: rehashing mapped entries\n"); +#endif + + if (nbuckets == NULL) { + return 0; + } + + for (nb_i = 0; nb_i < ncount; nb_i++) { /* clear table */ + nbuckets[nb_i].state = BUCKET_NO_STATE; + } + + for (ob_i = 0; ob_i < ocount; ob_i++) { + const struct endid_info_bucket *ob = &obuckets[ob_i]; + uint64_t hash; + fsm_state_t nstate; + + if (ob->state == BUCKET_NO_STATE) { + continue; + } + + assert(ob->state < mapping_count); + nstate = mapping[ob->state]; + + if (nstate == FSM_STATE_REMAP_NO_STATE) { + info->buckets_used--; /* discarded */ + continue; + } + + hash = hash_id(nstate); + for (nb_i = 0; nb_i < ncount; nb_i++) { + struct endid_info_bucket *nb = &nbuckets[(hash + nb_i) & nmask]; + if (nb->state == BUCKET_NO_STATE) { + nb->state = nstate; + nb->ids = ob->ids; + moved++; + break; + } else { + continue; /* collision */ + } + } + } + + assert(moved == info->buckets_used); + + f_free(fsm->opt->alloc, info->buckets); + info->bucket_count = ncount; + info->buckets = nbuckets; + return 1; +} + void fsm_endid_iter(const struct fsm *fsm, fsm_endid_iter_cb *cb, void *opaque) @@ -517,7 +638,7 @@ fsm_endid_iter(const struct fsm *fsm, count = b->ids->count; for (id_i = 0; id_i < count; id_i++) { - if (!cb(b->state, b->ids->ids[id_i], opaque)) { + if (!cb(fsm, b->state, id_i, b->ids->ids[id_i], opaque)) { break; } @@ -580,7 +701,7 @@ fsm_endid_iter_state(const struct fsm *fsm, fsm_state_t state, fprintf(stderr, "fsm_endid_iter_state[%d], ids[%ld] -> %d\n", b->state, id_i, b->ids->ids[id_i]); #endif - if (!cb(b->state, b->ids->ids[id_i], opaque)) { + if (!cb(fsm, b->state, id_i, b->ids->ids[id_i], opaque)) { return; } id_i++; @@ -602,10 +723,13 @@ struct dump_env { }; static int -dump_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) +dump_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque) { struct dump_env *env = opaque; - fprintf(env->f, "state[%u]: %u\n", state, id); + fprintf(env->f, "endids: state[%u]: %u\n", state, id); + (void)fsm; + (void)nth; return 1; } diff --git a/src/libfsm/endids.h b/src/libfsm/endids.h index 5389bd8b0..0982017ae 100644 --- a/src/libfsm/endids.h +++ b/src/libfsm/endids.h @@ -32,10 +32,15 @@ int fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, struct fsm *dst_fsm, fsm_state_t dst_state); +int +fsm_endid_compact(struct fsm *fsm, + const fsm_state_t *mapping, size_t mapping_count); + /* Callback when iterating over the endids. * Return 0 to halt, or non-zero to continue. */ typedef int -fsm_endid_iter_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque); +fsm_endid_iter_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque); void fsm_endid_iter(const struct fsm *fsm, diff --git a/src/libfsm/endids_internal.h b/src/libfsm/endids_internal.h deleted file mode 100644 index 27450af3b..000000000 --- a/src/libfsm/endids_internal.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef ENDIDS_INTERNAL_H -#define ENDIDS_INTERNAL_H - -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include "internal.h" -#include "endids.h" - -#define BUCKET_NO_STATE ((fsm_state_t)-1) -#define DEF_BUCKET_COUNT 4 -#define DEF_BUCKET_ID_COUNT 16 - -struct endid_info { - /* Add-only hash table, with a state ID and an associated - * non-empty ordered array of unique end IDs. The state is the - * key. Grows when the buckets are more than half full. */ - unsigned bucket_count; - unsigned buckets_used; - - struct endid_info_bucket { - fsm_state_t state; - struct end_info_ids { - unsigned count; - unsigned ceil; - fsm_end_id_t ids[1]; - } *ids; - } *buckets; -}; - -#endif diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 06da7739e..e63ae50ce 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -21,50 +22,77 @@ #include "endids.h" #define DUMP_EPSILON_CLOSURES 0 -#define DEF_PENDING_CAPTURE_ACTIONS_CEIL 2 #define LOG_RM_EPSILONS_CAPTURES 0 -#define DEF_CARRY_ENDIDS_COUNT 2 +#define LOG_COPYING 0 +#define LOG_RESULT 0 -struct remap_env { - char tag; - const struct fsm_alloc *alloc; - struct state_set **rmap; - int ok; - - size_t count; - size_t ceil; - struct remap_action { - fsm_state_t state; - enum capture_action_type type; - unsigned capture_id; - fsm_state_t to; - } *actions; -}; +/* #define DEF_CARRY_ENDIDS_COUNT 2 */ +/* #define DEF_CARRY_CAPTUREIDS_COUNT 2 */ -static int -remap_capture_actions(struct fsm *nfa, struct state_set **eclosures); +#if LOG_RESULT +#include +#endif -static int -remap_capture_action_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque); +#define DEF_END_METADATA_ENDIDS_CEIL 4 +#define DEF_END_METADATA_CAPTUREIDS_CEIL 4 +#define DEF_END_METADATA_PROGRAMIDS_CEIL 4 +struct carry_end_metadata_env { + struct fsm *fsm; + const struct fsm_alloc *alloc; + + struct { + size_t ceil; + fsm_end_id_t *ids; + } end; + struct { + int ok; + size_t count; + size_t ceil; + unsigned *ids; + } capture; + struct { + int ok; + size_t count; + size_t ceil; + uint32_t *ids; + } program; +}; static int -carry_endids(struct fsm *fsm, struct state_set *states, - fsm_state_t s); +carry_end_metadata(struct carry_end_metadata_env *env, + fsm_state_t end_state, fsm_state_t dst_state); int fsm_remove_epsilons(struct fsm *nfa) { +#if LOG_RESULT + fprintf(stderr, "==== before\n"); + fsm_print_fsm(stderr, nfa); + fsm_capture_dump(stderr, "#### before_remove_epsilons", nfa); + fprintf(stderr, "====\n"); +#endif + const size_t state_count = fsm_countstates(nfa); int res = 0; struct state_set **eclosures = NULL; - fsm_state_t s; + fsm_state_t s, start_id; + const struct fsm_alloc *alloc = nfa->opt->alloc; INIT_TIMERS(); + struct carry_end_metadata_env em_env = { 0 }; + em_env.fsm = nfa; + em_env.alloc = alloc; + assert(nfa != NULL); + if (!fsm_getstart(nfa, &start_id)) { + goto cleanup; + } + + /* TODO: This could successfully exit early if none of the + * states have epsilon edges. */ + TIME(&pre); eclosures = fsm_epsilon_closure(nfa); TIME(&post); @@ -107,20 +135,14 @@ fsm_remove_epsilons(struct fsm *nfa) * end states. * * Similarly, any end state metadata on states - * in its epsilon-closure is copied to it. - * - * Capture actions are copied in a later pass. */ + * in its epsilon-closure is copied to it. */ if (fsm_isend(nfa, es_id)) { #if LOG_COPYING fprintf(stderr, "remove_epsilons: setting end on %d (due to %d)\n", s, es_id); #endif fsm_setend(nfa, s, 1); - /* - * Carry through end IDs, if present. This isn't anything to do - * with the NFA conversion; it's meaningful only to the caller. - */ - if (!carry_endids(nfa, eclosures[s], s)) { + if (!carry_end_metadata(&em_env, es_id, s)) { goto cleanup; } } @@ -150,14 +172,7 @@ fsm_remove_epsilons(struct fsm *nfa) state->epsilons = NULL; } -#if LOG_RESULT - fprintf(stderr, "=== %s: about to update capture actions\n", __func__); - fsm_print_fsm(stderr, nfa); -#endif - - if (!remap_capture_actions(nfa, eclosures)) { - goto cleanup; - } + fsm_capture_integrity_check(nfa); #if LOG_RESULT fsm_print_fsm(stderr, nfa); @@ -169,253 +184,136 @@ fsm_remove_epsilons(struct fsm *nfa) if (eclosures != NULL) { fsm_closure_free(eclosures, state_count); } - - return res; -} - -static int -remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) -{ - int res = 0; - fsm_state_t s, i; - struct state_set **rmap; - struct state_iter si; - fsm_state_t si_s; - struct remap_env env = { 'R', NULL, NULL, 1, 0, 0, NULL }; - env.alloc = nfa->opt->alloc; - - /* build a reverse mapping */ - rmap = f_calloc(nfa->opt->alloc, nfa->statecount, sizeof(rmap[0])); - if (rmap == NULL) { - goto cleanup; - } - - for (s = 0; s < nfa->statecount; s++) { - if (eclosures[s] == NULL) { continue; } - for (state_set_reset(eclosures[s], &si); state_set_next(&si, &si_s); ) { - if (si_s == s) { - continue; /* ignore identical states */ - } -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, "remap_capture_actions: %u <- %u\n", - s, si_s); -#endif - if (!state_set_add(&rmap[si_s], nfa->opt->alloc, s)) { - goto cleanup; - } - } + if (em_env.end.ids != NULL) { + f_free(alloc, em_env.end.ids); } - env.rmap = rmap; - - /* Iterate over the current set of actions with the reverse - * mapping (containing only states which will be skipped, - * collecting info about every new capture action that will need - * to be added. - * - * It can't be added during the iteration, because that would - * modify the hash table as it's being iterated over. */ - fsm_capture_action_iter(nfa, remap_capture_action_cb, &env); - - /* Now that we're done iterating, add those actions. */ - for (i = 0; i < env.count; i++) { - const struct remap_action *a = &env.actions[i]; - if (!fsm_capture_add_action(nfa, a->state, a->type, - a->capture_id, a->to)) { - goto cleanup; - } + if (em_env.program.ids != NULL) { + f_free(alloc, em_env.program.ids); } - - res = 1; - -cleanup: - if (env.actions != NULL) { - f_free(nfa->opt->alloc, env.actions); + if (em_env.capture.ids != NULL) { + f_free(alloc, em_env.capture.ids); } - if (rmap != NULL) { - for (i = 0; i < nfa->statecount; i++) { - state_set_free(rmap[i]); - } - f_free(nfa->opt->alloc, rmap); - } return res; - } static int -add_pending_capture_action(struct remap_env *env, - fsm_state_t state, enum capture_action_type type, - unsigned capture_id, fsm_state_t to) +collect_captureid_cb(fsm_state_t state, unsigned id, void *opaque) { - struct remap_action *a; - if (env->count == env->ceil) { - struct remap_action *nactions; - const size_t nceil = (env->ceil == 0 - ? DEF_PENDING_CAPTURE_ACTIONS_CEIL : 2*env->ceil); - assert(nceil > 0); - nactions = f_realloc(env->alloc, - env->actions, - nceil * sizeof(nactions[0])); - if (nactions == NULL) { - return 0; - } - - env->ceil = nceil; - env->actions = nactions; - } + struct carry_end_metadata_env *env = opaque; + (void)state; - a = &env->actions[env->count]; -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, "add_pending_capture_action: state %d, type %s, capture_id %u, to %d\n", - state, fsm_capture_action_type_name[type], capture_id, to); -#endif + if (env->capture.count == env->capture.ceil) { + const size_t nceil = (env->capture.ceil == 0) + ? DEF_END_METADATA_CAPTUREIDS_CEIL + : 2 * env->capture.ceil; + unsigned *nids; + assert(nceil > env->capture.ceil); + nids = f_realloc(env->alloc, env->capture.ids, + nceil * sizeof(env->capture.ids[0])); + if (nids == NULL) { + env->capture.ok = 0; + return 0; + } + env->capture.ceil = nceil; + env->capture.ids = nids; + } - a->state = state; - a->type = type; - a->capture_id = capture_id; - a->to = to; - env->count++; + env->capture.ids[env->capture.count] = id; + env->capture.count++; return 1; } static int -remap_capture_action_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +collect_progid_cb(fsm_state_t state, unsigned id, void *opaque) { - struct state_iter si; - fsm_state_t si_s; - struct remap_env *env = opaque; - assert(env->tag == 'R'); - -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, "remap_capture_action_cb: state %d, type %s, capture_id %u, to %d\n", - state, fsm_capture_action_type_name[type], capture_id, to); -#endif - - for (state_set_reset(env->rmap[state], &si); state_set_next(&si, &si_s); ) { - struct state_iter si_to; - fsm_state_t si_tos; - -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, " -- rcac: state %d -> %d\n", state, si_s); -#endif - - if (!add_pending_capture_action(env, si_s, type, capture_id, to)) { - goto fail; - } - - if (to == CAPTURE_NO_STATE) { - continue; - } - - for (state_set_reset(env->rmap[to], &si_to); state_set_next(&si, &si_tos); ) { -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, " -- rcac: to %d -> %d\n", to, si_tos); -#endif - - if (!add_pending_capture_action(env, si_tos, type, capture_id, to)) { - goto fail; - } - - } - } + struct carry_end_metadata_env *env = opaque; + uint32_t prog_id = (uint32_t)id; + (void)state; + + if (env->program.count == env->program.ceil) { + const size_t nceil = (env->program.ceil == 0) + ? DEF_END_METADATA_PROGRAMIDS_CEIL + : 2 * env->program.ceil; + unsigned *nids; + assert(nceil > env->program.ceil); + nids = f_realloc(env->alloc, env->program.ids, + nceil * sizeof(env->program.ids[0])); + if (nids == NULL) { + env->program.ok = 0; + return 0; + } + env->program.ceil = nceil; + env->program.ids = nids; + } + env->program.ids[env->program.count] = prog_id; + env->program.count++; return 1; - -fail: - env->ok = 0; - return 0; } -struct collect_env { - char tag; - const struct fsm_alloc *alloc; - size_t count; - size_t ceil; - fsm_end_id_t *ids; - int ok; -}; - +/* Because we're modifying the FSM in place, we can't iterate and add + * new entries -- it could lead to the underlying hash table resizing. + * Instead, collect, then add in a second pass. */ static int -collect_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +carry_end_metadata(struct carry_end_metadata_env *env, + fsm_state_t end_state, fsm_state_t dst_state) { - struct collect_env *env = opaque; - assert(env->tag == 'E'); - - (void)state; - - if (env->count == env->ceil) { - const size_t nceil = 2 * env->ceil; - fsm_end_id_t *nids; - assert(nceil > env->ceil); - nids = f_realloc(env->alloc, env->ids, - nceil * sizeof(*env->ids)); - if (nids == NULL) { - env->ok = 0; - return 0; + size_t i; + const size_t id_count = fsm_getendidcount(env->fsm, end_state); + if (id_count > 0) { /* copy end IDs */ + enum fsm_getendids_res id_res; + size_t written; + if (id_count > env->end.ceil) { /* grow buffer */ + const size_t nceil = (env->end.ceil == 0) + ? DEF_END_METADATA_ENDIDS_CEIL + : 2*env->end.ceil; + assert(nceil > 0); + env->end.ids = f_realloc(env->alloc, + env->end.ids, nceil * sizeof(env->end.ids[0])); + if (env->end.ids == NULL) { + return 0; + } } - env->ceil = nceil; - env->ids = nids; - } - env->ids[env->count] = id; - env->count++; + id_res = fsm_getendids(env->fsm, end_state, + id_count, env->end.ids, &written); + assert(id_res != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); - return 1; -} - -/* fsm_remove_epsilons can't use fsm_endid_carry directly, because the src - * and dst FSMs are the same -- that would lead to adding entries to a - * hash table, possibly causing it to resize, while iterating over it. - * - * Instead, collect entries that need to be added (if not already - * present), and then add them in a second pass. */ -static int -carry_endids(struct fsm *fsm, struct state_set *states, - fsm_state_t dst_state) -{ - struct state_iter it; - fsm_state_t s; - size_t i; + for (i = 0; i < id_count; i++) { +#if LOG_COPYING + fprintf(stderr, "carry_end_metadata: setting end ID %u on %d (due to %d)\n", + env->end.ids[i], dst_state, end_state); +#endif + if (!fsm_setendid_state(env->fsm, dst_state, env->end.ids[i])) { + return 0; + } + } + } - struct collect_env env; - env.tag = 'E'; /* for fsm_remove_epsilons */ - env.alloc = fsm->opt->alloc; - env.count = 0; - env.ceil = DEF_CARRY_ENDIDS_COUNT; - env.ids = f_malloc(fsm->opt->alloc, - env.ceil * sizeof(*env.ids)); - if (env.ids == NULL) { + env->capture.ok = 1; + env->capture.count = 0; + fsm_capture_iter_active_for_end_state(env->fsm, end_state, + collect_captureid_cb, env); + if (!env->capture.ok) { return 0; } - env.ok = 1; - - /* collect from states */ - for (state_set_reset(states, &it); state_set_next(&it, &s); ) { - if (!fsm_isend(fsm, s)) { - continue; - } - - fsm_endid_iter_state(fsm, s, collect_cb, &env); - if (!env.ok) { - goto cleanup; + for (i = 0; i < env->capture.count; i++) { + if (!fsm_capture_set_active_for_end(env->fsm, + env->capture.ids[i], dst_state)) { + return 0; } } - /* add them */ - for (i = 0; i < env.count; i++) { - enum fsm_endid_set_res sres; - sres = fsm_endid_set(fsm, dst_state, env.ids[i]); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - env.ok = 0; - goto cleanup; + env->program.count = 0; + fsm_capture_iter_program_ids_for_end_state(env->fsm, end_state, + collect_progid_cb, env); + for (i = 0; i < env->program.count; i++) { + if (!fsm_capture_associate_program_with_end_state(env->fsm, + env->program.ids[i], dst_state)) { + return 0; } } -cleanup: - f_free(fsm->opt->alloc, env.ids); - - return env.ok; + return 1; } - diff --git a/src/libfsm/exec.c b/src/libfsm/exec.c index 9f7b21802..47d27a50e 100644 --- a/src/libfsm/exec.c +++ b/src/libfsm/exec.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -25,7 +26,6 @@ static int transition(const struct fsm *fsm, fsm_state_t state, int c, - size_t offset, struct fsm_capture *captures, fsm_state_t *next) { assert(state < fsm->statecount); @@ -35,31 +35,72 @@ transition(const struct fsm *fsm, fsm_state_t state, int c, return 0; } - if (captures != NULL && fsm_capture_has_capture_actions(fsm, state)) { - fsm_capture_update_captures(fsm, state, *next, - offset, captures); - } - return 1; } int fsm_exec(const struct fsm *fsm, - int (*fsm_getc)(void *opaque), void *opaque, - fsm_state_t *end, struct fsm_capture *captures) + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end) { fsm_state_t state; int c; size_t offset = 0; - unsigned i; - size_t capture_count; assert(fsm != NULL); assert(fsm_getc != NULL); assert(end != NULL); - capture_count = fsm_countcaptures(fsm); + /* TODO: check prerequisites; that it has literal edges, DFA, etc */ + + /* TODO: pass struct of callbacks to call during each event; transitions etc */ + + if (!fsm_all(fsm, fsm_isdfa)) { + errno = EINVAL; + return -1; + } + + if (!fsm_getstart(fsm, &state)) { + errno = EINVAL; + return -1; + } + +#if LOG_EXEC + fprintf(stderr, "fsm_exec: starting at %d\n", state); +#endif + + while (c = fsm_getc(opaque), c != EOF) { + if (!transition(fsm, state, c, &state)) { +#if LOG_EXEC + fprintf(stderr, "fsm_exec: edge not found\n"); +#endif + return 0; + } +#if LOG_EXEC + fprintf(stderr, "fsm_exec: @ %zu, input '%c', new state %u\n", + offset, c, state); +#endif + offset++; + } + + if (!fsm_isend(fsm, state)) { + return 0; + } + + *end = state; + return 1; +} + +int +fsm_exec_with_captures(const struct fsm *fsm, const unsigned char *input, + size_t input_length, fsm_state_t *end, + struct fsm_capture *captures, size_t capture_buf_length) +{ + fsm_state_t state; + size_t offset = 0; + + assert(fsm != NULL); + assert(end != NULL); /* TODO: check prerequisites; that it has literal edges, DFA, etc */ /* TODO: pass struct of callbacks to call during each event; transitions etc */ @@ -74,17 +115,26 @@ fsm_exec(const struct fsm *fsm, return -1; } - for (i = 0; i < capture_count; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; + if (captures != NULL) { + const size_t capture_ceil = fsm_capture_ceiling(fsm); + if (capture_buf_length < capture_ceil) { + errno = EINVAL; + return -1; + } + + for (size_t i = 0; i < capture_ceil; i++) { + captures[i].pos[0] = FSM_CAPTURE_NO_POS; + captures[i].pos[1] = FSM_CAPTURE_NO_POS; + } } #if LOG_EXEC fprintf(stderr, "fsm_exec: starting at %d\n", state); #endif - while (c = fsm_getc(opaque), c != EOF) { - if (!transition(fsm, state, c, offset, captures, &state)) { + while (offset < input_length) { + const unsigned char c = input[offset]; + if (!transition(fsm, state, c, &state)) { #if LOG_EXEC fprintf(stderr, "fsm_exec: edge not found\n"); #endif @@ -102,15 +152,15 @@ fsm_exec(const struct fsm *fsm, return 0; } - /* Check for capture actions on end state */ - if (captures != NULL && fsm_capture_has_capture_actions(fsm, state)) { - fsm_capture_update_captures(fsm, state, NEXT_STATE_END, - offset, captures); + /* Resolve captures associated with the end state. */ + if (captures != NULL) { + if (!fsm_capture_resolve_during_exec(fsm, state, + input, offset, captures, capture_buf_length)) { + assert(errno != 0); + return -1; + } } - fsm_capture_finalize_captures(fsm, capture_count, captures); - *end = state; return 1; } - diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 1dde1fc96..272f2cc88 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -124,10 +124,6 @@ struct fsm_edge { struct fsm_state { unsigned int end:1; - /* If 0, then this state has no need for checking - * the fsm->capture_info struct. */ - unsigned int has_capture_actions:1; - /* meaningful within one particular transformation only */ unsigned int visited:1; diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index f9f6bf003..bac177ff2 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -106,6 +106,7 @@ fsm_shortest fsm_example fsm_exec +fsm_exec_with_captures # exec fsm_fgetc @@ -127,11 +128,19 @@ fsm_vm_match_buffer fsm_vm_match_file # -fsm_countcaptures -fsm_capture_has_capture_actions +fsm_capture_ceiling fsm_capture_set_path fsm_capture_rebase_capture_id -fsm_capture_alloc +fsm_capture_alloc_capture_buffer +fsm_capture_free_capture_buffer fsm_capture_dump +fsm_capture_set_active_for_end +fsm_capture_add_program +fsm_capture_associate_program_with_end_state + +# capture_vm* +fsm_capvm_program_free +fsm_capvm_program_dump +fsm_capvm_program_exec fsm_minimise_test_oracle diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index b0df7ea71..ef8b13271 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -17,25 +17,27 @@ #include #include #include +#include #include "capture.h" +#include "capture_vm.h" #include "internal.h" #include "endids.h" #define LOG_MERGE_ENDIDS 0 - -struct copy_capture_env { - char tag; - struct fsm *dst; - int ok; -}; +#define LOG_COPY_CAPTURE_PROGRAMS 0 static int -copy_capture_actions(struct fsm *dst, struct fsm *src); +copy_end_metadata(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src); static int copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); +static int +copy_active_capture_ids(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src); + static struct fsm * merge(struct fsm *dst, struct fsm *src, fsm_state_t *base_dst, fsm_state_t *base_src, @@ -72,17 +74,12 @@ merge(struct fsm *dst, struct fsm *src, *base_dst = 0; *base_src = dst->statecount; *capture_base_dst = 0; - *capture_base_src = fsm_countcaptures(dst); + *capture_base_src = fsm_capture_ceiling(dst); for (i = 0; i < src->statecount; i++) { state_set_rebase(&src->states[i].epsilons, *base_src); edge_set_rebase(&src->states[i].edges, *base_src); } - - /* FIXME: instead of rebasing these here, they could - * also be updated in copy_capture_actions below. */ - fsm_capture_rebase_capture_id(src, *capture_base_src); - fsm_capture_rebase_capture_action_states(src, *base_src); } memcpy(dst->states + dst->statecount, src->states, @@ -90,22 +87,10 @@ merge(struct fsm *dst, struct fsm *src, dst->statecount += src->statecount; dst->endcount += src->endcount; - /* We need to explicitly copy over the capture actions and end - * ID info here because they're stored on the FSMs as a whole, - * rather than individual states; `memcpy`ing the states alone - * won't transfer them. - * - * They're stored separately because they are likely to only - * be on a small portion of the states, and adding two extra - * NULL pointers to `struct fsm_state` increases memory usage - * significantly. */ - - if (!copy_capture_actions(dst, src)) { - /* non-recoverable -- destructive operation */ - return NULL; - } - - if (!copy_end_ids(dst, src, *base_src)) { + /* We need to explicitly copy over end metadata here. They're + * stored separately because they are likely to only be on a + * small portion of the states. */ + if (!copy_end_metadata(dst, src, *base_src, *capture_base_src)) { /* non-recoverable -- destructive operation */ return NULL; } @@ -123,16 +108,91 @@ merge(struct fsm *dst, struct fsm *src, return dst; } +struct copy_capture_programs_env { + const struct fsm_alloc *alloc; + const struct fsm *src; + struct fsm *dst; + int ok; + fsm_state_t state_base_src; + unsigned capture_base_src; + +#define DEF_MAPPING_CEIL 1 + size_t mapping_used; + size_t mapping_ceil; + /* TODO: could cache last_map to check first if this becomes expensive */ + struct prog_mapping { + unsigned src_prog_id; + unsigned dst_prog_id; + } *mappings; +}; + static int -copy_capture_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, +copy_capture_programs_cb(fsm_state_t src_state, unsigned src_prog_id, void *opaque) { - struct copy_capture_env *env = opaque; - assert(env->tag == 'C'); + struct copy_capture_programs_env *env = opaque; + + const fsm_state_t dst_state = src_state + env->state_base_src; + assert(dst_state < fsm_countstates(env->dst)); + +#if LOG_COPY_CAPTURE_PROGRAMS + fprintf(stderr, "%s: src %p, dst %p, src_prog_id %u, src_state %d, dst_state %d, capture_base_src %u\n", + __func__, (void *)env->src, (void *)env->dst, + src_prog_id, src_state, dst_state, env->capture_base_src); +#endif + int found = 0; + uint32_t dst_prog_id; + + for (size_t i = 0; i < env->mapping_used; i++) { + const struct prog_mapping *m = &env->mappings[i]; + if (m->src_prog_id == src_prog_id) { + dst_prog_id = m->dst_prog_id; + found = 1; + } + } + + if (!found) { + if (env->mapping_used == env->mapping_ceil) { /* grow */ + const size_t nceil = 2*env->mapping_ceil; + struct prog_mapping *nmappings = f_realloc(env->alloc, + env->mappings, nceil * sizeof(nmappings[0])); + if (nmappings == NULL) { + env->ok = 0; + return 0; + } + + env->mapping_ceil = nceil; + env->mappings = nmappings; + } + + const struct capvm_program *p = fsm_capture_get_program_by_id(env->src, + src_prog_id); + assert(p != NULL); + + struct capvm_program *cp = capvm_program_copy(env->alloc, p); + if (cp == NULL) { + env->ok = 0; + return 0; + } + capvm_program_rebase(cp, env->capture_base_src); + + /* add program, if not present */ + if (!fsm_capture_add_program(env->dst, + cp, &dst_prog_id)) { + f_free(env->alloc, cp); + env->ok = 0; + return 0; + } - if (!fsm_capture_add_action(env->dst, state, type, - capture_id, to)) { + struct prog_mapping *m = &env->mappings[env->mapping_used]; + m->src_prog_id = src_prog_id; + m->dst_prog_id = dst_prog_id; + env->mapping_used++; + } + + /* associate with end states */ + if (!fsm_capture_associate_program_with_end_state(env->dst, + dst_prog_id, dst_state)) { env->ok = 0; return 0; } @@ -141,18 +201,55 @@ copy_capture_cb(fsm_state_t state, } static int -copy_capture_actions(struct fsm *dst, struct fsm *src) +copy_capture_programs(struct fsm *dst, const struct fsm *src, + fsm_state_t state_base_src, unsigned capture_base_src) { - struct copy_capture_env env; - env.tag = 'C'; - env.dst = dst; - env.ok = 1; + const struct fsm_alloc *alloc = src->opt->alloc; + struct prog_mapping *mappings = f_malloc(alloc, + DEF_MAPPING_CEIL * sizeof(mappings[0])); + if (mappings == NULL) { + return 0; + } - fsm_capture_action_iter(src, copy_capture_cb, &env); + struct copy_capture_programs_env env = { + .alloc = alloc, + .src = src, + .dst = dst, + .ok = 1, + .state_base_src = state_base_src, + .capture_base_src = capture_base_src, + .mapping_ceil = DEF_MAPPING_CEIL, + .mappings = mappings, + }; + fsm_capture_iter_program_ids_for_all_end_states(src, + copy_capture_programs_cb, &env); + + f_free(alloc, env.mappings); return env.ok; } +static int +copy_end_metadata(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src) +{ + /* TODO: inline */ + + if (!copy_end_ids(dst, src, base_src)) { + return 0; + } + + if (!copy_active_capture_ids(dst, src, base_src, capture_base_src)) { + return 0; + } + + if (!copy_capture_programs(dst, src, base_src, capture_base_src)) { + return 0; + } + + return 1; +} + struct copy_end_ids_env { char tag; struct fsm *dst; @@ -162,11 +259,14 @@ struct copy_end_ids_env { }; static int -copy_end_ids_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +copy_end_ids_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque) { enum fsm_endid_set_res sres; struct copy_end_ids_env *env = opaque; assert(env->tag == 'M'); + (void)fsm; + (void)nth; #if LOG_MERGE_ENDIDS > 1 fprintf(stderr, "merge[%d] <- %d\n", @@ -188,7 +288,6 @@ copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) struct copy_end_ids_env env; env.tag = 'M'; /* for Merge */ env.dst = dst; - env.src = src; env.base_src = base_src; env.ok = 1; @@ -196,6 +295,45 @@ copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) return env.ok; } +struct copy_active_capture_ids_env { + char tag; + struct fsm *dst; + fsm_state_t base_src; + unsigned capture_base_src; + int ok; +}; + +static int +copy_active_capture_ids_cb(fsm_state_t state, unsigned capture_id, void *opaque) +{ + struct copy_active_capture_ids_env *env = opaque; + assert(env->tag == 'A'); + + if (!fsm_capture_set_active_for_end(env->dst, + capture_id + env->capture_base_src, + state + env->base_src)) { + env->ok = 0; + return 0; + } + return 1; +} + +static int +copy_active_capture_ids(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src) +{ + struct copy_active_capture_ids_env env; + env.tag = 'A'; + env.dst = dst; + env.base_src = base_src; + env.capture_base_src = capture_base_src; + env.ok = 1; + + fsm_capture_iter_active_for_all_end_states(src, + copy_active_capture_ids_cb, &env); + return env.ok; +} + struct fsm * fsm_mergeab(struct fsm *a, struct fsm *b, fsm_state_t *base_b) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 7d95b78ad..6863395e2 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,54 @@ #include "minimise_internal.h" #include "minimise_test_oracle.h" +static int +label_sets_match(const uint64_t a[256/64], const uint64_t b[256/64]); + +static int +split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm); + +#if EXPENSIVE_CHECKS +#include + +static void +check_done_ec_offset(const struct min_env *env); + +static int +all_end_states_are_currently_together(const struct min_env *env); +#endif + +#define DEF_CAPTURE_ID_CEIL 4 +struct end_metadata { + struct end_metadata_end { + unsigned count; + fsm_end_id_t *ids; + } end; + + struct end_metadata_capture { + unsigned count; + unsigned ceil; + unsigned *ids; + } capture; + + struct end_metadata_program { + unsigned count; + unsigned ceil; + unsigned *ids; + } program; +}; + +static int +collect_end_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_end *e); + +static int +collect_capture_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_capture *c); + +static int +collect_capture_program_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_program *p); + int fsm_minimise(struct fsm *fsm) { @@ -51,19 +100,36 @@ fsm_minimise(struct fsm *fsm) assert(fsm != NULL); assert(fsm_all(fsm, fsm_isdfa)); +#if LOG_INIT > 1 + fprintf(stderr, "=== BEFORE TRIM, %d states\n", fsm_countstates(fsm)); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "#### pre_minimise", fsm); + fprintf(stderr, "=== BEFORE TRIM\n"); +#endif + /* The algorithm used below won't remove states without a path * to an end state, because it cannot prove they're * unnecessary, so they must be trimmed away first. */ + TIME(&pre); if (fsm_trim(fsm, FSM_TRIM_START_AND_END_REACHABLE, &shortest_end_distance) < 0) { return 0; } + TIME(&post); + DIFF_MSEC("trim", pre, post, NULL); if (fsm->statecount == 0) { r = 1; goto cleanup; } +#if LOG_INIT > 1 + fprintf(stderr, "=== AFTER TRIM, %d states\n", fsm_countstates(fsm)); + fprintf(stderr, "# pre_minimise\n"); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "#### pre_minimise", fsm); +#endif + TIME(&pre); collect_labels(fsm, labels, &label_count); TIME(&post); @@ -109,6 +175,8 @@ fsm_minimise(struct fsm *fsm) goto cleanup; } + fsm_capture_integrity_check(dst); + #if EXPENSIVE_CHECKS if (!fsm_capture_has_captures(fsm)) { struct fsm *oracle = fsm_minimise_test_oracle(fsm); @@ -249,6 +317,12 @@ build_minimised_mapping(const struct fsm *fsm, goto cleanup; } + /* This only needs to be run once, but must run before the main + * fixpoint loop below, because it potentially refines ECs. */ + if (!split_ecs_by_end_metadata(&env, fsm)) { + goto cleanup; + } + #if LOG_INIT for (i = 0; i < env.ec_count; i++) { fprintf(stderr, "# --ec[%lu]: %d\n", i, env.ecs[i]); @@ -325,7 +399,7 @@ build_minimised_mapping(const struct fsm *fsm, } } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS check_done_ec_offset(&env); #endif } @@ -361,6 +435,12 @@ build_minimised_mapping(const struct fsm *fsm, } #endif +#if EXPENSIVE_CHECKS + for (i = 0; i < fsm->statecount; i++) { + assert(mapping[i] < fsm->statecount); + } +#endif + #if LOG_STEPS fprintf(stderr, "# done in %lu iteration(s), %lu step(s), %ld -> %ld states, label_count %lu\n", env.iter, env.steps, fsm->statecount, @@ -399,7 +479,7 @@ dump_ecs(FILE *f, const struct min_env *env) #endif } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS static void check_descending_EC_counts(const struct min_env *env) { @@ -607,7 +687,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, /* The dead state is not a member of any EC. */ env->state_ecs[env->dead_state] = NO_ID; -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS check_descending_EC_counts(env); #endif @@ -642,7 +722,456 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, #endif } -#if EXPENSIVE_INTEGRITY_CHECKS +static int +accum_endids(const struct fsm *fsm, fsm_state_t end_state, + size_t nth, fsm_end_id_t id, void *opaque) +{ + fsm_end_id_t *buf = opaque; + (void)fsm; + (void)end_state; + buf[nth] = id; + +#if LOG_ECS + fprintf(stderr, " %d", id); +#endif + + return 1; +} + +SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() +static void +incremental_hash_of_ids(uint64_t *accum, fsm_end_id_t id) +{ + (*accum) += hash_id(id); +} + +static int +same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) +{ + if (a->end.count != b->end.count) { + return 0; + } + + if (a->capture.count != b->capture.count) { + return 0; + } + + /* compare -- these must be sorted */ + + for (size_t i = 0; i < a->end.count; i++) { + if (a->end.ids[i] != b->end.ids[i]) { + return 0; + } + } + + for (size_t i = 0; i < a->capture.count; i++) { + if (a->capture.ids[i] != b->capture.ids[i]) { + return 0; + } + } + + return 1; +} + +static int +split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) +{ + int res = 0; + + struct end_metadata *end_md; + fsm_state_t *htab = NULL; + + const size_t state_count = fsm_countstates(fsm); + +#if EXPENSIVE_CHECKS + /* Invariant: For each EC, either all or none of the states + * are end states. We only partition the set(s) of end states + * here. */ + assert(all_end_states_are_currently_together(env)); +#endif + + /* Use the hash table to assign to new groups. */ + + end_md = f_calloc(fsm->opt->alloc, + state_count, sizeof(end_md[0])); + if (end_md == NULL) { + goto cleanup; + } + + size_t bucket_count = 1; + while (bucket_count < state_count) { + bucket_count *= 2; /* power of 2 ceiling */ + } + const size_t mask = bucket_count - 1; + + htab = f_malloc(fsm->opt->alloc, + bucket_count * sizeof(htab[0])); + if (htab == NULL) { + goto cleanup; + } + + /* First pass: collect end state metadata */ + for (size_t ec_i = 0; ec_i < env->ec_count; ec_i++) { + fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); +#if LOG_ECS + fprintf(stderr, "## EC %zu\n", ec_i); +#endif + while (s != NO_ID) { + struct end_metadata *e = &end_md[s]; + if (!fsm_isend(fsm, s)) { + break; /* this EC has non-end states, skip */ + } + + if (!collect_end_ids(fsm, s, &e->end)) { + goto cleanup; + } + + if (!collect_capture_ids(fsm, s, &e->capture)) { + goto cleanup; + } + + if (!collect_capture_program_ids(fsm, s, &e->program)) { + goto cleanup; + } + + s = env->jump[s]; + } + } + +#if LOG_ECS + fprintf(stderr, "==== BEFORE PARTITIONING BY END METADATA\n"); + dump_ecs(stderr, env); + fprintf(stderr, "====\n"); +#endif + + /* Second pass: partition ECs into groups with identical end IDs. + * for each group with different end IDs, unlink them. */ + const size_t max_ec = env->ec_count; + for (size_t ec_i = 0; ec_i < max_ec; ec_i++) { + fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); + fsm_state_t prev = NO_ID; + + for (size_t i = 0; i < bucket_count; i++) { + htab[i] = NO_ID; /* reset hash table */ + } + + while (s != NO_ID) { + const struct end_metadata *s_md = &end_md[s]; + + uint64_t hash = 0; + const fsm_state_t next = env->jump[s]; + + for (size_t eid_i = 0; eid_i < s_md->end.count; eid_i++) { + incremental_hash_of_ids(&hash, s_md->end.ids[eid_i]); + } + for (size_t pid_i = 0; pid_i < s_md->program.count; pid_i++) { + incremental_hash_of_ids(&hash, s_md->program.ids[pid_i]); + } + + for (size_t b_i = 0; b_i < bucket_count; b_i++) { + fsm_state_t *b = &htab[(b_i + hash) & mask]; + const fsm_state_t other = *b; + const struct end_metadata *other_md = &end_md[other]; + + if (other == NO_ID) { /* empty hash bucket */ + *b = s; + if (prev == NO_ID) { + /* keep the first state, along with other states + * with matching end IDs, in this EC. no-op. */ +#if LOG_ECS + fprintf(stderr, " -- keeping state s %d in EC %u\n", + s, env->state_ecs[s]); +#endif + prev = s; + } else { /* not first (prev is set), so it landed somewhere else */ + /* unlink and assign new EC */ +#if LOG_ECS + fprintf(stderr, " -- moving state s %d from EC %u to EC %u\n", + s, env->state_ecs[s], env->ec_count); +#endif + env->jump[prev] = env->jump[s]; /* unlink */ + env->ecs[env->ec_count] = s; /* head of new EC */ + env->state_ecs[s] = env->ec_count; + env->jump[s] = NO_ID; + env->ec_count++; + } + break; + } else if (same_end_metadata(s_md, other_md)) { + if (env->state_ecs[other] == ec_i) { + /* keep in the current EC -- no-op */ +#if LOG_ECS + fprintf(stderr, " -- keeping state s %d in EC %u\n", + s, env->state_ecs[s]); +#endif + prev = s; + } else { + /* unlink and link to other state's EC */ +#if LOG_ECS + fprintf(stderr, " -- appending s %d to EC %u, after state %d, before %d\n", + s, env->state_ecs[other], other, env->jump[other]); +#endif + assert(prev != NO_ID); + env->jump[prev] = env->jump[s]; /* unlink */ + env->state_ecs[s] = env->state_ecs[other]; + env->jump[s] = env->jump[other]; + env->jump[other] = s; /* link after other */ + } + break; + } else { + continue; /* collision */ + } + } + + s = next; + } + + /* If this EC only has one entry and it's before the + * done_ec_offset, then set that here so that invariants + * will be restored while sweeping forward after this loop. */ + + if (env->jump[MASK_EC_HEAD(env->ecs[ec_i])] == NO_ID && ec_i < env->done_ec_offset) { + env->done_ec_offset = ec_i; /* will be readjusted later */ + } + +#if LOG_ECS + fprintf(stderr, "==== AFTER PARTITIONING BY END METADATA -- EC %zu\n", ec_i); + dump_ecs(stderr, env); + fprintf(stderr, "==== (done_ec_offset: %d)\n", env->done_ec_offset); +#endif + } + +#if LOG_ECS + fprintf(stderr, "==== AFTER PARTITIONING BY END IDs\n"); + dump_ecs(stderr, env); + fprintf(stderr, "==== (done_ec_offset: %d)\n", env->done_ec_offset); +#endif + + /* Sweep forward and swap ECs as necessary so all single-entry + * ECs are at the end -- they're done. */ + size_t ec_i = env->done_ec_offset; + + while (ec_i < env->ec_count) { + const fsm_state_t head = MASK_EC_HEAD(env->ecs[ec_i]); + if (env->jump[head] == NO_ID) { + /* offset stays where it is */ +#if LOG_ECS + fprintf(stderr, "ec_i: %zu / %u -- branch a\n", ec_i, env->ec_count); +#endif + env->ecs[ec_i] = SET_SMALL_EC_FLAG(head); + } else { + /* this EC has more than one state, but is after + * the done_ec_offset, so swap it with an EC at + * the boundary. */ + const fsm_state_t n_ec_i = env->done_ec_offset; +#if LOG_ECS + fprintf(stderr, "ec_i: %zu / %u -- branch b -- swap %ld and %d\n", + ec_i, env->ec_count, ec_i, n_ec_i); +#endif + + /* swap ec[n_ec_i] and ec[ec_i] */ + const fsm_state_t tmp = env->ecs[ec_i]; + env->ecs[ec_i] = env->ecs[n_ec_i]; + env->ecs[n_ec_i] = tmp; + /* note: this may set the SMALL_EC_FLAG. */ + update_ec_links(env, ec_i); + update_ec_links(env, n_ec_i); + env->done_ec_offset++; + } + ec_i++; + } + +#if LOG_ECS + fprintf(stderr, "==== (done_ec_offset is now: %d, ec_count %u)\n", env->done_ec_offset, env->ec_count); + dump_ecs(stderr, env); +#endif + + /* check that all ECs are before/after done_ec_offset */ + for (size_t ec_i = 0; ec_i < env->ec_count; ec_i++) { + const fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); +#if LOG_ECS + fprintf(stderr, " -- ec_i %zu: s %d\n", ec_i, s); +#endif + if (ec_i < env->done_ec_offset) { + assert(env->jump[s] != NO_ID); + } else { + assert(env->jump[s] == NO_ID); + } + } + + res = 1; + +cleanup: + if (htab != NULL) { + f_free(fsm->opt->alloc, htab); + } + if (end_md != NULL) { + size_t i; + for (i = 0; i < state_count; i++) { + struct end_metadata *e = &end_md[i]; + if (e->end.ids != NULL) { + f_free(fsm->opt->alloc, e->end.ids); + } + if (e->capture.ids != NULL) { + f_free(fsm->opt->alloc, e->capture.ids); + } + if (e->program.ids != NULL) { + f_free(fsm->opt->alloc, e->program.ids); + } + } + f_free(fsm->opt->alloc, end_md); + } + + return res; +} + +static int +cmp_end_ids(const void *pa, const void *pb) +{ + const fsm_end_id_t a = *(fsm_end_id_t *)pa; + const fsm_end_id_t b = *(fsm_end_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static int +cmp_unsigned(const void *pa, const void *pb) +{ + const unsigned a = *(unsigned *)pa; + const unsigned b = *(unsigned *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static int +collect_end_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_end *e) +{ + e->count = fsm_getendidcount(fsm, s); + + if (e->count > 0) { + e->ids = f_malloc(fsm->opt->alloc, + e->count * sizeof(e->ids[0])); + if (e->ids == NULL) { + return 0; + } + +#if LOG_ECS + fprintf(stderr, "%d:", s); +#endif + fsm_iterendids(fsm, s, + accum_endids, (void *)e->ids); +#if LOG_ECS + fprintf(stderr, "\n"); +#endif + /* sort, to make comparison easier later */ + qsort(e->ids, e->count, + sizeof(e->ids[0]), cmp_end_ids); + } + return 1; +} + +struct collect_capture_env { + int ok; + const struct fsm_alloc *alloc; + struct end_metadata_capture *c; + struct end_metadata_program *p; +}; + +static int +collect_capture_cb(fsm_state_t state, unsigned capture_id, + void *opaque) +{ + struct collect_capture_env *env = opaque; + struct end_metadata_capture *c = env->c; + (void)state; + if (c->count == c->ceil) { + const size_t nceil = (c->count == 0) + ? DEF_CAPTURE_ID_CEIL + : 2*c->ceil; + unsigned *nids = f_realloc(env->alloc, c->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + env->ok = 0; + return 0; + } + c->ids = nids; + c->ceil = nceil; + } + + c->ids[c->count] = capture_id; + c->count++; + return 1; +} + +static int +collect_capture_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_capture *c) +{ + struct collect_capture_env env = { + .ok = 1, + .alloc = fsm->opt->alloc, + .c = c, + }; + fsm_capture_iter_active_for_end_state(fsm, s, + collect_capture_cb, &env); + + if (env.ok) { + if (c->ids == NULL) { + assert(c->count == 0); + } else { + qsort(c->ids, c->count, sizeof(c->ids[0]), cmp_unsigned); + } + } + + return env.ok; +} + +static int +collect_capture_program_ids_cb(fsm_state_t state, unsigned prog_id, + void *opaque) +{ + struct collect_capture_env *env = opaque; + struct end_metadata_program *p = env->p; + (void)state; + if (p->count == p->ceil) { + const size_t nceil = (p->count == 0) + ? DEF_CAPTURE_ID_CEIL + : 2*p->ceil; + unsigned *nids = f_realloc(env->alloc, p->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + env->ok = 0; + return 0; + } + p->ids = nids; + p->ceil = nceil; + } + + p->ids[p->count] = prog_id; + p->count++; + return 1; +} + +static int +collect_capture_program_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_program *p) +{ + struct collect_capture_env env = { + .ok = 1, + .alloc = fsm->opt->alloc, + .p = p, + }; + fsm_capture_iter_program_ids_for_end_state(fsm, s, + collect_capture_program_ids_cb, &env); + + if (env.ok) { + if (p->ids == NULL) { + assert(p->count == 0); + } else { + qsort(p->ids, p->count, sizeof(p->ids[0]), cmp_unsigned); + } + } + + return env.ok; +} + +#if EXPENSIVE_CHECKS static void check_done_ec_offset(const struct min_env *env) { @@ -657,13 +1186,34 @@ check_done_ec_offset(const struct min_env *env) * worth the added complexity to avoid checking ECs 0 and 1. */ for (i = 0; i < env->ec_count; i++) { const fsm_state_t head = MASK_EC_HEAD(env->ecs[i]); - if (i >= done_ec_offset) { + if (i >= env->done_ec_offset) { assert(head == NO_ID || env->jump[head] == NO_ID); } else if (i >= 2) { assert(env->jump[head] != NO_ID); } } } + +static int +all_end_states_are_currently_together(const struct min_env *env) +{ + /* For each EC, either all or none of the states in it + * are end states. */ + for (size_t i = 0; i < env->ec_count; i++) { + const fsm_state_t head = MASK_EC_HEAD(env->ecs[i]); + const int ec_first_is_end = fsm_isend(env->fsm, head); + + fsm_state_t s = env->jump[head]; + while (s != NO_ID) { + if (fsm_isend(env->fsm, s) != ec_first_is_end) { + return 0; + } + s = env->jump[s]; + } + } + + return 1; +} #endif static int @@ -813,7 +1363,7 @@ try_partition(struct min_env *env, unsigned char label, const unsigned dead_state_ec = env->state_ecs[env->dead_state]; const struct fsm_state *states = env->fsm->states; -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Count states here, to compare against the partitioned * EC' counts later. */ size_t state_count = 0, psrc_count, pdst_count; @@ -853,7 +1403,7 @@ try_partition(struct min_env *env, unsigned char label, first_ec = dead_state_ec; } #if LOG_PARTITIONS > 1 - fprintf(stderr, "# --- try_partition: label '%c' -> EC %d\n", label, first_ec); + fprintf(stderr, "# --- try_partition: label '%c' -> first_ec %d\n", label, first_ec); #endif partition_counts[0] = 1; @@ -893,7 +1443,7 @@ try_partition(struct min_env *env, unsigned char label, partition_counts[0]++; prev = cur; cur = env->jump[cur]; - } else { /* unlink, split */ + } else if (to_ec != first_ec) { /* definitely different destination EC: unlink, split */ fsm_state_t next; #if LOG_PARTITIONS > 1 fprintf(stderr, "# try_partition: unlinking -- label '%c', src %u, dst %u, first_ec %d, cur %u -> to_ec %d\n", label, ec_src, ec_dst, first_ec, cur, to_ec); @@ -908,10 +1458,21 @@ try_partition(struct min_env *env, unsigned char label, env->ecs[ec_dst] = cur; cur = next; partition_counts[1]++; + } else { + /* Restrict the ones that will be marked as checked + * to the common subset before continuing, so that any + * other labels will still be checked in a later pass. */ + for (size_t i = 0; i < 4; i++) { + checked_labels[i] &= cur_label_set[i]; + } + + partition_counts[0]++; + prev = cur; + cur = env->jump[cur]; } } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Count how many states were split into each EC * and check that the sum matches the original count. */ psrc_count = 0; diff --git a/src/libfsm/print/Makefile b/src/libfsm/print/Makefile index c2911318b..9c6f42bbf 100644 --- a/src/libfsm/print/Makefile +++ b/src/libfsm/print/Makefile @@ -20,6 +20,9 @@ SRC += src/libfsm/print/vmasm.c .for src in ${SRC:Msrc/libfsm/print/*.c} CFLAGS.${src} += -I src # XXX: for internal.h DFLAGS.${src} += -I src # XXX: for internal.h + +CFLAGS.${src} += -std=c99 +DFLAGS.${src} += -std=c99 .endfor .for src in ${SRC:Msrc/libfsm/print/*.c} diff --git a/src/libfsm/state.c b/src/libfsm/state.c index acf2bff25..c4425077a 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -17,6 +17,8 @@ #include #include "internal.h" +#include "capture.h" +#include "endids.h" int fsm_addstate(struct fsm *fsm, fsm_state_t *state) @@ -33,17 +35,12 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) const size_t factor = 2; /* a guess */ const size_t n = fsm->statealloc * factor; struct fsm_state *tmp; - size_t i; tmp = f_realloc(fsm->opt->alloc, fsm->states, n * sizeof *fsm->states); if (tmp == NULL) { return 0; } - for (i = fsm->statealloc; i < n; i++) { - tmp[i].has_capture_actions = 0; - } - fsm->statealloc = n; fsm->states = tmp; } @@ -253,6 +250,18 @@ fsm_compact_states(struct fsm *fsm, } } + if (!fsm_endid_compact(fsm, mapping, orig_statecount)) { + goto error; + } + + if (!fsm_capture_id_compact(fsm, mapping, orig_statecount)) { + goto error; + } + + if (!fsm_capture_program_association_compact(fsm, mapping, orig_statecount)) { + goto error; + } + assert(dst == kept); assert(kept == fsm->statecount); @@ -278,4 +287,9 @@ fsm_compact_states(struct fsm *fsm, *removed = removed_count; } return 1; + +error: + f_free(fsm->opt->alloc, mapping); + + return 0; } diff --git a/src/libfsm/trim.c b/src/libfsm/trim.c index 6a9a25f09..36bf9145d 100644 --- a/src/libfsm/trim.c +++ b/src/libfsm/trim.c @@ -10,13 +10,16 @@ #include #include +#include #include #include +#include #include #include #include "internal.h" +#include "capture.h" #define DEF_EDGES_CEIL 8 #define DEF_ENDS_CEIL 8 @@ -42,17 +45,18 @@ save_edge(const struct fsm_alloc *alloc, size_t *count, size_t *ceil, struct edge **edges, fsm_state_t from, fsm_state_t to); -static int -cmp_edges_by_to(const void *pa, const void *pb) +static fsm_state_t +get_max_to(const struct edge *edges, size_t edge_count) { - const struct edge *a = (const struct edge *)pa; - const struct edge *b = (const struct edge *)pb; - - return a->to < b->to ? -1 - : a->to > b->to ? 1 - : a->from < b->from ? -1 - : a->from > b->from ? 1 - : 0; + size_t i; + fsm_state_t res = edges[0].to; + for (i = 1; i < edge_count; i++) { + const fsm_state_t to = edges[i].to; + if (to > res) { + res = to; + } + } + return res; } static int @@ -61,7 +65,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, { /* Use a queue to walk breath-first over all states reachable * from the start state. Note all end states. Collect all the - * edges, then sort them by the note they lead to, to convert it + * edges, then sort them by the node they lead to, to convert it * to a reverse edge index. Then, enqueue all the end states, * and again use the queue to walk the graph breadth-first, but * this time iterating bottom-up from the end states, and mark @@ -81,10 +85,13 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, fsm_state_t max_end; const size_t state_count = fsm->statecount; + fsm_state_t max_to; + unsigned *pv = NULL; size_t *offsets = NULL; + INIT_TIMERS(); - if (!fsm_getstart(fsm, &start)) { + if (!fsm_getstart(fsm, &start) || state_count == 0) { return 1; /* nothing is reachable */ } @@ -225,7 +232,33 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } /* Sort edges by state they lead to, inverting the index. */ - qsort(edges, edge_count, sizeof(edges[0]), cmp_edges_by_to); + max_to = edge_count == 0 ? 0 : get_max_to(edges, edge_count); +#if LOG_TRIM + fprintf(stderr, " -- edge count %zu, got max_to %u\n", edge_count, max_to); +#endif + TIME(&pre); + pv = permutation_vector_with_size_and_offset(fsm->opt->alloc, + edge_count, max_to, edges, sizeof(edges[0]), offsetof(struct edge, to)); + TIME(&post); + DIFF_MSEC("trim_pv_so", pre, post, NULL); + + if (EXPENSIVE_CHECKS) { + size_t i; + int ok = 1; +#if LOG_TRIM + fprintf(stderr, "\n#i\tedge\tpv\tsorted, max_to %u\n", max_to); +#endif + for (i = 0; i < edge_count; i++) { +#if LOG_TRIM + fprintf(stderr, "%zu\t%u\t%u\t%u\n", + i, edges[i].to, pv[i], edges[pv[i]].to); +#endif + if (i > 0 && edges[pv[i]].to < edges[pv[i - 1]].to) { + ok = 0; + } + } + assert(ok); + } max_end = 0; @@ -282,7 +315,6 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, * offsets[i - 1], to represent zero entries. */ { size_t i; - const fsm_state_t max_to = edges[edge_count - 1].to; const size_t offset_count = fsm_countstates(fsm); offsets = f_calloc(fsm->opt->alloc, @@ -292,7 +324,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } for (i = 0; i < edge_count; i++) { - const fsm_state_t to = edges[i].to; + const fsm_state_t to = edges[pv[i]].to; offsets[to] = i + 1; } @@ -312,8 +344,8 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (LOG_TRIM > 1) { size_t i; for (i = 0; i < edge_count; i++) { - fprintf(stderr, "mark_states: edges[%zu]: %d -> %d\n", - i, edges[i].from, edges[i].to); + fprintf(stderr, "mark_states: edges[pv[%zu]]: %d -> %d\n", + i, edges[pv[i]].from, edges[pv[i]].to); } } @@ -331,13 +363,13 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } for (e_i = base; e_i < limit; e_i++) { - const fsm_state_t from = edges[e_i].from; + const fsm_state_t from = edges[pv[e_i]].from; const unsigned end_distance = (sed == NULL ? 0 : sed[s_id]); assert(from < state_count); if (LOG_TRIM > 0) { - fprintf(stderr, "mark_states: edges[%zu]: from: %d, visited? %d\n", + fprintf(stderr, "mark_states: edges[pv[%zu]]: from: %d, visited? %d\n", e_i, from, fsm->states[from].visited); } @@ -370,6 +402,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (ends != NULL) { f_free(fsm->opt->alloc, ends); } if (offsets != NULL) { f_free(fsm->opt->alloc, offsets); } if (q != NULL) { queue_free(q); } + if (pv != NULL) { f_free(fsm->opt->alloc, pv); } return res; } @@ -457,7 +490,7 @@ integrity_check(const char *descr, const struct fsm *fsm) struct edge_iter edge_iter; struct fsm_edge e; -#ifdef NDEBUG +#if defined(NDEBUG) || !EXPENSIVE_CHECKS return; #endif @@ -484,8 +517,14 @@ integrity_check(const char *descr, const struct fsm *fsm) } } + fsm_capture_integrity_check(fsm); + if (LOG_TRIM > 1) { fprintf(stderr, "integrity check: %s...PASS\n", descr); + if (LOG_TRIM > 2) { + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "post_trim", fsm); + } } } @@ -504,6 +543,8 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, return 1; } + integrity_check("pre", fsm); + if (shortest_end_distance != NULL && mode == FSM_TRIM_START_AND_END_REACHABLE) { size_t s_i; diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 455dc8359..a99b49be2 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -32,6 +32,7 @@ fsm_union(struct fsm *a, struct fsm *b, if (combine_info == NULL) { combine_info = &combine_info_internal; } + memset(combine_info, 0x00, sizeof(*combine_info)); assert(a != NULL); assert(b != NULL); @@ -101,8 +102,16 @@ fsm_union_array(size_t fsm_count, fsms[0] = NULL; memset(bases, 0x00, fsm_count * sizeof(bases[0])); +#if EXPENSIVE_CHECKS + size_t capture_count = fsm_capture_count(res); +#endif + for (i = 1; i < fsm_count; i++) { struct fsm_combine_info ci; +#if EXPENSIVE_CHECKS + capture_count += fsm_capture_count(fsms[i]); +#endif + struct fsm *combined = fsm_union(res, fsms[i], &ci); fsms[i] = NULL; if (combined == NULL) { @@ -138,5 +147,8 @@ fsm_union_array(size_t fsm_count, } #endif +#if EXPENSIVE_CHECKS + assert(fsm_capture_count(res) == capture_count); +#endif return res; } diff --git a/src/libre/Makefile b/src/libre/Makefile index a88a92418..508b77a76 100644 --- a/src/libre/Makefile +++ b/src/libre/Makefile @@ -10,13 +10,17 @@ SRC += src/libre/ast_new_from_fsm.c SRC += src/libre/ast_rewrite.c SRC += src/libre/ac.c SRC += src/libre/re_strings.c +SRC += src/libre/re_capvm_compile.c # generated SRC += src/libre/class_name.c -.for src in ${SRC:Msrc/libre/ast_compile.c} +.for src in ${SRC:Msrc/libre/ast*.c} ${SRC:Msrc/libre/re*.c} CFLAGS.${src} += -I src # XXX: for internal.h DFLAGS.${src} += -I src # XXX: for internal.h + +CFLAGS.${src} += -std=c99 +DFLAGS.${src} += -std=c99 .endfor LIB += libre diff --git a/src/libre/ast.c b/src/libre/ast.c index 2e1d21817..6bd6063f5 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -494,6 +494,7 @@ ast_expr_cmp(const struct ast_expr *a, const struct ast_expr *b) case AST_EXPR_GROUP: if (a->u.group.id < b->u.group.id) { return -1; } if (a->u.group.id > b->u.group.id) { return +1; } + /* .repeated flag is ignored here */ return ast_expr_cmp(a->u.group.e, b->u.group.e); @@ -753,6 +754,7 @@ ast_make_expr_group(struct ast_expr_pool **poolp, enum re_flags re_flags, struct res->re_flags = re_flags; res->u.group.e = e; res->u.group.id = id; + res->u.group.repeated = 0; /* may be set during analysis */ return res; } @@ -770,6 +772,7 @@ ast_make_expr_anchor(struct ast_expr_pool **poolp, enum re_flags re_flags, enum res->type = AST_EXPR_ANCHOR; res->re_flags = re_flags; res->u.anchor.type = type; + res->u.anchor.is_end_nl = 0; /* may be set later */ return res; } diff --git a/src/libre/ast.h b/src/libre/ast.h index 233744847..20cc3cdb7 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -7,6 +7,11 @@ #ifndef RE_AST_H #define RE_AST_H +#include +#include +#include +#include + /* * This is a duplicate of struct lx_pos, but since we're linking to * code with several distinct lexers, there isn't a clear lexer.h @@ -62,7 +67,9 @@ enum ast_anchor_type { * followed by nullable nodes. * * - AST_FLAG_UNSATISFIABLE - * The node caused the regex to become unsatisfiable. + * The node is unsatisfiable (can never match anything). + * This can cause AST subtrees to be pruned, or for the + * entire regex to become unsatisfiable. * * - AST_FLAG_NULLABLE * The node is not always evaluated, such as nodes that @@ -159,6 +166,8 @@ struct ast_expr { size_t count; /* used */ size_t alloc; /* allocated */ struct ast_expr **n; + int contains_empty_groups; + int nullable_alt_inside_plus_repeat; } alt; struct { @@ -172,12 +181,15 @@ struct ast_expr { struct ast_expr_repeat { struct ast_expr *e; unsigned min; - unsigned max; + unsigned max; /* can be AST_COUNT_UNBOUNDED */ + int contains_empty_groups; + int contains_nullable_alt; } repeat; struct { struct ast_expr *e; unsigned id; + int repeated; /* set during analysis */ } group; struct { @@ -235,9 +247,12 @@ ast_pool_free(struct ast_expr_pool *pool); struct ast_expr_pool * ast_expr_pool_save(void); +#define AST_NO_MAX_CAPTURE_ID ((long)-1) + struct ast { struct ast_expr_pool *pool; struct ast_expr *expr; + long max_capture_id; int has_unanchored_start; int has_unanchored_end; }; diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 945eeb814..70e65cad4 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -19,7 +19,7 @@ #define LOG_ANALYSIS 0 #define LOG_FIRST_ANALYSIS (0 + LOG_ANALYSIS) -#define LOG_REPEATED_GROUPS (0 + LOG_ANALYSIS) +#define LOG_REPETITION_CASES (0 + LOG_ANALYSIS) #define LOG_FORKING (0 + LOG_ANALYSIS) #define LOG_ANCHORING (0 + LOG_ANALYSIS) #define LOG_CONCAT_FLAGS (0 + LOG_ANALYSIS) @@ -35,6 +35,11 @@ /* Mask for end-anchor flags */ #define END_ANCHOR_FLAG_MASK (AST_FLAG_ANCHORED_END | AST_FLAG_END_NL) +struct capture_env { + long max_capture_id; + int use_captures; +}; + static int is_nullable(const struct ast_expr *n) { @@ -224,7 +229,7 @@ analysis_iter(struct ast_expr *n) } case AST_EXPR_ANCHOR: - /* anchor flags will be handled on the second pass */ + /* anchor flags will be handled on the next pass */ break; case AST_EXPR_SUBTRACT: @@ -252,23 +257,281 @@ analysis_iter(struct ast_expr *n) return AST_ANALYSIS_OK; } -/* Analysis for unanchored starts/ends uses three values, because some - * details decide the whole subtree is/isn't, others should defer to - * analysis elsewhere it the tree, but an overall result of undecided - * still decides yes. */ -enum unanchored_analysis_res { - UA_NO = 'N', - UA_YES = 'Y', - UA_UNDECIDED = 'U', +static int +is_only_anchors(struct ast_expr *expr) +{ + if (can_consume_input(expr)) { return 0; } + + switch (expr->type) { + case AST_EXPR_ANCHOR: + return 1; + + case AST_EXPR_CONCAT: + if (expr->u.concat.count == 0) { return 0; } + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (!is_only_anchors(expr->u.concat.n[i]) + && can_consume_input(expr->u.concat.n[i])) { + return 0; + } + } + return 1; + + case AST_EXPR_ALT: + assert(expr->u.alt.count > 0); + for (size_t i = 0; i < expr->u.alt.count; i++) { + /* earlier matches will shadow later ones */ + if (is_only_anchors(expr->u.alt.n[i])) { + return 1; + } + } + return 0; + + case AST_EXPR_REPEAT: + if (expr->u.repeat.min == 0 && expr->u.repeat.max == 0) { + return 0; + } + return is_only_anchors(expr->u.repeat.e); + + case AST_EXPR_GROUP: + return is_only_anchors(expr->u.group.e); + + default: + break; + } + + return 0; +} + +static enum ast_analysis_res +analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_parent, + int shadowed_by_previous_alt_case, struct ast_expr *repeat_plus_ancestor) +{ + enum ast_analysis_res res = AST_ANALYSIS_OK; + + LOG(3 - LOG_REPETITION_CASES, "%s: node %p, type %s, shadowed_by_previous_alt_case %d\n", + __func__, (void *)n, ast_node_type_name(n->type), shadowed_by_previous_alt_case); + + if (shadowed_by_previous_alt_case) { + assert(outermost_repeat_parent == NULL + || outermost_repeat_parent->type == AST_EXPR_ALT); + } + + switch (n->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + case AST_EXPR_ANCHOR: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + break; + + case AST_EXPR_CONCAT: { + /* If this CONCAT array of nodes always consumes input, then + * it cannot be repeated empty, so it cannot produce the + * special case that needs outermost_repeat_parent for + * AST_EXPR_REPEAT's case below. + * + * An example input that needs this is 'x(()x)*' for "xx", + * because the 'x' prevents the outermost group + * from repeating and matching empty again after consuming + * a run of "x"s. */ + if (always_consumes_input(n)) { + outermost_repeat_parent = NULL; + } + + for (size_t i = 0; i < n->u.concat.count; i++) { + res = analysis_iter_repetition(n->u.concat.n[i], outermost_repeat_parent, + shadowed_by_previous_alt_case, repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + } + break; + } + + case AST_EXPR_ALT: { + /* See AST_EXPR_GROUP below for why this matters. */ + int new_shadowed_by_previous_alt_case = shadowed_by_previous_alt_case; + + /* FIXME: check nesting of this construct */ + + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT node %p, repeat_plus_ancestor %p\n", + __func__, (void *)n, (void *)repeat_plus_ancestor); + + for (size_t i = 0; i < n->u.alt.count; i++) { + /* If this is an ALT inside of a repeated subtree that contains + * a capture, this will need special handling. */ + if (outermost_repeat_parent != NULL) { + LOG(3 - LOG_REPETITION_CASES, + "%s: setting outermost_repeat_parent to %p for alt branch %zu, repeat_plus_ancestor %p\n", + __func__, (void *)n, i, (void *)repeat_plus_ancestor); + outermost_repeat_parent = n; + } + + if (is_nullable(n->u.alt.n[i]) || is_only_anchors(n->u.alt.n[i])) { + LOG(3 - LOG_REPETITION_CASES, + "%s: setting new_shadowed_by_previous_alt_case for alt branch %zu, repeat_plus_ancestor %p\n", + __func__, i, (void *)repeat_plus_ancestor); + new_shadowed_by_previous_alt_case = 1; + if (repeat_plus_ancestor != NULL) { + n->u.alt.nullable_alt_inside_plus_repeat = 1; + assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT); + assert(repeat_plus_ancestor->u.repeat.min == 1); + assert(repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); + repeat_plus_ancestor->u.repeat.contains_nullable_alt = 1; + + /* Repetition of an alt subtree which has a capture group child that + * only contains only* anchors is not handled properly yet. This + * isn't actually _useful_, it's just something that comes up + * in fuzzing, so reject it as an unsupported PCRE construct. + * + * An example input that triggers this is '^(($)|)+$' . */ + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } + } + + res = analysis_iter_repetition(n->u.alt.n[i], + outermost_repeat_parent, + new_shadowed_by_previous_alt_case, + repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + } + break; + } + + case AST_EXPR_REPEAT: + { + struct ast_expr *child = n->u.repeat.e; + + LOG(3 - LOG_REPETITION_CASES, "%s: REPEAT node %p, min %u max %u nullable? %d, !cci %d\n", + __func__, (void *)n, n->u.repeat.min, n->u.repeat.max, + is_nullable(child), can_consume_input(child)); + + if (n->u.repeat.min == 1 && + n->u.repeat.max == AST_COUNT_UNBOUNDED) { + LOG(3 - LOG_REPETITION_CASES, "%s: setting repeat_plus_ancestor to %p\n", + __func__, (void *)n); + repeat_plus_ancestor = n; + } else { + repeat_plus_ancestor = NULL; + } + + /* Special cases for a repeated group that contains possibly empty captures, + * in order to correctly reflect their repeating one more time and capture + * at the end (but without an infinite loop). + * + * For example, '^((x?))*$' will always end up with capture groups 1 and 2 + * at the end of the input for any number of "x"s, since the outermost ()* + * can always repeat once more time, consuming nothing, and clobber the + * existing captures. We mark repeated groups so that the compiled capture + * program can move saving the captures after the repetition, instead + * behaving like `^((?:x?)*(())$`. + * + * However, if the repeated subtree always consumes input, such as with + * '^(()a)+b$', then clear any passed in outermost_repeat_parent, because + * having to consume input will prevent that extra repetition of the + * empty captures. */ + if (always_consumes_input(n)) { + res = analysis_iter_repetition(child, NULL, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + } else if (outermost_repeat_parent == NULL && n->u.repeat.max > 1) { + LOG(3 - LOG_REPETITION_CASES, "%s: recursing with outermost_repeat_parent set to %p\n", + __func__, (void *)n); + res = analysis_iter_repetition(child, n, 0, + repeat_plus_ancestor); + } else { + LOG(3 - LOG_REPETITION_CASES, "%s: recursing with outermost_repeat_parent %p\n", + __func__, (void *)outermost_repeat_parent); + res = analysis_iter_repetition(child, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + } + if (res != AST_ANALYSIS_OK) { return res; } + break; + } + + case AST_EXPR_GROUP: + LOG(3 - LOG_REPETITION_CASES, + "%s: GROUP %p, repeat_plus_ancestor %p\n", + __func__, (void *)n, (void *)repeat_plus_ancestor); + + + if (outermost_repeat_parent != NULL && (is_nullable(n) || !can_consume_input(n))) { + int should_mark_repeated = 1; + /* If the outermost_repeat_parent is an ALT node and a previous ALT subtree + * matching the empty string is shadowing this group, then do not mark it + * as repeated, because that can lead to incorrect handling in somewhat + * contrived regexes like '^(?:|(|x))*$'. */ + if (outermost_repeat_parent->type == AST_EXPR_ALT && shadowed_by_previous_alt_case) { + LOG(3 - LOG_REPETITION_CASES, + "%s: hit group shadowed_by_previous_alt_case, skipping\n", __func__); + should_mark_repeated = 0; + } + + if (n->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END)) { + LOG(3 - LOG_REPETITION_CASES, + "%s: hit repeating anchor, skipping\n", __func__); + should_mark_repeated = 0; + } + + if (should_mark_repeated) { + LOG(3 - LOG_REPETITION_CASES, "%s: setting group %u to repeated\n", + __func__, n->u.group.id); + n->u.group.repeated = 1; + assert(outermost_repeat_parent->type == AST_EXPR_REPEAT || + outermost_repeat_parent->type == AST_EXPR_ALT); + LOG(3 - LOG_REPETITION_CASES, "%s: setting contains_empty_groups on outermost_repeat_parent %p\n", + __func__, (void *)outermost_repeat_parent); + if (outermost_repeat_parent->type == AST_EXPR_REPEAT) { + outermost_repeat_parent->u.repeat.contains_empty_groups = 1; + } else if (outermost_repeat_parent->type == AST_EXPR_ALT) { + outermost_repeat_parent->u.alt.contains_empty_groups = 1; + } else { + assert(!"type mismatch"); + } + } + } + + if (repeat_plus_ancestor != NULL && (is_nullable(n) || !can_consume_input(n))) { + assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT + && repeat_plus_ancestor->u.repeat.min == 1 + && repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); + LOG(3 - LOG_REPETITION_CASES, + "%s: not yet implemented, skipping\n", __func__); + /* return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; */ + } + + res = analysis_iter_repetition(n->u.group.e, outermost_repeat_parent, + shadowed_by_previous_alt_case, repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + break; + + case AST_EXPR_SUBTRACT: + res = analysis_iter_repetition(n->u.subtract.a, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + res = analysis_iter_repetition(n->u.subtract.b, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + break; + + default: + assert(!"unreached"); + } + return res; +} + +enum anchoring_analysis_res { + ANCHOR_ANALYSIS_ANCHORED = 'a', + ANCHOR_ANALYSIS_UNANCHORED = 'u', + ANCHOR_ANALYSIS_UNDECIDED = '_', }; -static enum unanchored_analysis_res +static enum anchoring_analysis_res analysis_iter_unanchored_start(const struct ast_expr *n) { if (n->flags & AST_FLAG_ANCHORED_START) { LOG(4 - LOG_UNANCHORED_FLAGS, "%s: n (%p)->flags & AST_FLAG_ANCHORED_START -> N\n", __func__, (void *)n); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; } LOG(4 - LOG_UNANCHORED_FLAGS, "%s: node %p, type %s\n", @@ -278,7 +541,7 @@ analysis_iter_unanchored_start(const struct ast_expr *n) case AST_EXPR_EMPTY: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; case AST_EXPR_TOMBSTONE: break; @@ -287,12 +550,12 @@ analysis_iter_unanchored_start(const struct ast_expr *n) switch (n->u.anchor.type) { case AST_ANCHOR_START: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ^ anchor -> N\n", __func__); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; case AST_ANCHOR_END: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: $ anchor -> U\n", __func__); /* will be handled by other cases */ - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; default: assert(!"unreached"); @@ -308,7 +571,7 @@ analysis_iter_unanchored_start(const struct ast_expr *n) case AST_EXPR_RANGE: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> Y\n", __func__, ast_node_type_name(n->type)); - return UA_YES; + return ANCHOR_ANALYSIS_UNANCHORED; case AST_EXPR_CONCAT: { size_t i; @@ -316,8 +579,8 @@ analysis_iter_unanchored_start(const struct ast_expr *n) /* has unanchored start if first non-nullable child does */ for (i = 0; i < n->u.concat.count; i++) { const struct ast_expr *child = n->u.concat.n[i]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_start(child); - if (child_res != UA_UNDECIDED) { + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_start(child); + if (child_res != ANCHOR_ANALYSIS_UNDECIDED) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), child_res); return child_res; @@ -335,19 +598,19 @@ analysis_iter_unanchored_start(const struct ast_expr *n) size_t i; /* if all children agree, return that result, otherwise undecided */ - const enum unanchored_analysis_res first_child_res = analysis_iter_unanchored_start(n->u.alt.n[0]); + const enum anchoring_analysis_res first_child_res = analysis_iter_unanchored_start(n->u.alt.n[0]); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child 0 -- %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), first_child_res); for (i = 1; i < n->u.alt.count; i++) { const struct ast_expr *child = n->u.alt.n[i]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_start(child); + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_start(child); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child %zd -- %s -> %c (child res)\n", __func__, i, ast_node_type_name(n->type), child_res); if (child_res != first_child_res) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child result) contracts first, returning U\n", __func__, ast_node_type_name(n->type), child_res); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } } @@ -360,7 +623,7 @@ analysis_iter_unanchored_start(const struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U (repeat.min == 0)\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } return analysis_iter_unanchored_start(n->u.repeat.e); @@ -374,16 +637,16 @@ analysis_iter_unanchored_start(const struct ast_expr *n) assert(!"unreached"); } - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } -static enum unanchored_analysis_res +static enum anchoring_analysis_res analysis_iter_unanchored_end(const struct ast_expr *n) { if (n->flags & AST_FLAG_ANCHORED_END) { LOG(4 - LOG_UNANCHORED_FLAGS, "%s: node (%p)->flags & AST_FLAG_ANCHORED_END -> N\n", __func__, (void *)n); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; } LOG(4 - LOG_UNANCHORED_FLAGS, "%s: node %p, type %s\n", @@ -393,7 +656,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) case AST_EXPR_EMPTY: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; case AST_EXPR_TOMBSTONE: break; @@ -403,12 +666,12 @@ analysis_iter_unanchored_end(const struct ast_expr *n) case AST_ANCHOR_START: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ^ %s -> U\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; case AST_ANCHOR_END: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: $ %s -> N\n", __func__, ast_node_type_name(n->type)); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; default: assert(!"unreached"); @@ -424,7 +687,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) case AST_EXPR_RANGE: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> Y\n", __func__, ast_node_type_name(n->type)); - return UA_YES; + return ANCHOR_ANALYSIS_UNANCHORED; case AST_EXPR_CONCAT: { size_t i; @@ -432,8 +695,8 @@ analysis_iter_unanchored_end(const struct ast_expr *n) /* has unanchored end if last non-nullable child does */ for (i = n->u.concat.count; i > 0; i--) { const struct ast_expr *child = n->u.concat.n[i - 1]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_end(child); - if (child_res != UA_UNDECIDED) { + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_end(child); + if (child_res != ANCHOR_ANALYSIS_UNDECIDED) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), child_res); return child_res; @@ -451,19 +714,19 @@ analysis_iter_unanchored_end(const struct ast_expr *n) size_t i; /* if all children agree, return that result, otherwise undecided */ - const enum unanchored_analysis_res first_child_res = analysis_iter_unanchored_end(n->u.alt.n[0]); + const enum anchoring_analysis_res first_child_res = analysis_iter_unanchored_end(n->u.alt.n[0]); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child 0 -- %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), first_child_res); for (i = 1; i < n->u.alt.count; i++) { const struct ast_expr *child = n->u.alt.n[i]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_end(child); + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_end(child); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child %zd -- %s -> %c (child res)\n", __func__, i, ast_node_type_name(n->type), child_res); if (child_res != first_child_res) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child result) contracts first, returning U\n", __func__, ast_node_type_name(n->type), child_res); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } } @@ -476,7 +739,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U (repeat.min == 0)\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } return analysis_iter_unanchored_end(n->u.repeat.e); @@ -490,7 +753,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) assert(!"unreached"); } - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } static void @@ -548,9 +811,68 @@ struct anchoring_env { /* Corresponding flag for end anchors while sweeping backward. */ int followed_by_consuming; + /* Special case for detecting '$[^a]', which matches "\n" with + * a capture group 0 of (0,1) in PCRE. */ + int followed_by_consuming_newline; + + /* Flag for tracking whether we're in a part of the subtree that + * is always before a start anchor. This influences satisfiability + * and edge cases like '()*^'. */ int before_start_anchor; + + /* Flag used to detect and reject the awkward case in '$[^a]', + * where (according to PCRE) the character class after the '$' + * should match the literal newline, but nothing else, and only + * once. Because $ is actually a zero-width assertion that + * execution is either at the end of input or a trailing + * newline, it has the rather surprising result that '$[^a]' + * will not match "x" but *will* match "x\n" (because it has a $ + * before a trailing newline, and because the newline matches + * the non-skippable [^a]). We just return an unsupported + * error for this case. */ + enum newline_after_end_anchor_state { + NAEAS_NONE, + NAEAS_WOULD_MATCH_ONCE, + } newline_after_end_anchor_state; + + int after_end_anchor; }; +/* Does the subtree match a literal '\n'? */ +static int +matches_newline(const struct ast_expr *n) +{ + switch (n->type) { + case AST_EXPR_LITERAL: + return n->u.literal.c == '\n'; + + case AST_EXPR_SUBTRACT: + return matches_newline(n->u.subtract.a) + && !matches_newline(n->u.subtract.b); + + case AST_EXPR_RANGE: + { + const struct ast_endpoint *f = &n->u.range.from; + const struct ast_endpoint *t = &n->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL + || t->type != AST_ENDPOINT_LITERAL) { + /* not implemented */ + LOG(1 - LOG_ANCHORING, "%s: not implemented\n", __func__); + return 0; + } + + const int res = f->u.literal.c <= '\n' && t->u.literal.c >= '\n'; + LOG(1 - LOG_ANCHORING, "%s: RANGE res %d\n", __func__, res); + return res; + } + + default: + break; + } + + return 0; +} + /* Tree walker that analyzes the AST, marks which nodes and subtrees are * anchored at the start and/or end, and determines which subtrees are * unsatisfiable due to start anchoring. @@ -585,7 +907,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_ANCHOR_START: /* * If it's not possible to get here without consuming - * any input and there's a start anchor, the regex is + * any input and there's a start anchor, the subtree is * inherently unsatisfiable. */ set_flags(n, AST_FLAG_ANCHORED_START); @@ -615,6 +937,9 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) set_flags(n, AST_FLAG_ANCHORED_END); if (n->u.anchor.is_end_nl && !(env->re_flags & RE_ANCHORED)) { set_flags(n, AST_FLAG_END_NL); + if (env->newline_after_end_anchor_state == NAEAS_NONE) { + env->newline_after_end_anchor_state = NAEAS_WOULD_MATCH_ONCE; + } } break; @@ -627,6 +952,15 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) * These are the types that actually consume input. */ case AST_EXPR_LITERAL: + if (n->u.literal.c == '\n' && + (env->newline_after_end_anchor_state == NAEAS_WOULD_MATCH_ONCE)) { + LOG(3 - LOG_ANCHORING, + "%s: LITERAL: rejecting non-optional newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } + break; case AST_EXPR_CODEPOINT: case AST_EXPR_RANGE: break; /* handled outside switch/case */ @@ -658,6 +992,13 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (res != AST_ANALYSIS_OK && res != AST_ANALYSIS_UNSATISFIABLE) { /* unsat is handled below */ + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + + /* FIXME: check this */ + assert(child->flags & AST_FLAG_UNSATISFIABLE); + + set_flags(n, AST_FLAG_UNSATISFIABLE); + } return res; } @@ -688,6 +1029,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) env->past_always_consuming = 1; } + env->newline_after_end_anchor_state = child_env.newline_after_end_anchor_state; } /* flow ANCHORED_START and ANCHORED_END flags upward */ @@ -701,14 +1043,6 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: setting ANCHORED_START due to child %zu (%p)'s analysis\n", __func__, i, (void *)child); set_flags(n, AST_FLAG_ANCHORED_START); - - if (after_always_consumes) { - LOG(3 - LOG_ANCHORING, - "%s: setting %p and child %p UNSATISFIABLE due to ^-anchored child that always consumes input\n", - __func__, (void *)n, (void *)child); - set_flags(n, AST_FLAG_UNSATISFIABLE); - set_flags(child, AST_FLAG_UNSATISFIABLE); - } } if (always_consumes_input(child)) { @@ -771,16 +1105,21 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) for (i = 0; i < n->u.concat.count; i++) { struct ast_expr *child = n->u.concat.n[i]; if (after_end_anchor) { - if (always_consumes_input(child)) { + if (child->type == AST_EXPR_REPEAT + && (child->flags & AST_FLAG_UNSATISFIABLE) + && child->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, - "%s: after_end_anchor & ALWAYS_CONSUMES on child %p -> UNSATISFIABLE\n", - __func__, (void *)child); - set_flags(child, AST_FLAG_UNSATISFIABLE); + "%s: setting unsatisfiable {0,*} repeat after $ anchor to {0,0}\n", + __func__); + child->u.repeat.max = 0; } if (child->type == AST_EXPR_REPEAT - && (child->flags & AST_FLAG_UNSATISFIABLE) + && (child->flags & AST_FLAG_END_NL) && child->u.repeat.min == 0) { + LOG(3 - LOG_ANCHORING, + "%s: setting {0,*} repeat subtree matching \n after $ anchor to {0,0}\n", + __func__); child->u.repeat.max = 0; } } else if (!after_end_anchor @@ -831,6 +1170,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } else if (res == AST_ANALYSIS_OK) { all_set_past_always_consuming &= child_env.past_always_consuming; any_sat = 1; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE + || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + assert(child->flags & AST_FLAG_UNSATISFIABLE); + continue; } else { return res; } @@ -868,7 +1211,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (all_end_anchored) { LOG(3 - LOG_ANCHORING, "%s: ALT: all_end_anchored -> ANCHORED_END\n", __func__); - /* FIXME: AST_FLAG_END_NL: need to determine how this interacts + /* TODO: AST_FLAG_END_NL: need to determine how this interacts * with alt nodes. `^(?:(a)\z|(a)$)` */ set_flags(n, AST_FLAG_ANCHORED_END); } @@ -910,15 +1253,26 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) return res; } - if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_END && n->u.repeat.min > 0) { - /* FIXME: if repeating something that is always - * anchored at the end, repeat.max could be - * capped at 1, but I have not yet found any - * inputs where that change is necessary to - * produce a correct result. */ - LOG(3 - LOG_ANCHORING, - "%s: REPEAT: repeating ANCHORED_END subtree >0 times -> ANCHORED_END\n", __func__); - set_flags(n, n->u.repeat.e->flags & END_ANCHOR_FLAG_MASK); + /* If the child subtree is anchored at the start and/or end, then this + * node can be repeated at most once. */ + const int child_is_anchored = n->u.repeat.e->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END); + if (child_is_anchored) { + if (n->u.repeat.min >= 1) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: copying anchor flags from child subtree since we cannot repeat it 0 times\n", __func__); + set_flags(n, n->u.repeat.e->flags & END_ANCHOR_FLAG_MASK); + } + + if (n->u.repeat.min > 1) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: anchored, so clamping min repeat to at most once\n", __func__); + n->u.repeat.min = 1; + } + if (n->u.repeat.max > 1) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: anchored, so clamping max repeat to at most once\n", __func__); + n->u.repeat.max = 1; + } } break; @@ -928,27 +1282,22 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) /* This flows anchoring flags upward even when the node * is unsatisfiable, because that info can impact how * the node's unsatisfiability is handled. */ -#define PROPAGATE_CHILD_FLAGS(TAG, N, CHILD) \ - do { \ - struct ast_expr *child = CHILD; \ - if (child->flags & AST_FLAG_ANCHORED_START) { \ - set_flags(N, AST_FLAG_ANCHORED_START); \ - } \ - if (child->flags & AST_FLAG_ANCHORED_END) { \ - set_flags(N, AST_FLAG_ANCHORED_END); \ - } \ - if (res == AST_ANALYSIS_UNSATISFIABLE) { \ - LOG(3 - LOG_ANCHORING, \ - "%s: %s: setting UNSATISFIABLE due to unsatisfiable child\n", \ - __func__, TAG); \ - set_flags(N, AST_FLAG_UNSATISFIABLE); \ - } \ - if (res != AST_ANALYSIS_OK) { \ - return res; \ - } \ - } while(0) - - PROPAGATE_CHILD_FLAGS("GROUP", n, n->u.group.e); + struct ast_expr *child = n->u.group.e; + if (child->flags & AST_FLAG_ANCHORED_START) { + set_flags(n, AST_FLAG_ANCHORED_START); + } + if (child->flags & AST_FLAG_ANCHORED_END) { + set_flags(n, AST_FLAG_ANCHORED_END); + } + if (res == AST_ANALYSIS_UNSATISFIABLE) { + LOG(3 - LOG_ANCHORING, + "%s: GROUP: setting UNSATISFIABLE due to unsatisfiable childn", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + } + if (res != AST_ANALYSIS_OK) { + return res; + } break; case AST_EXPR_SUBTRACT: @@ -959,6 +1308,15 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.subtract.a->flags & AST_FLAG_ANCHORED_END) { set_flags(n, n->u.subtract.a->flags & END_ANCHOR_FLAG_MASK); } + + if (env->newline_after_end_anchor_state == NAEAS_WOULD_MATCH_ONCE) { + LOG(3 - LOG_ANCHORING, + "%s: SUBTRACT: rejecting non-optional newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } + if (res != AST_ANALYSIS_OK) { if (res == AST_ANALYSIS_UNSATISFIABLE) { set_flags(n, AST_FLAG_UNSATISFIABLE); @@ -1002,9 +1360,9 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) enum ast_analysis_res res; /* - * Second pass, in reverse, specifically checking for end-anchored - * subtrees that are unsatisfiable because they are followed by - * nodes that always consume input. + * Second anchoring pass, in reverse, specifically checking for + * end-anchored subtrees that are unsatisfiable because they are + * followed by nodes that always consume input. * * Also check for subtrees that always consume input appearing * before a start anchor and mark them as unsatisfiable. @@ -1016,6 +1374,14 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) switch (n->type) { case AST_EXPR_EMPTY: + if (env->before_start_anchor) { + /* Needed for cases like '()*^' matching "". */ + LOG(3 - LOG_ANCHORING, "%s: skipping EMPTY before ^\n", __func__); + + set_flags(n, AST_FLAG_ANCHORED_START); + } + break; + case AST_EXPR_TOMBSTONE: break; @@ -1029,10 +1395,19 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) break; case AST_ANCHOR_END: - /* should already be set during forward pass */ - assert(n->flags & AST_FLAG_ANCHORED_END); + /* Normally this will have been set during the forward pass, + * but if it's inside an unsatisfiable CONCAT node whose earlier + * children casued it to be rejected, forward analysis will not + * have reached it. */ + set_flags(n, AST_FLAG_ANCHORED_END); - if (env->followed_by_consuming) { + if (env->followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: RANGE: rejecting possible newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } else if (env->followed_by_consuming) { LOG(3 - LOG_ANCHORING, "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", __func__); @@ -1098,6 +1473,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) set_flags(n, AST_FLAG_UNSATISFIABLE); } } else if (res != AST_ANALYSIS_OK) { + set_flags(n, AST_FLAG_UNSATISFIABLE); return res; } @@ -1120,6 +1496,13 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) __func__, (void *)child); env->before_start_anchor = 1; } + + if (!env->followed_by_consuming_newline && child_env.followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: setting followed_by_consuming_newline due to child %p's analysis\n", + __func__, (void *)child); + env->followed_by_consuming_newline = 1; + } } break; @@ -1128,6 +1511,8 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_ALT: { int any_sat = 0; int all_set_followed_by_consuming = 1; + int all_set_followed_by_consuming_newline = 1; + int any_set_followed_by_consuming_newline = 0; int all_set_before_start_anchor = 1; assert(n->u.alt.count > 0); @@ -1153,7 +1538,15 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) } else if (res == AST_ANALYSIS_OK) { all_set_followed_by_consuming &= child_env.followed_by_consuming; all_set_before_start_anchor &= child_env.before_start_anchor; + + any_set_followed_by_consuming_newline |= child_env.followed_by_consuming_newline; + all_set_followed_by_consuming_newline &= child_env.followed_by_consuming_newline; + any_sat = 1; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE + || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + assert(child->flags & AST_FLAG_UNSATISFIABLE); + continue; } else { return res; } @@ -1166,6 +1559,13 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming = 1; } + if (!env->followed_by_consuming_newline && any_set_followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: ALT: any_set_followed_by_consuming_newline -> setting env->followed_by_consuming_newline for feature PCRE rejection\n", + __func__); + env->followed_by_consuming_newline = 1; + } + if (!env->before_start_anchor && all_set_before_start_anchor) { LOG(3 - LOG_ANCHORING, "%s: ALT: all_set_before_start_anchor -> setting env->before_start_anchor\n", @@ -1190,6 +1590,8 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, "%s: REPEAT: UNSATISFIABLE but can be repeated 0 times, ignoring\n", __func__); + /* Set the REPEAT node to repeat 0 times (to + * prune it) rather than making it as unsatisfiable. */ n->u.repeat.max = 0; /* skip */ break; } else if (n->u.repeat.min > 0) { @@ -1254,6 +1656,10 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) return AST_ANALYSIS_UNSATISFIABLE; } + if (n->flags & AST_FLAG_CAN_CONSUME && matches_newline(n)) { + env->followed_by_consuming_newline = 1; + } + return AST_ANALYSIS_OK; } @@ -1318,7 +1724,7 @@ assign_firsts(struct ast_expr *n) * subexpression is compiled, the links to the global self-loop * are created, which the REPEAT node then copies. * - * It probably makes sense to not go further + * FIXME: needs tests. It probably makes sense to not go further * here because the top layer of the repeated section * should only link to the global start once. */ @@ -1401,7 +1807,7 @@ assign_lasts(struct ast_expr *n) * subexpression is compiled, the links to the global self-loop * are created, which the REPEAT node then copies. * - * It probably makes sense to not go further + * FIXME: needs tests. It probably makes sense to not go further * here because the top layer of the repeated section * should only link to the global start once. */ @@ -1422,6 +1828,106 @@ assign_lasts(struct ast_expr *n) } } +static enum ast_analysis_res +analysis_iter_captures(struct capture_env *env, struct ast_expr *n) +{ + enum ast_analysis_res res; + + switch (n->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + case AST_EXPR_ANCHOR: + break; + + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + break; + + case AST_EXPR_CONCAT: { + size_t i; + + for (i = 0; i < n->u.concat.count; i++) { + res = analysis_iter_captures(env, n->u.concat.n[i]); + if (res != AST_ANALYSIS_OK) { + return res; + } + } + + break; + } + + case AST_EXPR_ALT: { + size_t i; + + for (i = 0; i < n->u.alt.count; i++) { + res = analysis_iter_captures(env, n->u.alt.n[i]); + if (res != AST_ANALYSIS_OK) { + return res; + } + } + + break; + } + + case AST_EXPR_REPEAT: { + res = analysis_iter_captures(env, n->u.repeat.e); + if (res != AST_ANALYSIS_OK) { + return res; + } + +/* Set this to 1 when running the fuzzer, so that it ignores + * uninteresting failures from regexes like '.{1000000}' that use + * repetition to hit memory limits. + * + * This should be set by the build system when building for fuzzing. */ +#ifndef FUZZING_LIMITS +#define FUZZING_LIMITS 0 +#endif + +#if FUZZING_LIMITS + if ((n->u.repeat.max != AST_COUNT_UNBOUNDED && n->u.repeat.max >= 10) + || (n->u.repeat.min >= 10)) { + fprintf(stderr, "%s: rejecting regex with {count} >= 10 (%u)\n", + __func__, n->u.repeat.max); + return AST_ANALYSIS_ERROR_MEMORY; + } +#endif + + break; + } + + case AST_EXPR_GROUP: + if (env->max_capture_id == AST_NO_MAX_CAPTURE_ID + || n->u.group.id > env->max_capture_id) { + env->max_capture_id = n->u.group.id; + } + + res = analysis_iter_captures(env, n->u.group.e); + if (res != AST_ANALYSIS_OK) { + return res; + } + break; + + case AST_EXPR_SUBTRACT: + res = analysis_iter_captures(env, n->u.subtract.a); + if (res != AST_ANALYSIS_OK) { + return res; + } + + res = analysis_iter_captures(env, n->u.subtract.b); + if (res != AST_ANALYSIS_OK) { + return res; + } + break; + + default: + assert(!"unreached"); + } + + return AST_ANALYSIS_OK; +} + enum ast_analysis_res ast_analysis(struct ast *ast, enum re_flags flags) { @@ -1434,14 +1940,26 @@ ast_analysis(struct ast *ast, enum re_flags flags) assert(ast->expr != NULL); /* - * First pass -- track nullability, clean up some artifacts from - * parsing. + * First pass -- track nullability, which subtrees can/always consume + * input, and clean up some artifacts from parsing. */ res = analysis_iter(ast->expr); if (res != AST_ANALYSIS_OK) { return res; } + /* Next pass, check for capture IDs. */ + { + struct capture_env env = { .max_capture_id = AST_NO_MAX_CAPTURE_ID }; + env.use_captures = !(flags & RE_NOCAPTURE); + + res = analysis_iter_captures(&env, ast->expr); + if (res != AST_ANALYSIS_OK) { + return res; + } + ast->max_capture_id = env.max_capture_id; + } + /* * Next pass: set anchoring, now that nullability info from * the first pass is in place and some other things have been @@ -1449,8 +1967,10 @@ ast_analysis(struct ast *ast, enum re_flags flags) * start anchors. */ { - /* first anchoring analysis pass, sweeping forward */ - struct anchoring_env env = { .re_flags = flags }; + struct anchoring_env env = { + .re_flags = flags, + .newline_after_end_anchor_state = NAEAS_NONE, + }; res = analysis_iter_anchoring(&env, ast->expr); if (res != AST_ANALYSIS_OK) { return res; } @@ -1458,20 +1978,30 @@ ast_analysis(struct ast *ast, enum re_flags flags) res = analysis_iter_reverse_anchoring(&env, ast->expr); if (res != AST_ANALYSIS_OK) { return res; } - /* - * Next passes, mark all nodes in a first and/or last - * position. This is informed by the anchoring flags, so - * that needs to happen first. - */ - assign_firsts(ast->expr); - assign_lasts(ast->expr); - - ast->has_unanchored_start = (analysis_iter_unanchored_start(ast->expr) != UA_NO); - ast->has_unanchored_end = (analysis_iter_unanchored_end(ast->expr) != UA_NO); - LOG(2 - LOG_UNANCHORED_FLAGS, - "%s: has_unanchored_start %d, has_unanchored_end %d\n", - __func__, ast->has_unanchored_start, ast->has_unanchored_end); } + /* Next pass, mark some cases that need special handling + * due to repetition. For example, with cases like + * ^((x?))*$ the inner capture will always need to repeat + * one more time to match () after any 'x's. + * + * This needs to happen after the anchoring passes. */ + res = analysis_iter_repetition(ast->expr, NULL, 0, NULL); + if (res != AST_ANALYSIS_OK) { return res; } + + /* + * Next passes, mark all nodes in a first and/or last + * position. This is informed by the anchoring flags, so + * that needs to happen first. + */ + assign_firsts(ast->expr); + assign_lasts(ast->expr); + + ast->has_unanchored_start = (analysis_iter_unanchored_start(ast->expr) != ANCHOR_ANALYSIS_ANCHORED); + ast->has_unanchored_end = (analysis_iter_unanchored_end(ast->expr) != ANCHOR_ANALYSIS_ANCHORED); + LOG(2 - LOG_UNANCHORED_FLAGS, + "%s: has_unanchored_start %d, has_unanchored_end %d\n", + __func__, ast->has_unanchored_start, ast->has_unanchored_end); + return res; } diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index f9ec8ebbb..5390cce57 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -30,7 +30,9 @@ enum ast_analysis_res { AST_ANALYSIS_UNSATISFIABLE, AST_ANALYSIS_ERROR_NULL = -1, - AST_ANALYSIS_ERROR_MEMORY = -2 + AST_ANALYSIS_ERROR_MEMORY = -2, + AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE = -3, + AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE = -4 }; enum ast_analysis_res diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index e0f321037..e1188c4dc 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,18 +16,25 @@ #include #include +#include #include #include #include +#include +#include + #include "class.h" #include "ast.h" #include "ast_compile.h" +#include "re_capvm_compile.h" +#include "libfsm/capture.h" #include "libfsm/internal.h" /* XXX */ #define LOG_LINKAGE 0 +#define LOG_TRAMPOLINE 0 #if LOG_LINKAGE #include "print.h" @@ -44,12 +52,12 @@ enum link_side { * Use the passed in start/end states (x and y) * * - LINK_GLOBAL - * Link to the global start/end state (env->start or env->end), + * Link to the global start/end state (env->start_inner or env->end_inner), * because this node has a ^ or $ anchor * * - LINK_GLOBAL_SELF_LOOP * Link to the unanchored self loop adjacent to the start/end - * state (env->start_any_loop or env->end_any_loop), because + * states (env->start_any_inner or env->end_any_inner), because * this node is in a FIRST or LAST position, but unanchored. */ enum link_types { @@ -58,7 +66,12 @@ enum link_types { LINK_GLOBAL_SELF_LOOP, }; +/* Call stack for AST -> NFA conversion. */ +#define DEF_COMP_STACK_CEIL 4 +#define NO_MAX_CAPTURE_IDS ((unsigned)-1) + struct comp_env { + const struct fsm_alloc *alloc; struct fsm *fsm; enum re_flags re_flags; struct re_err *err; @@ -72,21 +85,90 @@ struct comp_env { * Also, some states in a first/last context need to link * directly to the overall start/end states, either in * place of or along with the adjacent states. + * + * The inner start and end states are considered inside of + * match group 0, outer are not. */ - fsm_state_t start; - fsm_state_t end; + fsm_state_t start_outer; + fsm_state_t start_inner; fsm_state_t start_any_loop; + fsm_state_t start_any_inner; + int have_start_any_loop; + + /* States leading to the end, with and without an unanchored + * `.*` loop that consumes any trailing characters. */ + fsm_state_t end_inner; + fsm_state_t end_outer; + fsm_state_t end_nl_inner; fsm_state_t end_any_loop; - fsm_state_t end_nl; - int has_start_any_loop; + fsm_state_t end_any_inner; int has_end_any_loop; - int has_end_nl; + int has_end_nl_inner; + + /* bitset for active capture IDs */ + uint64_t *active_capture_ids; + long max_capture_id; /* upper bound */ + + /* Evaluation stack */ + struct comp_stack { + size_t ceil; + size_t depth; /* 0 -> empty */ + + struct comp_stack_frame { + struct ast_expr *n; + fsm_state_t x; + fsm_state_t y; + unsigned step; + + union { + struct { + fsm_state_t link; + } concat; + struct { + unsigned count; + } alt; + struct { + struct fsm_subgraph subgraph; + fsm_state_t na; + fsm_state_t nz; + } repeat; + } u; + } *frames; + } stack; }; static int -comp_iter(struct comp_env *env, - fsm_state_t x, fsm_state_t y, - struct ast_expr *n, const struct ast_expr *parent); +comp_iter(struct comp_env *env, fsm_state_t x, const struct ast *ast); + +static int +eval_stack_frame(struct comp_env *env); + +static int +eval_EMPTY(struct comp_env *env); +static int +eval_CONCAT(struct comp_env *env); +static int +eval_ALT(struct comp_env *env); +static int +eval_LITERAL(struct comp_env *env); +static int +eval_CODEPOINT(struct comp_env *env); +static int +eval_REPEAT(struct comp_env *env); +static int +eval_GROUP(struct comp_env *env); +static int +eval_ANCHOR(struct comp_env *env); +static int +eval_SUBTRACT(struct comp_env *env); +static int +eval_RANGE(struct comp_env *env); +static int +eval_TOMBSTONE(struct comp_env *env); + +static int +compile_capvm_program_for_stack_end_states(struct comp_env *env, + const struct ast *ast, uint32_t *prog_id); static int utf8(uint32_t cp, char c[]) @@ -192,6 +274,9 @@ expr_compile(struct ast_expr *e, enum re_flags flags, struct ast ast; ast.expr = e; + ast.max_capture_id = 0; + ast.has_unanchored_start = 0; + ast.has_unanchored_end = 0; return ast_compile(&ast, flags, opt, err); } @@ -207,11 +292,11 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, assert(to < env->fsm->statecount); if (re_flags & RE_ICASE) { - if (!fsm_addedge_literal(fsm, from, to, tolower((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)tolower((unsigned char) c))) { return 0; } - - if (!fsm_addedge_literal(fsm, from, to, toupper((unsigned char) c))) { + + if (!fsm_addedge_literal(fsm, from, to, (char)toupper((unsigned char) c))) { return 0; } } else { @@ -219,38 +304,50 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, return 0; } } - + return 1; } static int intern_start_any_loop(struct comp_env *env) { - fsm_state_t loop; + fsm_state_t loop, inner; assert(env != NULL); - if (env->has_start_any_loop) { + if (env->have_start_any_loop) { return 1; } assert(~env->re_flags & RE_ANCHORED); - assert(env->start < env->fsm->statecount); + assert(env->start_outer < env->fsm->statecount); if (!fsm_addstate(env->fsm, &loop)) { return 0; } + if (!fsm_addstate(env->fsm, &inner)) { + return 0; + } + +#if LOG_LINKAGE + fprintf(stderr, "%s: start_any: loop %d, inner: %d\n", __func__, loop, inner); +#endif + if (!fsm_addedge_any(env->fsm, loop, loop)) { return 0; } - if (!fsm_addedge_epsilon(env->fsm, env->start, loop)) { + if (!fsm_addedge_epsilon(env->fsm, env->start_outer, loop)) { + return 0; + } + if (!fsm_addedge_epsilon(env->fsm, loop, inner)) { return 0; } env->start_any_loop = loop; - env->has_start_any_loop = 1; + env->start_any_inner = inner; + env->have_start_any_loop = 1; return 1; } @@ -258,7 +355,7 @@ intern_start_any_loop(struct comp_env *env) static int intern_end_any_loop(struct comp_env *env) { - fsm_state_t loop; + fsm_state_t loop, inner; assert(env != NULL); @@ -267,21 +364,32 @@ intern_end_any_loop(struct comp_env *env) } assert(~env->re_flags & RE_ANCHORED); - assert(env->end < env->fsm->statecount); + assert(env->end_outer < env->fsm->statecount); if (!fsm_addstate(env->fsm, &loop)) { return 0; } + if (!fsm_addstate(env->fsm, &inner)) { + return 0; + } + +#if LOG_LINKAGE + fprintf(stderr, "%s: end_any: %d, inner: %d\n", __func__, loop, inner); +#endif if (!fsm_addedge_any(env->fsm, loop, loop)) { return 0; } - if (!fsm_addedge_epsilon(env->fsm, loop, env->end)) { + if (!fsm_addedge_epsilon(env->fsm, inner, loop)) { + return 0; + } + if (!fsm_addedge_epsilon(env->fsm, loop, env->end_outer)) { return 0; } env->end_any_loop = loop; + env->end_any_inner = inner; env->has_end_any_loop = 1; return 1; @@ -290,37 +398,39 @@ intern_end_any_loop(struct comp_env *env) static int intern_end_nl(struct comp_env *env) { - /* PCRE's end anchor $ matches a single optional newline. + /* PCRE's end anchor $ matches a single optional newline, + * which should exist outside of match group 0. * - * Intern states for a `\n?` that links to the global end. */ + * Intern states for a `\n?` that links to */ assert(env != NULL); - if (env->has_end_nl) { + if (env->has_end_nl_inner) { return 1; } assert(~env->re_flags & RE_ANCHORED); assert(env->re_flags & RE_END_NL); - assert(env->end < env->fsm->statecount); + assert(~env->re_flags & RE_END_NL_DISABLE); + assert(env->end_outer < env->fsm->statecount); - fsm_state_t end_nl; - if (!fsm_addstate(env->fsm, &end_nl)) { + fsm_state_t inner; + if (!fsm_addstate(env->fsm, &inner)) { return 0; } #if LOG_LINKAGE - fprintf(stderr, "%s: end_nl: %d\n", __func__, end_nl); + fprintf(stderr, "%s: end_nl_inner: %d\n", __func__, inner); #endif - if (!fsm_addedge_epsilon(env->fsm, end_nl, env->end)) { + if (!fsm_addedge_epsilon(env->fsm, inner, env->end_outer)) { return 0; } - if (!fsm_addedge_literal(env->fsm, end_nl, env->end, (char)'\n')) { + if (!fsm_addedge_literal(env->fsm, inner, env->end_outer, (char)'\n')) { return 0; } - env->end_nl = end_nl; - env->has_end_nl = 1; + env->end_nl_inner = inner; + env->has_end_nl_inner = 1; return 1; } @@ -337,8 +447,8 @@ can_have_backward_epsilon_edge(const struct ast_expr *e) return 0; case AST_EXPR_SUBTRACT: - /* XXX: not sure */ - return 1; + /* Single character class */ + return 0; case AST_EXPR_REPEAT: /* 0 and 1 don't have backward epsilon edges */ @@ -396,21 +506,32 @@ can_skip_concat_state_and_epsilon(const struct ast_expr *l, return 0; } +static const struct ast_expr * +get_parent_node_from_stack(const struct comp_stack *stack) +{ + if (stack->depth < 2) { return NULL; } + return stack->frames[stack->depth - 2].n; +} + static enum link_types -decide_linking(struct comp_env *env, - struct ast_expr *n, const struct ast_expr *parent, enum link_side side) +decide_linking(struct comp_env *env, fsm_state_t x, fsm_state_t y, + struct ast_expr *n, enum link_side side) { assert(n != NULL); assert(env != NULL); + (void)x; + (void)y; + + struct comp_stack *stack = &env->stack; + /* If the regex is implicitly anchored and the dialect does * not support anchoring, linking is always top-down. */ if ((env->re_flags & RE_ANCHORED)) { return LINK_TOP_DOWN; } - /* parent can be NULL, if we're at the root node, but it must - * never be the same node. */ + const struct ast_expr *parent = get_parent_node_from_stack(stack); assert(parent != n); /* Note: any asymmetry here should be due to special cases @@ -478,183 +599,71 @@ print_linkage(enum link_types t) #define EPSILON(FROM, TO) \ assert((FROM) != (TO)); \ if (!fsm_addedge_epsilon(env->fsm, (FROM), (TO))) { return 0; } - + #define ANY(FROM, TO) \ if (!fsm_addedge_any(env->fsm, (FROM), (TO))) { return 0; } #define LITERAL(FROM, TO, C) \ if (!addedge_literal(env, n->re_flags, (FROM), (TO), ((char)C))) { return 0; } -#define RECURSE(FROM, TO, NODE, PARENT) \ - if (!comp_iter(env, (FROM), (TO), (NODE), (PARENT))) { return 0; } - -static int -comp_iter_repeated(struct comp_env *env, - fsm_state_t x, fsm_state_t y, - struct ast_expr *n) -{ - fsm_state_t a, b; - fsm_state_t na, nz; - unsigned i; - - assert(n->type == AST_EXPR_REPEAT); - const unsigned min = n->u.repeat.min; - const unsigned max = n->u.repeat.max; - struct ast_expr *e = n->u.repeat.e; - - assert(min <= max); - - if (min == 0 && max == 0) { /* {0,0} */ - EPSILON(x, y); - } else if (min == 0 && max == 1) { /* '?' */ - RECURSE(x, y, e, n); - EPSILON(x, y); - } else if (min == 1 && max == 1) { /* {1,1} */ - RECURSE(x, y, e, n); - } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* '*' */ - NEWSTATE(na); - NEWSTATE(nz); - EPSILON(x,na); - EPSILON(nz,y); - - EPSILON(na, nz); - RECURSE(na, nz, e, n); - EPSILON(nz, na); - } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* '+' */ - NEWSTATE(na); - NEWSTATE(nz); - EPSILON(x,na); - EPSILON(nz,y); - - RECURSE(na, nz, e, n); - EPSILON(nz, na); - } else { - /* - * Make new beginning/end states for the repeated section, - * build its NFA, and link to its head. - */ - - struct fsm_subgraph subgraph; - fsm_state_t tail; - - fsm_subgraph_start(env->fsm, &subgraph); - - NEWSTATE(na); - NEWSTATE(nz); - RECURSE(na, nz, e, n); - EPSILON(x, na); /* link head to repeated NFA head */ - - b = nz; /* set the initial tail */ - - /* can be skipped */ - if (min == 0) { - EPSILON(na, nz); - } - fsm_subgraph_stop(env->fsm, &subgraph); - tail = nz; - - if (max != AST_COUNT_UNBOUNDED) { - for (i = 1; i < max; i++) { - /* copies the original subgraph; need to set b to the - * original tail - */ - b = tail; +#define RETURN(ENV) comp_stack_pop(ENV) - if (!fsm_subgraph_duplicate(env->fsm, &subgraph, &b, &a)) { - return 0; - } +#define RECURSE(ENV, FROM, TO, NODE) \ + if (!comp_stack_push(ENV, (FROM), (TO), (NODE))) { return 0; } - EPSILON(nz, a); - - /* To the optional part of the repeated count */ - if (i >= min) { - EPSILON(nz, b); - } - - na = a; /* advance head for next duplication */ - nz = b; /* advance tail for concenation */ - } - } else { - for (i = 1; i < min; i++) { - /* copies the original subgraph; need to set b to the - * original tail - */ - b = tail; - - if (!fsm_subgraph_duplicate(env->fsm, &subgraph, &b, &a)) { - return 0; - } - - EPSILON(nz, a); - - na = a; /* advance head for next duplication */ - nz = b; /* advance tail for concenation */ - } - - /* back link to allow for infinite repetition */ - EPSILON(nz,na); - } - - /* tail to last repeated NFA tail */ - EPSILON(nz, y); - } - - return 1; -} +#define TAILCALL(ENV, FROM, TO, NODE) \ + comp_stack_tailcall(ENV, (FROM), (TO), (NODE)); static int -comp_iter(struct comp_env *env, - fsm_state_t x, fsm_state_t y, - struct ast_expr *n, const struct ast_expr *parent) +set_linking(struct comp_env *env, struct ast_expr *n, + enum link_types link_start, enum link_types link_end, + fsm_state_t *px, fsm_state_t *py) { - enum link_types link_start, link_end; - - if (n == NULL) { - return 1; - } - - link_start = decide_linking(env, n, parent, LINK_START); - link_end = decide_linking(env, n, parent, LINK_END); + fsm_state_t x = *px; + fsm_state_t y = *py; #if LOG_LINKAGE - fprintf(stderr, "%s: decide_linking %p: start ", __func__, (void *) n); + fprintf(stderr, "%s: decide_linking %p [%s]: start ", + __func__, (void *) n, ast_node_type_name(n->type)); print_linkage(link_start); fprintf(stderr, ", end "); print_linkage(link_end); fprintf(stderr, ", x %d, y %d\n", x, y); #else (void) print_linkage; + (void)n; #endif switch (link_start) { case LINK_TOP_DOWN: break; case LINK_GLOBAL: - x = env->start; + x = env->start_inner; break; case LINK_GLOBAL_SELF_LOOP: if (!intern_start_any_loop(env)) { return 0; } - assert(env->has_start_any_loop); + assert(env->have_start_any_loop); - x = env->start_any_loop; + x = env->start_any_inner; break; default: - assert(!"unreachable"); + assert(!"match fail"); /* these should be mutually exclusive now */ } switch (link_end) { case LINK_TOP_DOWN: break; case LINK_GLOBAL: - if (env->re_flags & RE_END_NL && (n->flags & AST_FLAG_END_NL)) { + if (env->re_flags & RE_END_NL && !(env->re_flags & RE_END_NL_DISABLE) + && (n->flags & AST_FLAG_END_NL)) { if (!intern_end_nl(env)) { return 0; } - y = env->end_nl; + y = env->end_nl_inner; } else { - y = env->end; + y = env->end_inner; } break; case LINK_GLOBAL_SELF_LOOP: @@ -663,243 +672,784 @@ comp_iter(struct comp_env *env, } assert(env->has_end_any_loop); - y = env->end_any_loop; + y = env->end_any_inner; break; default: - assert(!"unreachable"); + assert(!"match fail"); /* these should be mutually exclusive now */ } #if LOG_LINKAGE fprintf(stderr, " ---> x: %d, y: %d\n", x, y); #endif + *px = x; + *py = y; + return 1; +} - switch (n->type) { - case AST_EXPR_EMPTY: - /* skip these, when possible */ - EPSILON(x, y); - break; +static void +comp_stack_pop(struct comp_env *env) +{ + assert(env->stack.depth > 0); + env->stack.depth--; +} - case AST_EXPR_CONCAT: - { - fsm_state_t curr_x; - size_t i; +static int +comp_stack_push(struct comp_env *env, fsm_state_t x, fsm_state_t y, struct ast_expr *n) +{ + struct comp_stack *stack = &env->stack; + assert(n != NULL); - const size_t count = n->u.concat.count; + if (stack->depth == stack->ceil) { + const size_t nceil = 2*stack->ceil; + struct comp_stack_frame *nframes = f_realloc(env->alloc, + stack->frames, nceil * sizeof(stack->frames[0])); +#if LOG_LINKAGE || LOG_TRAMPOLINE + fprintf(stderr, "comp_stack_push: reallocating comp_stack, %zu -> %zu frames\n", + stack->ceil, nceil); +#endif + if (nframes == NULL) { + return 0; + } + stack->ceil = nceil; + stack->frames = nframes; + } - curr_x = x; + assert(stack->depth < stack->ceil); - assert(count >= 1); + struct comp_stack_frame *sf = &stack->frames[stack->depth]; + memset(sf, 0x00, sizeof(*sf)); + sf->n = n; + sf->x = x; + sf->y = y; - if (!fsm_addstate_bulk(env->fsm, count - 1)) { - return 0; - } + stack->depth++; + return 1; +} - for (i = 0; i < count; i++) { - struct ast_expr *curr = n->u.concat.n[i]; +static void +comp_stack_tailcall(struct comp_env *env, + fsm_state_t x, fsm_state_t y, struct ast_expr *n) +{ + struct comp_stack *stack = &env->stack; - /* If a subtree is unsatisfiable but also nullable, ignore it. */ - const enum ast_flags nullable_and_unsat = AST_FLAG_NULLABLE - | AST_FLAG_UNSATISFIABLE; - if ((curr->flags & nullable_and_unsat) == nullable_and_unsat) { - /* if necessary, link the end */ - if (i == count - 1) { - EPSILON(curr_x, y); - } - continue; - } + assert(stack->depth > 0); - struct ast_expr *next = i == count - 1 - ? NULL - : n->u.concat.n[i + 1]; + /* Replace current stack frame. */ + struct comp_stack_frame *sf = &stack->frames[stack->depth - 1]; + memset(sf, 0x00, sizeof(*sf)); + sf->n = n; + sf->x = x; + sf->y = y; +} - fsm_state_t z; - if (i + 1 < count) { - if (!fsm_addstate(env->fsm, &z)) { - return 0; - } -#if LOG_LINKAGE - fprintf(stderr, "%s: added state z %d\n", __func__, z); -#endif - } else { - z = y; /* connect to right parent to close off subtree */ - } +#define JUST_ONE_PROG 1 - /* - * If nullable, add an extra state & epsilion as a one-way gate - */ - if (!can_skip_concat_state_and_epsilon(curr, next)) { - fsm_state_t diode; +static int +comp_iter(struct comp_env *env, + fsm_state_t x, const struct ast *ast) +{ + int res = 1; + assert(ast != NULL); + assert(ast->expr != NULL); - NEWSTATE(diode); - EPSILON(curr_x, diode); - curr_x = diode; - } + struct comp_stack_frame *frames = NULL; + uint64_t *active_capture_ids = NULL; + const bool use_captures = (env->re_flags & RE_NOCAPTURE) == 0; - RECURSE(curr_x, z, curr, n); + frames = f_calloc(env->alloc, + DEF_COMP_STACK_CEIL, sizeof(env->stack.frames[0])); + if (frames == NULL) { + goto alloc_fail; + } - curr_x = z; + { + const size_t capture_id_words = (env->max_capture_id == AST_NO_MAX_CAPTURE_ID) + ? 0 + : ((env->max_capture_id)/64 + 1); + active_capture_ids = f_calloc(env->alloc, capture_id_words, + sizeof(active_capture_ids[0])); + if (active_capture_ids == NULL) { + goto alloc_fail; } + } - break; + /* Add inner and outer end states. Like start_outer and start_inner, + * these represent the boundary between match group 0 (inner) and + * states outside it (the unanchored end loop). */ + if (!fsm_addstate(env->fsm, &env->end_inner)) { + goto alloc_fail; + } + if (!fsm_addstate(env->fsm, &env->end_outer)) { + goto alloc_fail; + } + if (!fsm_addedge_epsilon(env->fsm, env->end_inner, env->end_outer)) { + goto alloc_fail; } - case AST_EXPR_ALT: - { - size_t i; + fsm_setend(env->fsm, env->end_outer, 1); - const size_t count = n->u.alt.count; +#if LOG_LINKAGE + fprintf(stderr, "end: outer %d, inner %d\n", + env->end_outer, env->end_inner); +#endif - assert(count >= 1); +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: x %d, y %d\n", __func__, x, env->end_inner); +#endif - for (i = 0; i < count; i++) { - /* skip unsatisfiable ALT subtrees */ - if (n->u.alt.n[i]->flags & AST_FLAG_UNSATISFIABLE) { + env->stack.ceil = DEF_COMP_STACK_CEIL; + env->stack.depth = 1; + env->stack.frames = frames; + env->active_capture_ids = active_capture_ids; + + { /* set up the first stack frame */ + struct comp_stack_frame *sf = &env->stack.frames[0]; + sf->n = ast->expr; + sf->x = x; + sf->y = env->end_inner; + sf->step = 0; + } + +#if JUST_ONE_PROG + uint32_t prog_id; + if (use_captures) { + if (!compile_capvm_program_for_stack_end_states(env, ast, &prog_id)) { + goto alloc_fail; + } + } +#endif + + /* evaluate call stack until termination */ + while (res && env->stack.depth > 0) { + if (!eval_stack_frame(env)) { +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: res -> 0\n", __func__); +#endif + res = 0; + break; + } + } + + if (use_captures && res && env->max_capture_id != AST_NO_MAX_CAPTURE_ID) { + /* Set the active captures on the end state. */ + for (unsigned i = 0; i <= (unsigned)env->max_capture_id; i++) { + if (!u64bitset_get(env->active_capture_ids, i)) { continue; } + if (!fsm_capture_set_active_for_end(env->fsm, i, env->end_outer)) { + goto alloc_fail; + } + } - /* - * CONCAT handles adding extra states and - * epsilons when necessary, so there isn't much - * more to do here. - */ - RECURSE(x, y, n->u.alt.n[i], n); +#if !JUST_ONE_PROG + uint32_t prog_id; + if (!compile_capvm_program_for_stack_end_states(env, stack, ast, &prog_id)) { + goto alloc_fail; + } +#endif + +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: associated prog_id %u with state %d\n", + __func__, prog_id, stack->end_outer); +#endif + if (!fsm_capture_associate_program_with_end_state(env->fsm, + prog_id, env->end_outer)) { + goto alloc_fail; } - break; } - case AST_EXPR_LITERAL: - LITERAL(x, y, n->u.literal.c); - break; + f_free(env->alloc, env->stack.frames); + f_free(env->alloc, env->active_capture_ids); - case AST_EXPR_CODEPOINT: { - fsm_state_t a, b; - char c[4]; - int r, i; + return res; - r = utf8(n->u.codepoint.u, c); - if (!r) { - if (env->err != NULL) { - env->err->e = RE_EBADCP; - env->err->cp = n->u.codepoint.u; - } +alloc_fail: + /* TODO: set env->err to indicate alloc failure */ + if (frames != NULL) { + f_free(env->alloc, frames); + } + if (active_capture_ids != NULL) { + f_free(env->alloc, active_capture_ids); + } + return 0; +} + +static struct comp_stack_frame * +get_comp_stack_top(struct comp_env *env) +{ + struct comp_stack *stack = &env->stack; + assert(stack->depth > 0); + struct comp_stack_frame *sf = &stack->frames[stack->depth - 1]; + assert(sf->n != NULL); + return sf; +} + +static int +eval_stack_frame(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: depth %zu/%zu, type %s, step %u\n", __func__, + stack->depth, stack->ceil, ast_node_type_name(sf->n->type), sf->step); +#endif + + /* If this is the first time the trampoline has called this + * state, decide the linking. Some of the states below (such as + * AST_EXPR_CONCAT) can have multiple child nodes, so they will + * increment step and use it to resume where they left off as + * the trampoline returns execution to them. */ + enum link_types link_end; + if (sf->step == 0) { /* entering state */ + enum link_types link_start; + link_start = decide_linking(env, sf->x, sf->y, sf->n, LINK_START); + link_end = decide_linking(env, sf->x, sf->y, sf->n, LINK_END); + if (!set_linking(env, sf->n, link_start, link_end, &sf->x, &sf->y)) { return 0; } + } + +#if LOG_TRAMPOLINE > 1 + fprintf(stderr, "%s: x %d, y %d\n", __func__, sf->x, sf->y); +#endif + + switch (sf->n->type) { + case AST_EXPR_EMPTY: + return eval_EMPTY(env); + case AST_EXPR_CONCAT: + return eval_CONCAT(env); + case AST_EXPR_ALT: + return eval_ALT(env); + case AST_EXPR_LITERAL: + return eval_LITERAL(env); + case AST_EXPR_CODEPOINT: + return eval_CODEPOINT(env); + case AST_EXPR_REPEAT: + return eval_REPEAT(env); + case AST_EXPR_GROUP: + return eval_GROUP(env); + case AST_EXPR_ANCHOR: + return eval_ANCHOR(env); + case AST_EXPR_SUBTRACT: + return eval_SUBTRACT(env); + case AST_EXPR_RANGE: + return eval_RANGE(env); + case AST_EXPR_TOMBSTONE: + return eval_TOMBSTONE(env); + default: + assert(!"unreached"); + return 0; + } +} - a = x; +static int +eval_EMPTY(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); +#if LOG_LINKAGE + fprintf(stderr, "eval_EMPTY: step %u, x %d -> y %d\n", + sf->step, sf->x, sf->y); +#endif - for (i = 0; i < r; i++) { - if (i + 1 < r) { - NEWSTATE(b); - } else { - b = y; + EPSILON(sf->x, sf->y); + RETURN(env); + return 1; +} + +static int +eval_CONCAT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; + const size_t count = n->u.concat.count; + assert(count >= 1); + +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: eval_CONCAT: x %d, y %d, step %d\n", + sf->x, sf->y, sf->step); +#endif + + if (sf->step == 0) { + sf->u.concat.link = sf->x; + } + + while (sf->step < count) { + fsm_state_t curr_x = sf->u.concat.link; + struct ast_expr *curr = n->u.concat.n[sf->step]; + + /* If a subtree is unsatisfiable but also nullable, ignore it. */ + const enum ast_flags nullable_and_unsat = AST_FLAG_NULLABLE + | AST_FLAG_UNSATISFIABLE; + if ((curr->flags & nullable_and_unsat) == nullable_and_unsat) { + sf->step++; + + /* if necessary, link the end */ + if (sf->step == count) { + EPSILON(curr_x, sf->y); } + return 1; + } - LITERAL(a, b, c[i]); + struct ast_expr *next = sf->step == count - 1 + ? NULL + : n->u.concat.n[sf->step + 1]; - a = b; + fsm_state_t z; + if (sf->step + 1 < count) { + if (!fsm_addstate(env->fsm, &z)) { + return 0; + } +#if LOG_LINKAGE + fprintf(stderr, "%s: added state z %d\n", __func__, z); +#endif + } else { + z = sf->y; /* connect to right parent to close off subtree */ } - break; - } +#if LOG_LINKAGE + fprintf(stderr, "%s: curr_x %d, z %d\n", + __func__, curr_x, z); +#endif - case AST_EXPR_REPEAT: /* - * REPEAT breaks out into its own function, because - * there are several special cases + * If nullable, add an extra state & epsilon as a one-way gate */ - if (!comp_iter_repeated(env, x, y, n)) { - return 0; + if (!can_skip_concat_state_and_epsilon(curr, next)) { + fsm_state_t diode; + + NEWSTATE(diode); + EPSILON(curr_x, diode); + curr_x = diode; +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: added diode %d\n", diode); +#endif } - break; - case AST_EXPR_GROUP: - RECURSE(x, y, n->u.group.e, n); - break; +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: recurse CONCAT[%u/%zu]: link %d, z %d\n", + sf->step, count, sf->u.concat.link, z); +#endif + /* Set the right side link, which will become the + * left side link for the next step (if any). */ + sf->u.concat.link = z; + sf->step++; + RECURSE(env, curr_x, z, curr); + return 1; + } - case AST_EXPR_TOMBSTONE: - /* do not link -- intentionally pruned */ - break; + RETURN(env); + return 1; +} - case AST_EXPR_ANCHOR: - EPSILON(x, y); - break; +static int +eval_ALT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + const size_t count = sf->n->u.alt.count; + assert(count >= 1); - case AST_EXPR_SUBTRACT: { - struct fsm *a, *b; - struct fsm *q; - enum re_flags re_flags; +#if LOG_LINKAGE + fprintf(stderr, "eval_ALT: step %u\n", sf->step); +#endif - re_flags = n->re_flags; + if (sf->step < count) { + struct ast_expr *n; - /* wouldn't want to reverse twice! */ - re_flags &= ~(unsigned)RE_REVERSE; + /* + * CONCAT handles adding extra states and + * epsilons when necessary, so there isn't much + * more to do here. + */ +#if LOG_LINKAGE + fprintf(stderr, "eval_ALT: recurse ALT[%u/%zu]: x %d, y %d\n", + sf->step, count, sf->x, sf->y); +#endif - a = expr_compile(n->u.subtract.a, re_flags, - fsm_getoptions(env->fsm), env->err); - if (a == NULL) { - return 0; + n = sf->n->u.alt.n[sf->step]; + assert(n != NULL); + sf->step++; /* RECURSE can realloc the stack and make sf stale. */ + + if (!(n->flags & AST_FLAG_UNSATISFIABLE)) { + RECURSE(env, sf->x, sf->y, n); } + return 1; + } - b = expr_compile(n->u.subtract.b, re_flags, - fsm_getoptions(env->fsm), env->err); - if (b == NULL) { - fsm_free(a); - return 0; + RETURN(env); + return 1; +} + +static int +eval_LITERAL(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; +#if LOG_LINKAGE + fprintf(stderr, "%s: linking %d -> %d with literal '%c' (0x%02x)\n", + __func__, sf->x, sf->y, isprint(n->u.literal.c) ? n->u.literal.c : '.', + n->u.literal.c); +#endif + + LITERAL(sf->x, sf->y, n->u.literal.c); + + RETURN(env); + return 1; +} + +static int +eval_CODEPOINT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; + fsm_state_t a, b; + char c[4]; + int r, i; + + r = utf8(n->u.codepoint.u, c); + if (!r) { + if (env->err != NULL) { + env->err->e = RE_EBADCP; + env->err->cp = n->u.codepoint.u; } - q = fsm_subtract(a, b); - if (q == NULL) { - return 0; + return 0; + } + + a = sf->x; + + for (i = 0; i < r; i++) { + if (i + 1 < r) { + NEWSTATE(b); + } else { + b = sf->y; } + LITERAL(a, b, c[i]); + + a = b; + } + + RETURN(env); + return 1; +} + +static int +eval_REPEAT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + fsm_state_t a, b; + unsigned i, min, max; + + assert(sf->n->type == AST_EXPR_REPEAT); + struct ast_expr_repeat *n = &sf->n->u.repeat; + + min = n->min; + max = n->max; + + assert(min <= max); + + if (min == 0 && max == 0) { /* {0,0} */ + EPSILON(sf->x, sf->y); + RETURN(env); + return 1; + } else if (min == 0 && max == 1) { /* '?' */ + EPSILON(sf->x, sf->y); + TAILCALL(env, sf->x, sf->y, n->e); + return 1; + } else if (min == 1 && max == 1) { /* {1,1} */ + TAILCALL(env, sf->x, sf->y, n->e); + return 1; + } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* '*' */ + fsm_state_t na, nz; + NEWSTATE(na); + NEWSTATE(nz); + EPSILON(sf->x,na); + EPSILON(nz,sf->y); + + EPSILON(na, nz); + EPSILON(nz, na); + TAILCALL(env, na, nz, n->e); + return 1; + } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* '+' */ + fsm_state_t na, nz; + NEWSTATE(na); + NEWSTATE(nz); + EPSILON(sf->x, na); + EPSILON(nz, sf->y); + + EPSILON(nz, na); + TAILCALL(env, na, nz, n->e); + return 1; + } else if (sf->step == 0) { /* - * Subtraction produces quite a mess. We could trim or minimise here - * while q is self-contained, which might work out better than doing it - * in the larger FSM after merge. I'm not sure if it works out better - * overall or not. + * Make new beginning/end states for the repeated section, + * build its NFA, and link to its head. */ - if (fsm_empty(q)) { - EPSILON(x, y); - break; + fsm_subgraph_start(env->fsm, &sf->u.repeat.subgraph); + + sf->step++; /* resume after RECURSE */ + NEWSTATE(sf->u.repeat.na); + NEWSTATE(sf->u.repeat.nz); + RECURSE(env, sf->u.repeat.na, sf->u.repeat.nz, n->e); + return 1; + } else { + fsm_state_t tail; + assert(sf->step == 1); + EPSILON(sf->x, sf->u.repeat.na); /* link head to repeated NFA head */ + + b = sf->u.repeat.nz; /* set the initial tail */ + + /* can be skipped */ + if (min == 0) { + EPSILON(sf->u.repeat.na, sf->u.repeat.nz); } + fsm_subgraph_stop(env->fsm, &sf->u.repeat.subgraph); + tail = sf->u.repeat.nz; - if (!fsm_unionxy(env->fsm, q, x, y)) { - return 0; + if (max != AST_COUNT_UNBOUNDED) { + for (i = 1; i < max; i++) { + /* copies the original subgraph; need to set b to the + * original tail + */ + b = tail; + + if (!fsm_subgraph_duplicate(env->fsm, &sf->u.repeat.subgraph, &b, &a)) { + return 0; + } + + EPSILON(sf->u.repeat.nz, a); + + /* To the optional part of the repeated count */ + if (i >= min) { + EPSILON(sf->u.repeat.nz, b); + } + + sf->u.repeat.na = a; /* advance head for next duplication */ + sf->u.repeat.nz = b; /* advance tail for concenation */ + } + } else { + for (i = 1; i < min; i++) { + /* copies the original subgraph; need to set b to the + * original tail + */ + b = tail; + + if (!fsm_subgraph_duplicate(env->fsm, &sf->u.repeat.subgraph, &b, &a)) { + return 0; + } + + EPSILON(sf->u.repeat.nz, a); + + sf->u.repeat.na = a; /* advance head for next duplication */ + sf->u.repeat.nz = b; /* advance tail for concenation */ + } + + /* back link to allow for infinite repetition */ + EPSILON(sf->u.repeat.nz, sf->u.repeat.na); } - break; + /* tail to last repeated NFA tail */ + EPSILON(sf->u.repeat.nz, sf->y); + RETURN(env); + return 1; } +} + +static void +set_active_capture_ids(struct comp_env *env, unsigned id) +{ +#if LOG_LINKAGE + fprintf(stderr, "set_active_capture_ids: id %u\n", id); +#endif + assert(env->active_capture_ids != NULL); + u64bitset_set(env->active_capture_ids, id); +} - case AST_EXPR_RANGE: { - unsigned int i; +static int +eval_GROUP(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); - if (n->u.range.from.type != AST_ENDPOINT_LITERAL || n->u.range.to.type != AST_ENDPOINT_LITERAL) { - /* not yet supported */ - return 0; + if (env->re_flags & RE_NOCAPTURE) { + /* passthrough, disable captures */ + if (sf->step == 0) { + sf->step++; + RECURSE(env, sf->x, sf->y, sf->n->u.group.e); + } else { + RETURN(env); } + return 1; + } - assert(n->u.range.from.u.literal.c <= n->u.range.to.u.literal.c); + if (sf->step == 0) { + struct ast_expr *n = sf->n; + set_active_capture_ids(env, n->u.group.id); - if (n->u.range.from.u.literal.c == 0x00 && - n->u.range.to.u.literal.c == 0xff) - { - ANY(x, y); +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: recurse GROUP: x %d, y %d\n", + sf->x, sf->y); +#endif + sf->step++; + RECURSE(env, sf->x, sf->y, n->u.group.e); + return 1; + } else { + assert(sf->step == 1); + + RETURN(env); + return 1; + } +} + +static int +eval_ANCHOR(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); +#if 1 + +#if LOG_LINKAGE + fprintf(stderr, "%s: ignoring anchor node %p, epsilon %d -> %d\n", + __func__, (void *)sf->n, sf->x, sf->y); +#endif + EPSILON(sf->x, sf->y); +#else + switch (sf->n->u.anchor.type) { + case AST_ANCHOR_START: + if (!(sf->n->flags & AST_FLAG_FIRST)) { +#if LOG_LINKAGE + fprintf(stderr, "%s: ignoring START anchor in non-FIRST location\n", + __func__); +#endif + EPSILON(sf->x, sf->y); break; } - for (i = n->u.range.from.u.literal.c; i <= n->u.range.to.u.literal.c; i++) { - LITERAL(x, y, i); +#if LOG_LINKAGE + fprintf(stderr, "%s: START anchor %p epsilon-linking %d -> %d\n", + __func__, (void *)sf->n, env->start_inner, sf->y); +#endif + EPSILON(env->start_inner, sf->y); + break; + + case AST_ANCHOR_END: + if (!(sf->n->flags & AST_FLAG_LAST)) { +#if LOG_LINKAGE + fprintf(stderr, "%s: ignoring END anchor in non-LAST location\n", + __func__); +#endif + EPSILON(sf->x, sf->y); + break; } +#if LOG_LINKAGE + fprintf(stderr, "%s: END anchor %p epsilon-linking %d -> %d\n", + __func__, (void *)sf->n, sf->x, stack->end_inner); +#endif + EPSILON(sf->x, stack->end_inner); break; - } default: assert(!"unreached"); + return 0; } +#endif + RETURN(env); + return 1; +} + +static int +eval_SUBTRACT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + + struct fsm *a, *b; + struct fsm *q; + enum re_flags re_flags = sf->n->re_flags; + + /* wouldn't want to reverse twice! */ + re_flags &= ~(unsigned)RE_REVERSE; + + /* Don't compile capture resolution programs again for the + * subtrees, just ignore capture behavior. */ + re_flags |= RE_NOCAPTURE; + + a = expr_compile(sf->n->u.subtract.a, re_flags, + fsm_getoptions(env->fsm), env->err); + if (a == NULL) { + return 0; + } + + b = expr_compile(sf->n->u.subtract.b, re_flags, + fsm_getoptions(env->fsm), env->err); + if (b == NULL) { + fsm_free(a); + return 0; + } + + q = fsm_subtract(a, b); + if (q == NULL) { + return 0; + } + + /* + * Subtraction produces quite a mess. We could trim or minimise here + * while q is self-contained, which might work out better than doing it + * in the larger FSM after merge. I'm not sure if it works out better + * overall or not. + */ + + if (fsm_empty(q)) { + EPSILON(sf->x, sf->y); + RETURN(env); + return 1; + } + + if (!fsm_unionxy(env->fsm, q, sf->x, sf->y)) { + return 0; + } + + RETURN(env); + return 1; +} + +static int +eval_RANGE(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; + unsigned int i; + + if (n->u.range.from.type != AST_ENDPOINT_LITERAL || n->u.range.to.type != AST_ENDPOINT_LITERAL) { + /* not yet supported */ + return 0; + } + + assert(n->u.range.from.u.literal.c <= n->u.range.to.u.literal.c); + + if (n->u.range.from.u.literal.c == 0x00 && + n->u.range.to.u.literal.c == 0xff) + { + ANY(sf->x, sf->y); + RETURN(env); + return 1; + } + + for (i = n->u.range.from.u.literal.c; i <= n->u.range.to.u.literal.c; i++) { + LITERAL(sf->x, sf->y, i); + } + + RETURN(env); + return 1; +} + +static int +eval_TOMBSTONE(struct comp_env *env) +{ + /* do not link -- intentionally pruned */ + (void)env; + RETURN(env); return 1; } @@ -908,6 +1458,8 @@ comp_iter(struct comp_env *env, #undef NEWSTATE #undef LITERAL #undef RECURSE +#undef RETURN +#undef TAILCALL struct fsm * ast_compile(const struct ast *ast, @@ -915,40 +1467,64 @@ ast_compile(const struct ast *ast, const struct fsm_options *opt, struct re_err *err) { - fsm_state_t x, y; + /* Start states inside and outside of match group 0, + * which represents the entire matched input, but does not + * include the implied /.*?/ loop at the start or end when + * a regex is unanchored. */ + fsm_state_t start_outer, start_inner; struct fsm *fsm; assert(ast != NULL); +#if LOG_LINKAGE + ast_print_tree(stderr, opt, re_flags, ast); +#endif + fsm = fsm_new(opt); if (fsm == NULL) { return NULL; } - if (!fsm_addstate(fsm, &x)) { + /* TODO: move these to the call stack, for symmetry? + * Or possibly combine comp_env and stack. */ + if (!fsm_addstate(fsm, &start_outer)) { goto error; } - if (!fsm_addstate(fsm, &y)) { + if (!fsm_addstate(fsm, &start_inner)) { goto error; } - fsm_setstart(fsm, x); - fsm_setend(fsm, y, 1); + if (!fsm_addedge_epsilon(fsm, start_outer, start_inner)) { + goto error; + } + + fsm_setstart(fsm, start_outer); + +#if LOG_LINKAGE + fprintf(stderr, "start: outer %d, inner %d\n", + start_outer, start_inner); +#endif { struct comp_env env; memset(&env, 0x00, sizeof(env)); + env.alloc = fsm->opt->alloc; env.fsm = fsm; env.re_flags = re_flags; env.err = err; - env.start = x; - env.end = y; + env.start_inner = start_inner; + env.start_outer = start_outer; - if (!comp_iter(&env, x, y, ast->expr, NULL)) { + env.max_capture_id = ast->max_capture_id; + + if (!comp_iter(&env, start_inner, ast)) { + if (err != NULL && err->e == 0) { + err->e = RE_EBADGROUP; + } goto error; } } @@ -980,3 +1556,25 @@ ast_compile(const struct ast *ast, return NULL; } +static int +compile_capvm_program_for_stack_end_states(struct comp_env *env, + const struct ast *ast, uint32_t *prog_id) +{ + /* compile and save program in ^, associate its id w/ end state */ + enum re_capvm_compile_ast_res res; + struct capvm_program *prog; + res = re_capvm_compile_ast(env->alloc, + ast, env->re_flags, &prog); + if (res != RE_CAPVM_COMPILE_AST_OK) { + if (env->err != NULL && env->err->e == 0 && errno != 0) { + env->err->e = RE_EERRNO; + } + return 0; + } + + if (!fsm_capture_add_program(env->fsm, prog, prog_id)) { + return 0; + } + + return 1; +} diff --git a/src/libre/ast_rewrite.c b/src/libre/ast_rewrite.c index d05fc0a82..adb0690d2 100644 --- a/src/libre/ast_rewrite.c +++ b/src/libre/ast_rewrite.c @@ -146,7 +146,7 @@ compile_subexpr(struct ast_expr *e, enum re_flags flags) return 0; } - fsm = ast_compile(&ast, flags | RE_ANCHORED, NULL, NULL); + fsm = ast_compile(&ast, flags | RE_ANCHORED | RE_NOCAPTURE, NULL, NULL); if (fsm == NULL) { return 0; } diff --git a/src/libre/print/tree.c b/src/libre/print/tree.c index 5d2f78691..58e1d6050 100644 --- a/src/libre/print/tree.c +++ b/src/libre/print/tree.c @@ -147,7 +147,9 @@ pp_iter(FILE *f, const struct fsm_options *opt, size_t indent, enum re_flags re_ case AST_EXPR_ALT: { size_t i, count = n->u.alt.count; - fprintf(f, "ALT (%u):\n", (unsigned)count); + fprintf(f, "ALT (%u):%s\n", + (unsigned)count, + n->u.alt.contains_empty_groups ? " (contains_empty_groups)" : ""); for (i = 0; i < count; i++) { pp_iter(f, opt, indent + 1 * IND, re_flags, n->u.alt.n[i]); } @@ -155,7 +157,9 @@ pp_iter(FILE *f, const struct fsm_options *opt, size_t indent, enum re_flags re_ } case AST_EXPR_LITERAL: - fprintf(f, "LITERAL '%c'\n", n->u.literal.c); + fprintf(f, "LITERAL '"); + print_char_or_esc(f, n->u.literal.c); + fprintf(f, "'\n"); break; case AST_EXPR_CODEPOINT: @@ -167,18 +171,27 @@ pp_iter(FILE *f, const struct fsm_options *opt, size_t indent, enum re_flags re_ fprintf_count(f, n->u.repeat.min); fprintf(f, ","); fprintf_count(f, n->u.repeat.max); - fprintf(f, "}\n"); + fprintf(f, "}%s\n", n->u.repeat.contains_empty_groups ? " (contains_empty_groups)" : ""); pp_iter(f, opt, indent + 1 * IND, re_flags, n->u.repeat.e); break; case AST_EXPR_GROUP: - fprintf(f, "GROUP %p: %u\n", (void *) n, n->u.group.id); + fprintf(f, "GROUP: %u%s\n", n->u.group.id, + n->u.group.repeated ? " (repeated)" : ""); pp_iter(f, opt, indent + 1 * IND, re_flags, n->u.group.e); break; case AST_EXPR_ANCHOR: assert(n->u.anchor.type == AST_ANCHOR_START || n->u.anchor.type == AST_ANCHOR_END); - fprintf(f, "ANCHOR %s\n", n->u.anchor.type == AST_ANCHOR_START ? "^" : "$"); + if (n->u.anchor.type == AST_ANCHOR_START) { + fprintf(f, "ANCHOR ^\n"); + } else if (n->u.anchor.type == AST_ANCHOR_END) { + assert(n->u.anchor.type == AST_ANCHOR_START || n->u.anchor.type == AST_ANCHOR_END); + fprintf(f, "ANCHOR $%s\n", + n->u.anchor.is_end_nl ? " (with \\n)" : ""); + } else { + assert(!"unreachable"); + } break; case AST_EXPR_SUBTRACT: diff --git a/src/libre/re.c b/src/libre/re.c index ade526ea7..9701672a2 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -37,12 +37,12 @@ re_dialect(enum re_dialect dialect) size_t i; static const struct dialect a[] = { - { RE_LIKE, parse_re_like, 0, RE_SINGLE | RE_ANCHORED }, - { RE_LITERAL, parse_re_literal, 0, RE_SINGLE | RE_ANCHORED }, - { RE_GLOB, parse_re_glob, 0, RE_SINGLE | RE_ANCHORED }, + { RE_LIKE, parse_re_like, 0, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE }, + { RE_LITERAL, parse_re_literal, 0, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE }, + { RE_GLOB, parse_re_glob, 0, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE }, { RE_NATIVE, parse_re_native, 0, 0 }, { RE_PCRE, parse_re_pcre, 0, RE_END_NL }, - { RE_SQL, parse_re_sql, 1, RE_SINGLE | RE_ANCHORED } + { RE_SQL, parse_re_sql, 1, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE } }; for (i = 0; i < sizeof a / sizeof *a; i++) { @@ -125,7 +125,15 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, if (res < 0) { ast_free(ast); - if (err != NULL) { err->e = RE_EERRNO; } + if (err != NULL) { + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + err->e = RE_EUNSUPPPCRE; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE) { + err->e = RE_EUNSUPCAPTUR; + } else if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + } + } return NULL; } diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c new file mode 100644 index 000000000..c67db2669 --- /dev/null +++ b/src/libre/re_capvm_compile.c @@ -0,0 +1,1689 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "re_capvm_compile.h" +#include "../libfsm/capture_vm.h" +#include "../libfsm/capture_vm_program.h" +#include "../libfsm/capture_vm_log.h" + +/* for EXPENSIVE_CHECKS */ +#include "../libfsm/internal.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +#include "ast.h" + +#define DEF_OPCODE_CEIL 8 +#define DEF_CHARCLASS_BUCKETS 8 +#define DEF_CHARCLASS_CEIL 4 +#define DEF_REPEATED_ALT_BACKPATCH_CEIL 1 +#define NO_BUCKET_ID ((uint32_t)-1) +#define NO_CAPTURE_ID ((uint32_t)-1) + +#define LOG_REPETITION_CASES 0 + +/* Placeholder markers for pending offset values (which would + * otherwise temporarily be uninitialized memory), chosen so + * they stand out visually in a debugger. */ +enum pending_offset { + PENDING_OFFSET_REPEAT_OPTIONAL_NEW = 11111111, + PENDING_OFFSET_REPEAT_OPTIONAL_CONT = 22222222, + PENDING_OFFSET_ALT_BACKPATCH_JMP = 33333333, + PENDING_OFFSET_ALT_BACKPATCH_NEW = 44444444, + PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS = 55555555, +}; + +struct capvm_compile_env { + const struct fsm_alloc *alloc; + enum re_flags re_flags; + struct capvm_program *program; + + uint32_t max_capture_seen; + + /* Hash table for interning character classes. + * Doubles and rehashes when half full. */ + struct charclass_htab { + uint32_t bucket_count; + uint32_t buckets_used; + uint32_t ids_used; + struct charclass_htab_bucket { + uint32_t id; /* or NO_BUCKET_ID for unused */ + struct capvm_char_class bitset; + } *buckets; + } charclass_htab; + +#define DEF_REPEATED_GROUPS_CEIL 8 + /* Linked list of nodes used at compile time to compile regexes + * such as '^(a((b*)*)*)$' as if they were '^(a(?:b*)(()))$'. + * Since the inner body of the repeated subexpression with the + * capture groups can be empty, it will always repeat after + * its body matches any input. We move the group captures to + * the end of the repeated subexpression to explicitly represent + * them always capturing afterward, because otherwise the + * infinite loop protection skips them. */ + struct repeated_group_info { + /* Ancestor node that should emit the SAVE opcodes; can + * be either a REPEAT or ALT. */ + const struct ast_expr *outermost_ancestor; + size_t ceil; + size_t count; + const struct ast_expr **groups; + /* linked list */ + struct repeated_group_info *prev; + } *repeated_groups; + + /* Linked list of nodes used for regexes like '^(($)|x)+$', + * which need to special-case the JMP instruction after the + * nullable '($)' case to replace it with a SPLIT to before + * and after the + repetition. */ + struct repeated_alt_backpatch_info { + const struct ast_expr *repeat; /* must be a + repeat */ + size_t ceil; + size_t used; + unsigned *opcode_offsets; + /* linked list */ + struct repeated_alt_backpatch_info *prev; + } *repeated_alt_backpatches; +}; + +static bool +ensure_program_capacity(const struct fsm_alloc *alloc, + struct capvm_program *p, uint32_t count) +{ +#define STRESS_GROWING (EXPENSIVE_CHECKS && 1) + + const uint32_t capacity = p->used + count; + + if (capacity > p->ceil) { +#if STRESS_GROWING + const uint32_t nceil = (p->ceil + 1 < capacity + ? capacity : p->ceil + 1); +#else + const uint32_t nceil = (p->ceil == 0 + ? DEF_OPCODE_CEIL + : 2*p->ceil); + /* This should always be enough for any capacity + * requested during compilation. */ + assert(nceil >= p->used + count); +#endif + LOG(3, "%s: growing %u -> %u (count %u)\n", + __func__, p->ceil, nceil, count); + struct capvm_opcode *nops = f_realloc(alloc, + p->ops, nceil * sizeof(p->ops[0])); + if (nops == NULL) { + return false; + } + +#if EXPENSIVE_CHECKS + for (size_t i = p->ceil; i < nceil; i++) { + /* out of range, will trigger asserts */ + nops[i].t = 'X'; + } +#endif + + p->ceil = nceil; + p->ops = nops; + } + return true; +} + +static void +check_program_for_invalid_labels(const struct capvm_program *p) +{ + for (uint32_t op_i = 0; op_i < p->used; op_i++) { + const struct capvm_opcode *op = &p->ops[op_i]; + switch (op->t) { + case CAPVM_OP_JMP: + assert(op->u.jmp != op_i); + break; + case CAPVM_OP_JMP_ONCE: + assert(op->u.jmp_once != op_i); + break; + case CAPVM_OP_SPLIT: + assert(op->u.split.cont < p->used); + assert(op->u.split.cont != op_i); + assert(op->u.split.new < p->used); + assert(op->u.split.new != op_i); + break; + + case CAPVM_OP_CHAR: + case CAPVM_OP_CHARCLASS: + case CAPVM_OP_MATCH: + case CAPVM_OP_SAVE: + case CAPVM_OP_ANCHOR: + break; + default: + assert(!"out of range"); + break; + } + } +} + +static uint32_t +get_program_offset(const struct capvm_program *p) +{ + assert(p->used < p->ceil); + +#if EXPENSIVE_CHECKS + struct capvm_opcode *op = &p->ops[p->used]; + op->t = 'X'; /* out of range */ +#endif + + return p->used; +} + +static uint32_t +reserve_program_opcode(struct capvm_program *p) +{ + assert(p->used < p->ceil); + const uint32_t res = p->used; + p->used++; + +#if EXPENSIVE_CHECKS + struct capvm_opcode *op = &p->ops[res]; + op->t = 'X'; /* out of range */ +#endif + + return res; +} + +static bool +grow_program_char_classes(const struct fsm_alloc *alloc, + struct capvm_program *p) +{ + const uint32_t nceil = (p->char_classes.ceil == 0 + ? DEF_CHARCLASS_CEIL + : 2*p->char_classes.ceil); + struct capvm_char_class *nsets = f_realloc(alloc, + p->char_classes.sets, nceil * sizeof(nsets[0])); + if (nsets == NULL) { + return false; + } + + p->char_classes.sets = nsets; + p->char_classes.ceil = nceil; + return true; +} + +static bool +intern_char_class(struct capvm_compile_env *env, + struct capvm_program *p, uint64_t chars[4], + uint32_t *id) +{ + LOG(5, "%s: used %u/%u\n", __func__, + env->charclass_htab.buckets_used, env->charclass_htab.bucket_count); + if (env->charclass_htab.buckets_used >= env->charclass_htab.bucket_count/2) { + const uint32_t ocount = env->charclass_htab.bucket_count; + const uint32_t ncount = (ocount == 0 + ? DEF_CHARCLASS_BUCKETS + : 2*env->charclass_htab.bucket_count); + LOG(3, "%s: growing from %u -> %u\n", __func__, ocount, ncount); + struct charclass_htab_bucket *nbuckets = + f_malloc(env->alloc, ncount * sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return false; + } + for (uint32_t n_i = 0; n_i < ncount; n_i++) { + nbuckets[n_i].id = NO_BUCKET_ID; + } + + const uint32_t nmask = ncount - 1; + assert((ncount & nmask) == 0); + + struct charclass_htab_bucket *obuckets = env->charclass_htab.buckets; + for (uint32_t o_i = 0; o_i < ocount; o_i++) { + if (obuckets[o_i].id == NO_BUCKET_ID) { + continue; + } + const uint64_t h = hash_fnv1a_64((const uint8_t *)obuckets[o_i].bitset.octets, + sizeof(obuckets[o_i].bitset)); + + for (uint32_t n_i = 0; n_i < ncount; n_i++) { + const uint64_t b = (h + n_i) & nmask; + if (nbuckets[b].id == NO_BUCKET_ID) { + memcpy(&nbuckets[b].bitset, + &obuckets[o_i].bitset, + sizeof(obuckets[o_i].bitset)); + nbuckets[b].id = obuckets[o_i].id; + break; + } + } + } + f_free(env->alloc, obuckets); + env->charclass_htab.bucket_count = ncount; + env->charclass_htab.buckets = nbuckets; + } + + assert(env->charclass_htab.buckets_used < env->charclass_htab.bucket_count/2); + + const uint32_t count = env->charclass_htab.bucket_count; + const uint32_t mask = count - 1; + struct charclass_htab_bucket *buckets = env->charclass_htab.buckets; + + const uint64_t h = hash_fnv1a_64((const uint8_t *)chars, + sizeof(buckets[0].bitset)); + for (uint32_t i = 0; i < count; i++) { + const uint64_t b = (h + i) & mask; + LOG(5, "%s: buckets[%lu].id == %d\n", + __func__, b, buckets[b].id); + if (buckets[b].id == NO_BUCKET_ID) { + memcpy(&buckets[b].bitset, chars, sizeof(buckets[b].bitset)); + if (p->char_classes.count == p->char_classes.ceil) { + if (!grow_program_char_classes(env->alloc, p)) { + return false; + } + } + + memcpy(&p->char_classes.sets[p->char_classes.count], + chars, sizeof(buckets[b].bitset)); + p->char_classes.count++; + buckets[b].id = env->charclass_htab.ids_used; + env->charclass_htab.ids_used++; + env->charclass_htab.buckets_used++; + *id = buckets[b].id; + + return true; + } else if (0 == memcmp(chars, &buckets[b].bitset, sizeof(buckets[b].bitset))) { + *id = buckets[b].id; + return true; /* already present, reuse */ + } else { + /* collision */ + } + } + + assert(!"unreachable"); + return false; +} + +static void +dump_endpoint(const struct ast_endpoint *e) +{ + switch (e->type) { + case AST_ENDPOINT_LITERAL: + fprintf(stderr, "endpoint[LITERAL]: 0x%02x '%c'\n", + e->u.literal.c, + isprint(e->u.literal.c) ? e->u.literal.c : '.'); + break; + case AST_ENDPOINT_CODEPOINT: + fprintf(stderr, "endpoint[CODEPOINT]: 0x%x\n", + e->u.codepoint.u); + break; + case AST_ENDPOINT_NAMED: + assert(!"todo?"); + break; + } +} + +static void +dump_pos(const struct ast_pos *p) +{ + fprintf(stderr, "pos: byte %u, line %u, col %u\n", + p->byte, p->line, p->col); +} + +static bool +active_node(const struct ast_expr *n) +{ + assert(n != NULL); + + switch (n->type) { + case AST_EXPR_TOMBSTONE: + return false; + default: + return true; + } +} + +static bool +subtree_represents_character_class(const struct ast_expr *expr, uint64_t cc[4]) +{ + for (size_t i = 0; i < 4; i++) { + cc[i] = 0; + } + + switch (expr->type) { + case AST_EXPR_EMPTY: + /* empty set */ + return false; + + case AST_EXPR_LITERAL: + u64bitset_set(cc, (uint8_t)expr->u.literal.c); + return true; + + case AST_EXPR_RANGE: + { + const struct ast_endpoint *f = &expr->u.range.from; + const struct ast_endpoint *t = &expr->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL + || t->type != AST_ENDPOINT_LITERAL) { + return false; + } + for (uint64_t c = (uint64_t)f->u.literal.c; c <= (uint64_t)t->u.literal.c; c++) { + u64bitset_set(cc, (uint8_t)c); + } + return true; + } + + case AST_EXPR_ALT: + { + /* union character classes from children */ + assert(expr->u.alt.count > 0); + for (size_t c_i = 0; c_i < expr->u.alt.count; c_i++) { + uint64_t child_cc[4]; + const struct ast_expr *child = expr->u.alt.n[c_i]; + if (subtree_represents_character_class(child, child_cc)) { + for (size_t cc_i = 0; cc_i < 4; cc_i++) { + cc[cc_i] |= child_cc[cc_i]; + } + } else { + return false; + } + } + return true; + } + + case AST_EXPR_SUBTRACT: + { + /* Only support AST_EXPR_SUBTRACT nodes where .a is a + * RANGE:0x00-0xff and .b is either a LITERAL, RANGE, EMPTY, + * or an ALT that itself represents a character class, */ + + const struct ast_expr *sub_a = expr->u.subtract.a; + if (sub_a->type != AST_EXPR_RANGE) { + return false; + } + + const struct ast_endpoint *f = &sub_a->u.range.from; + const struct ast_endpoint *t = &sub_a->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL || t->type != AST_ENDPOINT_LITERAL) { + return false; + } + + for (uint64_t i = 0; i < 256; i++) { + if (i >= (uint64_t)f->u.literal.c && i <= (uint64_t)f->u.literal.c) { + u64bitset_set(cc, i); + } + } + + for (size_t i = 0; i < 4; i++) { + cc[i] = ~(uint64_t)0; + } + + uint64_t neg_cc[4]; + if (subtree_represents_character_class(expr->u.subtract.b, neg_cc)) { + for (size_t cc_i = 0; cc_i < 4; cc_i++) { + cc[cc_i] &=~ neg_cc[cc_i]; + } + } else { + return false; + } + return true; + } + + default: + return false; + } +} + +static void +make_charclass_case_insensitive(uint64_t *cc) +{ + for (size_t i = 0; i < 256; i++) { + if (isalpha(i) && u64bitset_get(cc, i)) { + const char c = (char)i; + const uint64_t cl = (uint64_t)tolower(c); + const uint64_t cu = (uint64_t)toupper(c); + u64bitset_set(cc, cl); + u64bitset_set(cc, cu); + } + } +} + +static bool +can_safely_skip_JMP_ONCE(const struct ast_expr *expr) +{ + /* There are potentially cases where it's safe to skip the + * JMP_ONCE special case, which would save memory by not + * expanding the path an extra bit per iteration, but the + * criteria are subtle enough that it can probably wait. */ + (void)expr; + return false; +} + +static bool +push_repeated_group_info(struct capvm_compile_env *env, const struct ast_expr *expr) +{ + LOG(3 - LOG_REPETITION_CASES, + "%s: setting env->repeated_groups.outermost_ancestor <- %p\n", + __func__, (void *)expr); + + assert(expr != NULL); + assert(expr->type == AST_EXPR_REPEAT || expr->type == AST_EXPR_ALT); + + struct repeated_group_info *rgi = f_calloc(env->alloc, 1, sizeof(*rgi)); + if (rgi == NULL) { + return false; + } + rgi->outermost_ancestor = expr; + rgi->prev = env->repeated_groups; + env->repeated_groups = rgi; + LOG(3 - LOG_REPETITION_CASES, + "%s: push rgi, allocated %p, prev %p\n", + __func__, (void *)rgi, (void *)rgi->prev); + return true; +} + +static void +pop_repeated_group_info(struct capvm_compile_env *env, const struct ast_expr *expr) +{ + assert(expr != NULL); + assert(expr->type == AST_EXPR_REPEAT || expr->type == AST_EXPR_ALT); + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: pop rgi, expecting %p, got %p\n", + __func__, (void *)expr, (void *)rgi->outermost_ancestor); + assert(rgi->outermost_ancestor == expr); + struct repeated_group_info *prev = rgi->prev; + LOG(3 - LOG_REPETITION_CASES, + "%s: pop rgi, freeing %p, prev %p\n", + __func__, (void *)rgi, (void *)prev); + + env->repeated_groups = prev; + if (rgi->groups != NULL) { + f_free(env->alloc, rgi->groups); + } + f_free(env->alloc, rgi); +} + +static bool +push_repeated_alt_backpatch_info(struct capvm_compile_env *env, const struct ast_expr *expr) +{ + assert(expr + && expr->type == AST_EXPR_REPEAT + && expr->u.repeat.min == 1 + && expr->u.repeat.max == AST_COUNT_UNBOUNDED); + struct repeated_alt_backpatch_info *rabi = f_calloc(env->alloc, + 1, sizeof(*rabi)); + if (rabi == NULL) { + return false; + } + rabi->repeat = expr; + rabi->prev = env->repeated_alt_backpatches; + LOG(3 - LOG_REPETITION_CASES, + "%s: pushing node %p onto %p, prev link %p\n", + __func__, (void *)expr, (void *)rabi, (void *)rabi->prev); + env->repeated_alt_backpatches = rabi; + return true; +} + +static bool +append_repeated_alt_backpatch_offset(struct capvm_compile_env *env, unsigned offset) +{ + struct repeated_alt_backpatch_info *rabi = env->repeated_alt_backpatches; + assert(rabi != NULL); + if (rabi->used == rabi->ceil) { + const size_t nceil = (rabi->ceil == 0 + ? DEF_REPEATED_ALT_BACKPATCH_CEIL + : 2*rabi->ceil); + LOG(3 - LOG_REPETITION_CASES, + "%s: growing %zu -> %zu\n", __func__, rabi->ceil, nceil); + + unsigned *noffsets = f_realloc(env->alloc, + rabi->opcode_offsets, nceil * sizeof(noffsets[0])); + if (noffsets == NULL) { + return false; + } + rabi->ceil = nceil; + rabi->opcode_offsets = noffsets; + } + + LOG(3 - LOG_REPETITION_CASES, + "%s: pushing offset %u\n", __func__, offset); + rabi->opcode_offsets[rabi->used] = offset; + rabi->used++; + return true; +} + +static void +pop_repeated_alt_backpatch_info(struct capvm_compile_env *env, const struct ast_expr *expr) +{ + struct repeated_alt_backpatch_info *rabi = env->repeated_alt_backpatches; + assert(rabi != NULL); + assert(rabi->repeat == expr); + struct repeated_alt_backpatch_info *prev = rabi->prev; + LOG(3 - LOG_REPETITION_CASES, + "%s: popping %p, prev link %p\n", + __func__, (void *)rabi, (void *)prev); + f_free(env->alloc, rabi->opcode_offsets); + f_free(env->alloc, rabi); + env->repeated_alt_backpatches = prev; +} + +static void +backpatch_repeated_nullable_alt_split(struct capvm_compile_env *env, + const struct ast_expr *expr, struct capvm_program *p, unsigned split_new_dst); + +static bool +emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p); + +static bool +capvm_compile_iter_save_groups_in_skipped_subtree(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr); + +static bool +compile_kleene_star(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr); + +static bool +capvm_compile_iter(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr) +{ + LOG(4, "%s: expr %p, type %s, %u/%u used, re_flags 0x%02x\n", + __func__, (void *)expr, ast_node_type_name(expr->type), + p->used, p->ceil, expr->re_flags); + + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + break; + case AST_EXPR_CONCAT: + for (size_t i = 0; i < expr->u.concat.count; i++) { + /* append instructions from each consecutive node */ + const struct ast_expr *n = expr->u.concat.n[i]; + if (!capvm_compile_iter(env, p, n)) { return false; } + } + break; + case AST_EXPR_ALT: + { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + assert(expr->u.alt.count > 0); + + if (expr->u.alt.contains_empty_groups) { + if (!push_repeated_group_info(env, expr)) { + return false; + } + } + + /* If this ALT node represents a character class (such as a + * rewritten . character's ALT[0x00 - 0x09, 0x0b - 0xff] or + * a rewritten [abc-ef]'s ... , then produce the corresponding + * character class literal. The direct representation of the + * subtree would take several instructions and introduce + * unnecessary splits, increasing memory usage at runtime. */ + uint64_t cc[4]; + if (subtree_represents_character_class(expr, cc)) { + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op_cc = &p->ops[pos]; + op_cc->t = CAPVM_OP_CHARCLASS; + + if (expr->re_flags & RE_ICASE) { + make_charclass_case_insensitive(cc); + } + if (!intern_char_class(env, p, cc, &op_cc->u.charclass_id)) { + return false; + } + + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + break; + } + + uint32_t active_count = 0; + uint32_t last_active; + struct alt_flow_info { + bool is_active; + uint32_t backpatch; + }; + struct alt_flow_info *flow_info = f_calloc(env->alloc, + expr->u.alt.count, sizeof(flow_info[0])); + assert(flow_info != NULL); + + for (uint64_t i = 0; i < expr->u.alt.count; i++) { + const struct ast_expr *n = expr->u.alt.n[i]; + if (active_node(n)) { + last_active = i; + active_count++; + flow_info[i].is_active = true; + } + } + + /* If there are no children active this should terminate + * with an empty program. */ + LOG(3, "%s: active_count == %d\n", __func__, active_count); + if (active_count == 0) { + LOG(3, "%s: active_count == 0, skipping\n", __func__); + + for (uint64_t i = 0; i < expr->u.alt.count; i++) { + const struct ast_expr *n = expr->u.alt.n[i]; + capvm_compile_iter_save_groups_in_skipped_subtree(env, p, n); + if (n->flags & AST_FLAG_NULLABLE) { + break; + } + } + + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + + /* FIXME: may need distinct error case to not leak. + * There is currently no test reaching this, try + * using the fuzzer to trigger it. */ + return true; + } else if (active_count == 1) { + /* even if one of the later subtrees is active, an earlier + * subtree can still shadow it. */ + bool shadowed = false; + + for (uint64_t i = 0; i < expr->u.alt.count; i++) { + if (i != last_active) { /* evaluate for empty groups */ + const struct ast_expr *n = expr->u.alt.n[i]; + capvm_compile_iter_save_groups_in_skipped_subtree(env, p, n); + if (n->flags & AST_FLAG_NULLABLE) { + shadowed = true; + break; + } + } + } + + if (!shadowed) { + LOG(5, "narrowing to last_active %u\n", last_active); + assert(last_active < expr->u.alt.count); + const struct ast_expr *n = expr->u.alt.n[last_active]; + if (!capvm_compile_iter(env, p, n)) { + return false; + } + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + break; + } else { + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + return true; /* may need distinct error case to not leak */ + } + } + + LOG(3, "%s: compiling AST_EXPR_ALT with %u active nodes, last_active %u\n", + __func__, active_count, last_active); + + /* note: binarized split: for a|b|c, treat this like "a else (b else c)", + * leading to generated code like: + * + * // note: trying each case in order, earlier cases are more greedy + * - split_cont j1 + * - split_new j2 + * j1: + * - + * - jmp pos_after_all // or split pos_after_all, PLUS_BACKPATCH, see below + * j2: + * - split_cont j3 + * - split_new j4 + * j3: + * - + * - jmp pos_after_all + * j4: + * //// DO NOT EMIT split instructions here, treat like a final else + * - + * // fall through to pos_after_all + * pos_after_all: + * + * + * When an ALT case: + * - is nullable (can match the empty string) + * - is the first nullable case (shadowing cases after) + * - is in a subtree of a REPEAT{1,inf} (+) node whose entire subtree is nullable + * then that case's `jmp pos_after_all` should be replaced with + * `split pos_after_all pos_after_repeat_backjmp`, which will need a special + * label for batch-patching by the REPEAT later. This is necessary for cases + * like '^(?:($|x))+$', where the `jmp pos_after_all` would lead to code after + * the ALT that has already been executed at the current input position. + * */ + for (uint32_t c_i = 0; c_i < expr->u.alt.count; c_i++) { + const bool is_final_else_case = c_i == last_active; + LOG(3, "%s: %p c_i %u/%zu, is_final_else_case %d\n", + __func__, (void *)expr, c_i, expr->u.alt.count, is_final_else_case); + if (!flow_info[c_i].is_active) { continue; } + + if (is_final_else_case) { + /* Just add the case for the child node and + * then fall through to pos_after_all. */ + const struct ast_expr *n = expr->u.alt.n[c_i]; + LOG(3, "%s: %p recursing...\n", __func__, (void *)expr); + if (!capvm_compile_iter(env, p, n)) { + return false; + } + LOG(3, "%s: %p recursing...done (final-else-case)\n", __func__, (void *)expr); + + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT %p: contains_empty_groups: %d, outermost_ancestor: %p == %p ? %d\n", + __func__, (void *)expr, + expr->u.alt.contains_empty_groups, + (void *)(rgi ? rgi->outermost_ancestor : NULL), + (void *)expr, + (rgi ? rgi->outermost_ancestor == expr : 0)); + if (expr->u.alt.contains_empty_groups) { + assert(rgi != NULL); + LOG(3 - LOG_REPETITION_CASES, + "%s: outermost_ancestor match, count %zu\n", __func__, rgi->count); + if (!emit_repeated_groups(env, p)) { + return false; + } + } + } else { + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + const uint32_t pos_split_before_case = reserve_program_opcode(p); + struct capvm_opcode *op_split_before = &p->ops[pos_split_before_case]; + op_split_before->t = CAPVM_OP_SPLIT; + + /* greedier branch: trying the next case, in order */ + op_split_before->u.split.cont = get_program_offset(p); + + /* less greedy branch: moving on to the next case. + * will backpatch .new to after this case's JMP later */ + op_split_before->u.split.new = PENDING_OFFSET_ALT_BACKPATCH_NEW; + + const struct ast_expr *n = expr->u.alt.n[c_i]; + LOG(3, "%s: %p recursing...\n", __func__, (void *)expr); + if (!capvm_compile_iter(env, p, n)) { + return false; + } + LOG(3, "%s: %p recursing...done (non-final)\n", __func__, (void *)expr); + + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT %p: contains_empty_groups: %d, outermost_ancestor: %p == %p ? %d\n", + __func__, (void *)expr, expr->u.alt.contains_empty_groups, + (void *)(rgi ? rgi->outermost_ancestor : NULL), + (void *)expr, + (rgi ? rgi->outermost_ancestor == expr : 0)); + if (expr->u.alt.contains_empty_groups) { + assert(rgi != NULL); + LOG(3 - LOG_REPETITION_CASES, + "%s: outermost_ancestor match, count %zu\n", __func__, rgi->count); + if (!emit_repeated_groups(env, p)) { + return false; + } + } + + /* JMP or SPLIT, plus space after */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + /* Based on analysis, either emit a JMP or SPLIT. */ + if (n->u.alt.nullable_alt_inside_plus_repeat) { + const uint32_t pos_split_after = reserve_program_opcode(p); + flow_info[c_i].backpatch = pos_split_after; + struct capvm_opcode *op_split_after = &p->ops[pos_split_after]; + op_split_after->t = CAPVM_OP_SPLIT; + op_split_after->u.split.cont = PENDING_OFFSET_ALT_BACKPATCH_JMP; + op_split_after->u.split.new = PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS; + if (!append_repeated_alt_backpatch_offset(env, pos_split_after)) { + return false; + } + } else { + const uint32_t pos_jmp_after = reserve_program_opcode(p); + flow_info[c_i].backpatch = pos_jmp_after; + struct capvm_opcode *op_jmp = &p->ops[pos_jmp_after]; + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = PENDING_OFFSET_ALT_BACKPATCH_JMP; + } + + /* refresh pointer after possible realloc */ + op_split_before = &p->ops[pos_split_before_case]; + + /* and the original split jumps to after + * this case's JMP */ + op_split_before->u.split.new = get_program_offset(p); + } + } + + /* Ensure there's space for the next instruction, and then + * set every case's JMP suffix to it. */ + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_after_all = get_program_offset(p); + + for (size_t i = 0; i < expr->u.alt.count - 1; i++) { + const bool is_final_else_case = i == last_active; + assert(flow_info[i].backpatch < p->used); + if (is_final_else_case || !flow_info[i].is_active) { + continue; + } + + struct capvm_opcode *op_patch = &p->ops[flow_info[i].backpatch]; + if (op_patch->t == CAPVM_OP_JMP) { + assert(op_patch->u.jmp == PENDING_OFFSET_ALT_BACKPATCH_JMP); + op_patch->u.jmp = pos_after_all; + } else if (op_patch->t == CAPVM_OP_SPLIT) { + assert(op_patch->u.split.cont == PENDING_OFFSET_ALT_BACKPATCH_JMP); + op_patch->u.split.cont = pos_after_all; + /* This will be patched by an ancestor repeat node after returning. */ + assert(op_patch->u.split.cont == PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS); + } else { + assert(!"type mismatch"); + } + } + + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + break; + } + case AST_EXPR_LITERAL: + { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos]; + + if (expr->re_flags & RE_ICASE) { + uint64_t cc[4] = { 0 }; + u64bitset_set(cc, (uint64_t)expr->u.literal.c); + + op->t = CAPVM_OP_CHARCLASS; + make_charclass_case_insensitive(cc); + if (!intern_char_class(env, p, cc, &op->u.charclass_id)) { + return false; + } + } else { + op->t = CAPVM_OP_CHAR; + op->u.chr = (uint8_t)expr->u.literal.c; + } + break; + } + case AST_EXPR_CODEPOINT: + assert(!"not implemented, unreachable"); + break; + case AST_EXPR_REPEAT: + { + const unsigned min = expr->u.repeat.min; + const unsigned max = expr->u.repeat.max; + const struct ast_expr *e = expr->u.repeat.e; + + /* collect groups to emit */ + if (expr->u.repeat.contains_empty_groups) { + if (!push_repeated_group_info(env, expr)) { + return false; + } + } + + if (min == 1 && max == 1) { /* {1,1} */ + /* if repeating exactly once, just defer to subtree, + * but still do the repeated_group_info cleanup below */ + if (!capvm_compile_iter(env, p, e)) { + return false; + } + } else if (min == 0 && max == 1) { /* ? */ + /* split l1, l2 + * l1: + * l2: */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l1 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = pos_l1; + op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (!capvm_compile_iter(env, p, e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + op_split = &p->ops[pos_split]; /* refresh pointer */ + + const uint32_t after_expr = get_program_offset(p); + op_split->u.split.new = after_expr; + } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* * */ + if (!compile_kleene_star(env, p, expr)) { + return false; + } + } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* + */ + if (expr->u.repeat.contains_nullable_alt) { + if (!push_repeated_alt_backpatch_info(env, expr)) { + return false; + } + } + + /* l1: + * split l1, l2 + * l2: */ + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_l1 = get_program_offset(p); + + if (!capvm_compile_iter(env, p, e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + + /* Only emit the backwards jump for repetition branching + * if the subtree added any instructions. */ + if (get_program_offset(p) != pos_l1) { + if (!ensure_program_capacity(env->alloc, p, 3)) { + return false; + } + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l2 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = pos_l1; + op_split->u.split.new = pos_l2; + + /* Update any ALT nodes in the subtree whose SPLIT instructions + * are awaiting backpatching with pos_l2. */ + if (expr->u.repeat.contains_nullable_alt) { + backpatch_repeated_nullable_alt_split(env, expr, p, pos_l2); + } + } + + if (expr->u.repeat.contains_nullable_alt) { + pop_repeated_alt_backpatch_info(env, expr); + } + } else if (min == 0 && max == 0) { /* {0,0} */ + /* ignored, except any groups contained within that could match + * empty input still get emitted (unless unsatisfiable). */ + if (e->flags & AST_FLAG_UNSATISFIABLE) { + LOG(3, "%s: repeat{0,0} && UNSATISFIABILE -> skipping\n", __func__); + break; + } + + /* Unreachable group captures still need to be counted, otherwise + * subsequent ones would get shifted down. */ + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, e)) { return false; } + break; + } else { /* other bounded count */ + /* repeat the minimum number of times */ + for (size_t i = 0; i < min; i++) { + if (!capvm_compile_iter(env, p, e)) { return false; } + } + + if (max == AST_COUNT_UNBOUNDED) { + /* A repeat of {x,inf} should be treated like + * (?:subtree){x} (?:subtree)* . */ + if (!compile_kleene_star(env, p, expr)) { + return false; + } + } else { + /* then repeat up to the max as ? + * + * split_cont l1 + * split_new l2 + * l1: + * l2: */ + for (size_t i = min; i < max; i++) { + if (!ensure_program_capacity(env->alloc, p, 3)) { + return false; + } + + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l1 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = pos_l1; + op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (!capvm_compile_iter(env, p, e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + op_split = &p->ops[pos_split]; /* refresh pointer */ + + const uint32_t after_expr = get_program_offset(p); + op_split->u.split.new = after_expr; + } + } + } + + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: REPEAT %p: contains_empty_groups: %d, outermost_ancestor: %p == %p ? %d\n", + __func__, (void *)expr, expr->u.repeat.contains_empty_groups, + (void *)(rgi ? rgi->outermost_ancestor : NULL), + (void *)expr, + (rgi ? rgi->outermost_ancestor == expr : 0)); + if (expr->u.repeat.contains_empty_groups + && rgi != NULL + && rgi->outermost_ancestor == expr) { + LOG(3 - LOG_REPETITION_CASES, + "%s: outermost_ancestor match, count %zu\n", __func__, rgi->count); + if (!emit_repeated_groups(env, p)) { + return false; + } + pop_repeated_group_info(env, expr); + } + + break; + } + case AST_EXPR_GROUP: + { + const uint32_t id = expr->u.group.id; + const int is_repeated = expr->u.group.repeated; + + /* If the group is nullable and repeated, then move its save + * instructions to the end, since the final iteration matching + * nothing will always clobber any earlier saves. This is a + * workaround for cases that would otherwise incorrectly be + * halted by infinite loop prevention at runtime. */ + if (is_repeated && ((expr->flags & AST_FLAG_NULLABLE) + || !(expr->flags & AST_FLAG_CAN_CONSUME))) { + + struct repeated_group_info *rgi = env->repeated_groups; + + LOG(3 - LOG_REPETITION_CASES, + "%s: checking repeated group %u (capvm_compile_iter recurse), parent %p\n", + __func__, id, (void *)(rgi ? rgi->outermost_ancestor : NULL)); + if (!capvm_compile_iter(env, p, expr->u.group.e)) { return false; } + LOG(3 - LOG_REPETITION_CASES, + "%s: checking repeated group %u (capvm_compile_iter done), parent %p\n", + __func__, id, (void *)(rgi ? rgi->outermost_ancestor : NULL)); + + /* don't emit these here, parent repeat node will add them after. */ + if (rgi && rgi->outermost_ancestor != NULL) { + if (rgi->count == rgi->ceil) { + const size_t nceil = (rgi->ceil == 0 + ? DEF_REPEATED_GROUPS_CEIL + : 2*rgi->ceil); + const struct ast_expr **ngroups = f_realloc(env->alloc, + rgi->groups, + nceil * sizeof(ngroups[0])); + if (ngroups == NULL) { + return false; + } + rgi->groups = ngroups; + rgi->ceil = nceil; + } + + LOG(3 - LOG_REPETITION_CASES, + "%s: adding group %u (%p) to outermost_ancestor %p\n", + __func__, id, (void *)expr, + (void *)rgi->outermost_ancestor); + rgi->groups[rgi->count] = expr; + rgi->count++; + } + } else { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + if (!capvm_compile_iter(env, p, expr->u.group.e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*id + 1; + } + + if (id > env->max_capture_seen || env->max_capture_seen == NO_CAPTURE_ID) { + env->max_capture_seen = id; + } + + break; + } + + case AST_EXPR_ANCHOR: + { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos]; + op->t = CAPVM_OP_ANCHOR; + op->u.anchor = (expr->u.anchor.type == AST_ANCHOR_START + ? CAPVM_ANCHOR_START : CAPVM_ANCHOR_END); + break; + } + case AST_EXPR_SUBTRACT: + { + uint64_t cc[4]; + for (size_t i = 0; i < 4; i++) { + cc[i] = ~(uint64_t)0; + } + if (subtree_represents_character_class(expr, cc)) { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op_cc = &p->ops[pos]; + op_cc->t = CAPVM_OP_CHARCLASS; + + if (expr->re_flags & RE_ICASE) { + make_charclass_case_insensitive(cc); + } + + if (!intern_char_class(env, p, cc, &op_cc->u.charclass_id)) { + return false; + } + } else { + /* FIXME: should return UNSUPPORTED */ + assert(!"unreachable"); + return false; + } + break; + } + case AST_EXPR_RANGE: + { + uint64_t cc[4] = { 0 }; + if (!subtree_represents_character_class(expr, cc)) { + dump_endpoint(&expr->u.range.from); + dump_pos(&expr->u.range.start); + dump_endpoint(&expr->u.range.to); + dump_pos(&expr->u.range.end); + assert(!"unreachable"); + return false; + } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos]; + + op->t = CAPVM_OP_CHARCLASS; + if (expr->re_flags & RE_ICASE) { + make_charclass_case_insensitive(cc); + } + + if (!intern_char_class(env, p, cc, &op->u.charclass_id)) { + return false; + } + break; + } + default: + assert(!"matchfail"); + } + + return true; +} + +static bool +compile_kleene_star(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr) +{ + /* Note: min count may be > 0 because this is also + * used for unbounded repetition with a lower count, + * as in `a{3,}`, but in that case the {min} + * repetitions have already been handled by the caller. */ + assert(expr && expr->type == AST_EXPR_REPEAT && + expr->u.repeat.max == AST_COUNT_UNBOUNDED); + + /* l1: split l2, l3 + * l2: + * jmp_once l1 OR jmp l1 + * l3: */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + const uint32_t pos_l1 = reserve_program_opcode(p); + const uint32_t pos_l2 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_l1]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = PENDING_OFFSET_REPEAT_OPTIONAL_CONT; + op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (!capvm_compile_iter(env, p, expr->u.repeat.e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + /* It's more expensive to always emit JMP_ONCE because it + * extends the path each iteration, so we could detect when + * it would be safe to use a JMP instead. */ + if (can_safely_skip_JMP_ONCE(expr)) { + const uint32_t pos_jmp = reserve_program_opcode(p); + struct capvm_opcode *op_jmp = &p->ops[pos_jmp]; + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = pos_l1; + } else { + const uint32_t pos_jmp_once = reserve_program_opcode(p); + struct capvm_opcode *op_jmp_once = &p->ops[pos_jmp_once]; + op_jmp_once->t = CAPVM_OP_JMP_ONCE; + op_jmp_once->u.jmp_once = pos_l1; + } + + const uint32_t pos_l3 = get_program_offset(p); + op_split = &p->ops[pos_l1]; /* refresh pointer */ + op_split->u.split.cont = pos_l2; + op_split->u.split.new = pos_l3; + return true; +} + +static bool +emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p) +{ + struct repeated_group_info *rgi = env->repeated_groups; + for (size_t i = 0; i < rgi->count; i++) { + const struct ast_expr *group = rgi->groups[i]; + assert(group->u.group.repeated); + const unsigned id = group->u.group.id; + LOG(3 - LOG_REPETITION_CASES, + "%s: checking %zu/%zu: group_id %u\n", + __func__, i, rgi->count, id); + + if (group->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END)) { + /* if the otherwise empty group contains any anchors, + * then emit a subtree like (^)? so that its capture + * is only set when the anchors would match. */ + if (!ensure_program_capacity(env->alloc, p, 6)) { + return false; + } + + /* split l1, l2 + * l1: + * + * l2: save (start) + * save (end) + * */ + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l1 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = pos_l1; + op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (group->flags & AST_FLAG_ANCHORED_START) { + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_ANCHOR; + op->u.anchor = CAPVM_ANCHOR_START; + } + + if (group->flags & AST_FLAG_ANCHORED_END) { + const uint32_t pos_end = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_end]; + op->t = CAPVM_OP_ANCHOR; + op->u.anchor = CAPVM_ANCHOR_END; + } + + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*group->u.group.id; + op->u.save = 2*id + 1; + + const uint32_t after_expr = get_program_offset(p); + op_split = &p->ops[pos_split]; /* refresh pointer */ + op_split->u.split.new = after_expr; + } else { + /* simple case, emit SAVE pair */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*group->u.group.id; + op->u.save = 2*id + 1; + } + } + + /* clear, because an ALT's subtrees can have distinct repeated groups */ + rgi->count = 0; + + return true; +} + +static void +backpatch_repeated_nullable_alt_split(struct capvm_compile_env *env, + const struct ast_expr *expr, struct capvm_program *p, unsigned split_new_dst) +{ + struct repeated_alt_backpatch_info *rabi = env->repeated_alt_backpatches; + assert(rabi != NULL && rabi->repeat == expr); + + for (size_t op_i = 0; op_i < rabi->used; op_i++) { + const unsigned offset = rabi->opcode_offsets[op_i]; + assert(offset < p->used); + LOG(3 - LOG_REPETITION_CASES, + "%s: backpatching SPLIT instruction %u's .new to %u\n", + __func__, offset, split_new_dst); + struct capvm_opcode *op = &p->ops[offset]; + assert(op->t == CAPVM_OP_SPLIT); + assert(op->u.split.new == PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS); + op->u.split.new = split_new_dst; + } + + rabi->used = 0; +} + +static bool +capvm_compile_iter_save_groups_in_skipped_subtree(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr) +{ + /* Follow the subtree as far as any expressions that could + * contain GROUPs. Emit any empty groups. This is necessary for + * regexes like /()*^/ and /(x|(x|))^/ whose subtrees are + * otherwise pruned but would still match the empty string + * before ^. */ + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_ANCHOR: + case AST_EXPR_SUBTRACT: + case AST_EXPR_RANGE: + case AST_EXPR_TOMBSTONE: + /* none of these can contain groups */ + break; + + case AST_EXPR_CONCAT: + if (expr->flags & AST_FLAG_UNSATISFIABLE) { + return true; /* skip */ + } + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.concat.n[i])) { + return false; + } + } + break; + case AST_EXPR_ALT: + for (size_t i = 0; i < expr->u.alt.count; i++) { + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.alt.n[i])) { + return false; + } + } + break; + + case AST_EXPR_REPEAT: + return capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.repeat.e); + + case AST_EXPR_GROUP: + { + const uint32_t id = expr->u.group.id; + LOG(5, "%s: recording otherwise skipped group %u\n", __func__, id); + + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + if (id > env->max_capture_seen || env->max_capture_seen == NO_CAPTURE_ID) { + env->max_capture_seen = id; + } + + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*id + 1; + + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.group.e)) { + return false; + } + + break; + } + default: + assert(!"match fail"); + + } + return true; +} + +static enum re_capvm_compile_ast_res +capvm_compile(struct capvm_compile_env *env, + const struct ast *ast) +{ + struct capvm_program *p = f_calloc(env->alloc, 1, sizeof(*p)); + if (p == NULL) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + + LOG(3, "%s: has_unanchored: start? %d, end? %d\n", __func__, + ast->has_unanchored_start, + ast->has_unanchored_end); + + /* If the regex has an unanchored start, it gets a `.*` prefix, + * but with the labels swapped so that the unanchored start + * loop is NOT greedy. */ + if (ast->has_unanchored_start) { + if (!ensure_program_capacity(env->alloc, p, 4)) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + + /* l1: split l3, l2 + * l2: . + * jmp l1 + * l3: */ + const uint32_t l1 = get_program_offset(p); + const uint32_t split_pos = reserve_program_opcode(p); + struct capvm_opcode *op_split = &p->ops[split_pos]; + + const uint32_t l2 = get_program_offset(p); + const uint32_t op_cc_pos = reserve_program_opcode(p); + struct capvm_opcode *op_cc = &p->ops[op_cc_pos]; + + const uint32_t op_jmp_pos = reserve_program_opcode(p); + struct capvm_opcode *op_jmp = &p->ops[op_jmp_pos]; + + const uint32_t l3 = get_program_offset(p); + + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = l3; /* greedy */ + op_split->u.split.new = l2; /* non-greedy */ + + op_cc->t = CAPVM_OP_CHARCLASS; + uint64_t any[4]; + for (size_t i = 0; i < 4; i++) { + any[i] = ~(uint64_t)0; + } + if (!intern_char_class(env, p, any, &op_cc->u.charclass_id)) { + goto cleanup; + } + + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = l1; + } + + /* Compile the regex AST, assuming match group 0 is + * explicitly represented. */ + if (!capvm_compile_iter(env, p, ast->expr)) { + goto cleanup; + } + + /* Add the unanchored end loop, outside of match group 0 */ + if (ast->has_unanchored_end) { + if (!ensure_program_capacity(env->alloc, p, 4)) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + + /* l1: split l3, l2 + * l2: . + * jmp l1 + * l3: */ + const uint32_t l1 = reserve_program_opcode(p); + const uint32_t l2 = reserve_program_opcode(p); + const uint32_t l_jmp = reserve_program_opcode(p); + const uint32_t l3 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[l1]; + + struct capvm_opcode *op_any = &p->ops[l2]; + struct capvm_opcode *op_jmp = &p->ops[l_jmp]; + + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.cont = l3; /* greedy */ + op_split->u.split.new = l2; /* non-greedy */ + + op_any->t = CAPVM_OP_CHARCLASS; + uint64_t any[4]; + for (size_t i = 0; i < 4; i++) { + any[i] = ~(uint64_t)0; + } + if (!intern_char_class(env, p, any, &op_any->u.charclass_id)) { + goto cleanup; + } + + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = l1; + } + + /* add MATCH opcode at end */ + if (!ensure_program_capacity(env->alloc, p, 1)) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + const uint32_t pos_m = reserve_program_opcode(p); + struct capvm_opcode *op_m = &p->ops[pos_m]; + op_m->t = CAPVM_OP_MATCH; + + /* TODO: populate info about max threads, etc. in p, + * because it should be possible to calculate runtime + * memory limits at compile time. */ + env->program = p; + p->capture_count = (env->max_capture_seen == NO_CAPTURE_ID + ? 0 : env->max_capture_seen + 1); + + if (LOG_CAPVM > 2) { + LOG(0, "====\n"); + fsm_capvm_program_dump(stderr, p); + LOG(0, "====\n"); + } + + /* TODO: it may be worth exposing these static checks as + * something the caller can run at load-time */ + check_program_for_invalid_labels(p); + + return RE_CAPVM_COMPILE_AST_OK; + +cleanup: + fsm_capvm_program_free(env->alloc, p); + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; +} + +#define DUMP_AST 0 +#define DUMP_RESULT 0 /* should be 0 in production */ + +#if DUMP_AST || DUMP_RESULT +#include +#include "print.h" +static struct fsm_options opt = { .group_edges = 1 }; + +static unsigned +get_max_capture_id(const struct capvm_program *program) +{ + assert(program != NULL); + return (program->capture_count == 0 + ? 0 + : program->capture_base + program->capture_count - 1); +} + +#endif + +enum re_capvm_compile_ast_res +re_capvm_compile_ast(const struct fsm_alloc *alloc, + const struct ast *ast, + enum re_flags re_flags, + struct capvm_program **program) +{ +#if DUMP_AST + if (LOG_CAPVM > 2) { + ast_print_dot(stderr, &opt, re_flags, ast); + ast_print_tree(stderr, &opt, re_flags, ast); + } +#endif + + struct capvm_compile_env env = { + .alloc = alloc, + .re_flags = re_flags, + .max_capture_seen = NO_CAPTURE_ID, + }; + + enum re_capvm_compile_ast_res res; + res = capvm_compile(&env, ast); + + + struct repeated_group_info *rgi = env.repeated_groups; + while (rgi != NULL) { + struct repeated_group_info *prev = rgi->prev; + LOG(3 - LOG_REPETITION_CASES, + "%s: rgi cleanup, freeing %p, prev %p\n", + __func__, (void *)rgi, (void *)prev); + + if (rgi->groups != NULL) { + f_free(alloc, rgi->groups); + } + f_free(alloc, rgi); + rgi = prev; + } + + if (res == RE_CAPVM_COMPILE_AST_OK) { +#if DUMP_RESULT > 0 + if (DUMP_RESULT > 1 || getenv("DUMP")) { + ast_print_tree(stderr, &opt, re_flags, ast); + fsm_capvm_program_dump(stderr, env.program); + fprintf(stderr, "%s: max_capture_id %u\n", __func__, + get_max_capture_id(env.program)); + + } +#endif + + *program = env.program; + } + + free(env.charclass_htab.buckets); + + return res; +} diff --git a/src/libre/re_capvm_compile.h b/src/libre/re_capvm_compile.h new file mode 100644 index 000000000..b98ac9196 --- /dev/null +++ b/src/libre/re_capvm_compile.h @@ -0,0 +1,32 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef RE_CAPVM_COMPILE_H +#define RE_CAPVM_COMPILE_H + +/* The part of the capture VM interface that belongs in + * libre rather than libfsm, mostly related to compiling + * a libre AST into a capvm_program. */ + +#include + +#include "ast.h" +#include + +struct capvm_program; + +enum re_capvm_compile_ast_res { + RE_CAPVM_COMPILE_AST_OK, + RE_CAPVM_COMPILE_AST_ERROR_ALLOC = -1, +}; + +enum re_capvm_compile_ast_res +re_capvm_compile_ast(const struct fsm_alloc *alloc, + const struct ast *ast, + enum re_flags re_flags, + struct capvm_program **program); + +#endif diff --git a/src/libre/strerror.c b/src/libre/strerror.c index 009d61df2..d66e750a4 100644 --- a/src/libre/strerror.c +++ b/src/libre/strerror.c @@ -20,6 +20,8 @@ re_strerror(enum re_errno e) case RE_EERRNO: return strerror(errno); case RE_EBADDIALECT: return "Bad dialect"; case RE_EBADGROUP: return "Bad group"; + case RE_EUNSUPCAPTUR: return "Cannot support captures in this case"; + case RE_EUNSUPPPCRE: return "Unsupported PCRE edge case"; case RE_ENEGRANGE: return "Negative group range"; case RE_ENEGCOUNT: return "Negative count range"; diff --git a/src/lx/parser.act b/src/lx/parser.act index 78254e123..66d0591f0 100644 --- a/src/lx/parser.act +++ b/src/lx/parser.act @@ -388,7 +388,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->opt, @f, &err); + @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->opt, @f | RE_NOCAPTURE, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ diff --git a/src/re/main.c b/src/re/main.c index c83c4fc66..226267196 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,10 @@ * TODO: flags; -r for RE_REVERSE, etc */ +static int +exec_with_captures(struct fsm *fsm, + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end); + struct match { fsm_end_id_t i; const char *s; @@ -642,6 +647,7 @@ main(int argc, char *argv[]) int patterns; int ambig; int makevm; + int resolve_captures; size_t generate_bounds = 0; struct fsm_dfavm *vm; @@ -665,6 +671,7 @@ main(int argc, char *argv[]) patterns = 0; ambig = 0; makevm = 0; + resolve_captures = 0; print_fsm = NULL; print_ast = NULL; query = NULL; @@ -675,7 +682,7 @@ main(int argc, char *argv[]) { int c; - while (c = getopt(argc, argv, "h" "acCwXe:E:G:k:" "bi" "sq:r:l:F:" "upMmnftxyz"), c != -1) { + while (c = getopt(argc, argv, "h" "acCwXe:E:G:k:" "bi" "sq:r:l:F:" "upMmnRftxyz"), c != -1) { switch (c) { case 'a': opt.anonymous_states = 0; break; case 'c': opt.consolidate_edges = 0; break; @@ -714,6 +721,7 @@ main(int argc, char *argv[]) case 't': isliteral = 1; break; case 'z': patterns = 1; break; case 'M': makevm = 1; break; + case 'R': resolve_captures = 1; break; case 'G': generate_bounds = strtoul(optarg, NULL, 10); @@ -1235,8 +1243,10 @@ main(int argc, char *argv[]) if (vm != NULL) { e = fsm_vm_match_file(vm, f); + } else if (resolve_captures) { + assert(!"todo"); } else { - e = fsm_exec(fsm, fsm_fgetc, f, &state, NULL); + e = fsm_exec(fsm, fsm_fgetc, f, &state); } fclose(f); @@ -1247,8 +1257,10 @@ main(int argc, char *argv[]) if (vm != NULL) { e = fsm_vm_match_buffer(vm, s, strlen(s)); + } else if (resolve_captures) { + e = exec_with_captures(fsm, fsm_sgetc, &s, &state); } else { - e = fsm_exec(fsm, fsm_sgetc, &s, &state, NULL); + e = fsm_exec(fsm, fsm_sgetc, &s, &state); } } @@ -1281,3 +1293,56 @@ main(int argc, char *argv[]) return r; } } + +static int +exec_with_captures(struct fsm *fsm, + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end) +{ + int c; + size_t ceil = 16; + size_t used = 0; + unsigned char *buf = malloc(ceil); + size_t i; + size_t capture_ceil; + struct fsm_capture *captures; + int res; + + while (c = fsm_getc(opaque), c != EOF) { + if (used == ceil - 1) { + const size_t nceil = 2*ceil; + unsigned char *nbuf = realloc(buf, nceil); + if (nbuf == NULL) { + free(buf); + return -1; + } + ceil = nceil; + buf = nbuf; + } + buf[used] = c; + used++; + } + buf[used] = '\0'; + + capture_ceil = fsm_capture_ceiling(fsm); + + captures = malloc(capture_ceil * sizeof(captures[0])); + if (captures == NULL) { + free(buf); + return -1; + } + + res = fsm_exec_with_captures(fsm, buf, used, + end, captures, capture_ceil); + if (res == 1) { + for (i = 0; i < capture_ceil; i++) { + printf("-- %zu: %zd,%zd\n", + i, captures[i].pos[0], captures[i].pos[1]); + } + } else { + printf("-- no match\n"); + } + + free(buf); + free(captures); + return res; +} diff --git a/tests/capture/Makefile b/tests/capture/Makefile index 53d63ff2b..bdee42df4 100644 --- a/tests/capture/Makefile +++ b/tests/capture/Makefile @@ -7,18 +7,20 @@ TEST_OUTDIR.tests/capture = ${BUILD}/tests/capture .for n in ${TEST.tests/capture:T:R:C/^capture//} test:: ${TEST_OUTDIR.tests/capture}/res${n} SRC += ${TEST_SRCDIR.tests/capture}/capture${n}.c -CFLAGS.${TEST_SRCDIR.tests/capture}/capture${n}.c = -UNDEBUG +CFLAGS.${TEST_SRCDIR.tests/capture}/capture${n}.c = -UNDEBUG -std=c99 ${TEST_OUTDIR.tests/capture}/run${n}: ${TEST_OUTDIR.tests/capture}/capture${n}.o ${TEST_OUTDIR.tests/capture}/captest.o - ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/capture}/run${n} ${TEST_OUTDIR.tests/capture}/capture${n}.o ${TEST_OUTDIR.tests/capture}/captest.o ${BUILD}/lib/libfsm.a + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/capture}/run${n} ${TEST_OUTDIR.tests/capture}/capture${n}.o ${TEST_OUTDIR.tests/capture}/captest.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a ${TEST_OUTDIR.tests/capture}/res${n}: ${TEST_OUTDIR.tests/capture}/run${n} ( ${TEST_OUTDIR.tests/capture}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/capture}/res${n} -.for lib in ${LIB:Mlibfsm} +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} ${TEST_OUTDIR.tests/capture}/run${n}: ${BUILD}/lib/${lib:R}.a .endfor .endfor +CFLAGS.${TEST_SRCDIR.tests/capture}/captest.c += -std=c99 + ${TEST_OUTDIR.tests/capture}/captest.o: tests/capture/captest.c ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/capture}/captest.o tests/capture/captest.c diff --git a/tests/capture/captest.c b/tests/capture/captest.c index dc66f81d4..bf453e3ad 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -1,19 +1,19 @@ #include "captest.h" -#include -#include -#include - -#if CAPTEST_RUN_SINGLE_LOG #include -#endif +#include + +#include + +/* for fsm_capvm_program_exec */ +#include "../../src/libfsm/capture_vm.h" -#define FAIL(MSG) \ - fprintf(stderr, "FAIL: %s:%d -- %s\n", \ - __FILE__, __LINE__, MSG); \ - exit(EXIT_FAILURE) +struct captest_input { + const char *string; + size_t pos; +}; -int +static int captest_getc(void *opaque) { struct captest_input *input = opaque; @@ -22,189 +22,483 @@ captest_getc(void *opaque) return res == 0 ? EOF : res; } -int -captest_run_single(const struct captest_single_fsm_test_info *info) -{ - size_t i; - struct captest_input input; - fsm_state_t end; - int exec_res; - struct fsm_capture got_captures[MAX_TEST_CAPTURES]; - struct fsm_capture exp_captures[MAX_TEST_CAPTURES]; - size_t capture_count = 0; - struct fsm *fsm = captest_fsm_of_string(info->string, 0); +static struct fsm_options options = { + .group_edges = 1, +}; - input.string = info->string; - input.pos = 0; +#define MAX_INPUT_WITH_NL_LENGTH 1000 +static char +input_with_nl[MAX_INPUT_WITH_NL_LENGTH]; - if (fsm == NULL) { - FAIL("fsm_of_string"); +enum captest_run_case_res +captest_run_case(const struct captest_case_single *testcase, + int verbosity, bool trailing_newline, FILE *prog_output) +{ + bool dump_captures = false; + enum captest_run_case_res res = CAPTEST_RUN_CASE_ERROR; + struct re_err err; + + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + assert(prog_output != NULL); + } else { + assert(prog_output == NULL); } - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - exp_captures[i].pos[0] = FSM_CAPTURE_NO_POS; - exp_captures[i].pos[1] = FSM_CAPTURE_NO_POS; + if (verbosity > 0) { + printf("/%s/ <- \"%s%s\": ", + testcase->regex, testcase->input, + trailing_newline ? "\\n" : ""); } - for (i = 0; i < MAX_SINGLE_FSM_TEST_PATHS; i++) { - const struct captest_single_fsm_test_path *path = - &info->paths[i]; - if (path->start == 0 && path->end == 0 && i > 0) { - break; /* end of list */ - } + /* build regex */ + const enum re_flags flags = 0; + struct captest_input comp_input = { + .string = testcase->regex, + }; - /* no zero-width captures */ - assert(path->end > path->start); + struct fsm *fsm = re_comp(RE_PCRE, + captest_getc, &comp_input, + &options, flags, &err); - if (!fsm_capture_set_path(fsm, i, - path->start, path->end)) { - fprintf(stderr, - "failed to set capture path %lu\n", i); - FAIL("fsm_capture_set_path"); + if (testcase->match == SHOULD_REJECT_AS_UNSUPPORTED) { + if (fsm != NULL) { + fsm_free(fsm); + return CAPTEST_RUN_CASE_FAIL; } + return CAPTEST_RUN_CASE_PASS; + } - exp_captures[i].pos[0] = path->start; - exp_captures[i].pos[1] = path->end; + assert(fsm != NULL); - capture_count = i + 1; + if (!fsm_determinise(fsm)) { + return CAPTEST_RUN_CASE_ERROR; } - { - const unsigned count = fsm_countcaptures(fsm); - const unsigned expected = capture_count; - if (count != expected) { - fprintf(stderr, "expected %u, got %u\n", - expected, count); - FAIL("countcaptures"); - } + if (!fsm_minimise(fsm)) { + return CAPTEST_RUN_CASE_ERROR; } -#if CAPTEST_RUN_SINGLE_LOG - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "fsm", fsm); -#endif + if (verbosity > 3) { + fsm_print_fsm(stdout, fsm); + } - exec_res = fsm_exec(fsm, captest_getc, &input, &end, got_captures); - if (exec_res != 1) { FAIL("exec_res"); } - if (end != strlen(info->string)) { FAIL("exec end pos"); } + if (trailing_newline) { + const size_t length = strlen(testcase->input); + assert(length + 1 < MAX_INPUT_WITH_NL_LENGTH); + memcpy(input_with_nl, testcase->input, + length); + input_with_nl[length] = '\n'; + input_with_nl[length + 1] = '\0'; + } - { - fsm_end_id_t id_buf[1] = { ~0 }; - enum fsm_getendids_res gres; - size_t written; - if (1 != fsm_getendidcount(fsm, end)) { - FAIL("did not have exactly one end ID"); + const char *input = trailing_newline + ? input_with_nl + : testcase->input; + assert(input != NULL); + const size_t length = strlen(input); + + fsm_state_t end; /* unused but required by API */ + struct fsm_capture capture_buf[MAX_CAPTEST_SINGLE_CAPTURE_PAIRS]; + const size_t capture_buf_length = MAX_CAPTEST_SINGLE_CAPTURE_PAIRS; + + /* Initialize with values that are distinct from FSM_CAPTURE_NO_POS + * and will stand out visually. Should never see these. */ + for (size_t i = 0; i < MAX_CAPTEST_SINGLE_CAPTURE_PAIRS; i++) { + capture_buf[i].pos[0] = 88888888; + capture_buf[i].pos[1] = 99999999; + } + + /* If verbosity is exactly DUMP_PROGRAMS_VERBOSITY, then print out capture info and pass. */ + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + assert(prog_output != NULL); + if (!trailing_newline) { + const char *match_str = testcase->match == SHOULD_MATCH ? "SHOULD_MATCH" + : testcase->match == SHOULD_NOT_MATCH ? "SHOULD_NOT_MATCH" + : testcase->match == SHOULD_REJECT_AS_UNSUPPORTED ? "SHOULD_REJECT_AS_UNSUPPORTED" + : "ERROR"; + fprintf(prog_output, "regex \"%s\", input \"%s\", match %s, no_nl %d, count %zu:", + testcase->regex, testcase->input, match_str, testcase->no_nl, + testcase->count); + for (size_t i = 0; i < testcase->count; i++) { + fprintf(prog_output, " %zu:[%zd, %zd]", + i, testcase->expected[i].pos[0], testcase->expected[i].pos[1]); + } + fprintf(prog_output, "\n"); + fsm_capture_dump(prog_output, "capture_info", fsm); } + fsm_free(fsm); + return CAPTEST_RUN_CASE_PASS; + } + + /* first, execute with a capture buffer that is one cell too small and check for an error */ + const size_t capture_ceil = fsm_capture_ceiling(fsm); + assert(capture_ceil > 0); + const size_t insufficient_capture_buf_length = capture_ceil - 1; + errno = 0; + int exec_res = fsm_exec_with_captures(fsm, + (const unsigned char *)input, length, &end, capture_buf, insufficient_capture_buf_length); + assert(exec_res == -1); + assert(errno == EINVAL); + errno = 0; + + /* then, execute and check result & captures */ + exec_res = fsm_exec_with_captures(fsm, + (const unsigned char *)input, length, &end, capture_buf, capture_buf_length); + if (exec_res == -1) { + perror("fsm_exec_with_captures"); + return CAPTEST_RUN_CASE_ERROR; + } - gres = fsm_getendids(fsm, end, 1, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { - FAIL("failed to get end IDs"); + if (testcase->match == SHOULD_NOT_MATCH) { /* expect match failure */ + res = (exec_res == 0 + ? CAPTEST_RUN_CASE_PASS + : CAPTEST_RUN_CASE_FAIL); + } else if (exec_res == 0) { + res = CAPTEST_RUN_CASE_FAIL; /* didn't match, should have */ + } else { + res = CAPTEST_RUN_CASE_PASS; + if (verbosity > 1) { + dump_captures = true; } - if (0 != id_buf[0]) { - FAIL("failed to get end ID of 0"); + /* check captures against expected */ + for (size_t i = 0; i < testcase->count; i++) { + if (testcase->expected[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected[i].pos[1] != capture_buf[i].pos[1]) { + res = CAPTEST_RUN_CASE_FAIL; + dump_captures = true; + } } } - for (i = 0; i < capture_count; i++) { -#if CAPTEST_RUN_SINGLE_LOG - fprintf(stderr, "captest: capture %lu: exp (%ld, %ld), got (%ld, %ld)\n", - i, exp_captures[i].pos[0], exp_captures[i].pos[1], - got_captures[i].pos[0], got_captures[i].pos[1]); -#endif - if (got_captures[i].pos[0] != exp_captures[i].pos[0]) { - fprintf(stderr, "capture[%lu].pos[0]: exp %lu, got %lu\n", - i, exp_captures[i].pos[0], - got_captures[i].pos[0]); - FAIL("capture mismatch"); + switch (res) { + case CAPTEST_RUN_CASE_PASS: + if (verbosity > 0) { + printf("pass\n"); } - if (got_captures[i].pos[1] != exp_captures[i].pos[1]) { - fprintf(stderr, "capture[%lu].pos[1]: exp %lu, got %lu\n", - i, exp_captures[i].pos[1], - got_captures[i].pos[1]); - FAIL("capture mismatch"); + break; + case CAPTEST_RUN_CASE_FAIL: + if (verbosity == 0) { + printf("/%s/ <- \"%s%s\": FAIL\n", + testcase->regex, testcase->input, + trailing_newline ? "\\n" : ""); + } + if (verbosity > 0) { + printf("FAIL\n"); + } + break; + case CAPTEST_RUN_CASE_ERROR: + printf("ERROR\n"); + break; + } + + if (dump_captures) { + for (size_t i = 0; i < testcase->count; i++) { + printf("exp %zd, %zd, got %zd, %zd%s\n", + testcase->expected[i].pos[0], testcase->expected[i].pos[1], + capture_buf[i].pos[0], capture_buf[i].pos[1], + (testcase->expected[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected[i].pos[1] != capture_buf[i].pos[1]) + ? " *" : ""); } } fsm_free(fsm); - return 0; + return res; } -struct fsm * -captest_fsm_of_string(const char *string, unsigned end_id) +enum captest_run_case_res +captest_run_case_multi(const struct captest_case_multi *testcase, + int verbosity, bool trailing_newline, FILE *prog_output, + struct captest_case_multi_result *result) { - struct fsm *fsm = captest_fsm_with_options(); - const size_t length = strlen(string); - size_t i; + struct re_err err; + const enum re_flags flags = 0; - if (fsm == NULL) { - return NULL; + struct captest_case_multi_result ignored_result; + if (result == NULL) { + result = &ignored_result; } + memset(result, 0x00, sizeof(*result)); - if (!fsm_addstate_bulk(fsm, length + 1)) { - goto cleanup; + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + assert(prog_output != NULL); + } else { + assert(prog_output == NULL); } - fsm_setstart(fsm, 0); - for (i = 0; i < length; i++) { - if (!fsm_addedge_literal(fsm, i, i + 1, string[i])) { + /* build each regex, combining them and keeping track of capture offsets */ + struct fsm *fsms[testcase->regex_count]; + struct fsm_combined_base_pair bases[testcase->regex_count]; + struct fsm *combined_fsm = NULL; + + for (size_t i = 0; i < testcase->regex_count; i++) { + fsms[i] = NULL; + } + + /* compile each individually */ + for (size_t i = 0; i < testcase->regex_count; i++) { + struct captest_input comp_input = { + .string = testcase->regexes[i], + }; + + if (verbosity > 1) { + fprintf(stderr, "%s: compiling \"%s\"\n", + __func__, comp_input.string); + } + + struct fsm *fsm = re_comp(RE_PCRE, + captest_getc, &comp_input, + &options, flags, &err); + assert(fsm != NULL); + + if (!fsm_determinise(fsm)) { goto cleanup; } + + if (!fsm_minimise(fsm)) { + goto cleanup; + } + + if (verbosity > 3) { + char tag_buf[16] = { 0 }; + snprintf(tag_buf, sizeof(tag_buf), "fsm[%zu]", i); + + fprintf(stderr, "==== fsm[%zu]\n", i); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, tag_buf, fsm); + } + + fsms[i] = fsm; + } + + combined_fsm = fsm_union_array(testcase->regex_count, fsms, bases); + assert(combined_fsm != NULL); + if (verbosity > 0) { + fprintf(stderr, "%s: combined_fsm: %d states after fsm_union_array\n", + __func__, fsm_countstates(combined_fsm)); + } + if (verbosity > 1) { + for (size_t i = 0; i < testcase->regex_count; i++) { + fprintf(stderr, "%s: base[%zu]: state %d, capture %u\n", + __func__, i, bases[i].state, bases[i].capture); + } } - fsm_setend(fsm, length, 1); - if (!fsm_setendid(fsm, end_id)) { + if (!fsm_determinise(combined_fsm)) { goto cleanup; } + if (verbosity > 0) { + fprintf(stderr, "%s: combined_fsm: %d states after determinise\n", + __func__, fsm_countstates(combined_fsm)); + } + + if (!fsm_minimise(combined_fsm)) { + goto cleanup; + } + if (verbosity > 0) { + fprintf(stderr, "%s: combined_fsm: %d states after minimise\n", + __func__, fsm_countstates(combined_fsm)); + } + + /* If verbosity is exactly 9, then print out capture info and pass. */ + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + fsm_capture_dump(prog_output, "capture_info", combined_fsm); + fsm_free(combined_fsm); + return CAPTEST_RUN_CASE_PASS; + } + + if (verbosity > 3) { + fprintf(stderr, "==== combined\n"); + fsm_print_fsm(stderr, combined_fsm); + fsm_capture_dump(stderr, "combined", combined_fsm); + } + + /* for each input, execute and check result */ + const struct multi_case_input_info *info; + for (info = &testcase->inputs[0]; info->input != NULL; info++) { + if (trailing_newline) { + const size_t length = strlen(info->input); + assert(length + 1 < MAX_INPUT_WITH_NL_LENGTH); + memcpy(input_with_nl, info->input, + length); + input_with_nl[length] = '\n'; + input_with_nl[length + 1] = '\0'; + } + + const char *input = trailing_newline + ? input_with_nl + : info->input; + assert(input != NULL); + const size_t length = strlen(input); + + if (verbosity > 1) { + fprintf(stderr, "%s: input: %s\n", __func__, input); + } + + fsm_state_t end; /* unused but required by API */ + struct fsm_capture capture_buf[MAX_CAPTEST_MULTI_CAPTURE_PAIRS]; + const size_t capture_buf_length = MAX_CAPTEST_MULTI_CAPTURE_PAIRS; + for (size_t i = 0; i < capture_buf_length; i++) { + capture_buf[i].pos[0] = (size_t)-2; + capture_buf[i].pos[1] = (size_t)-3; + } + + /* execute and check result & captures */ + int exec_res = fsm_exec_with_captures(combined_fsm, + (const unsigned char *)input, length, &end, capture_buf, capture_buf_length); + if (exec_res == -1) { + perror("fsm_exec_with_captures"); + return CAPTEST_RUN_CASE_ERROR; + } + + /* The .regex field should be in ascending order so we know + * when we've reached the all-0 suffix of expected[]. */ + uint8_t prev_regex = 0; + for (const struct case_multi_expected *exp = &info->expected[0]; + exp->regex >= prev_regex; exp++) { + prev_regex = exp->regex; + bool match = true; + const unsigned capture_base = bases[exp->regex].capture; + const unsigned capture_id = capture_base + exp->capture; + assert(capture_id < MAX_CAPTEST_MULTI_CAPTURE_PAIRS); + const size_t exp_s = exp->pos[0]; + const size_t exp_e = exp->pos[1]; + const size_t got_s = capture_buf[capture_id].pos[0]; + const size_t got_e = capture_buf[capture_id].pos[1]; + if (exp_s == got_s && exp_e == got_e) { + result->pass++; + } else { + match = false; + result->fail++; + } + + if (!match || verbosity > 2) { + fprintf(stderr, "%s: regex %u, capture %u (%u + base %u), exp (%zd, %zd), got (%zd, %zd)%s\n", + __func__, exp->regex, + capture_id, exp->capture, capture_base, + exp_s, exp_e, got_s, got_e, + match ? "" : " *** mismatch ***"); + } + } + } + + fsm_free(combined_fsm); - return fsm; + /* this could populate a result struct so it can pass/fail multiple inputs */ + + return result->fail == 0 + ? CAPTEST_RUN_CASE_PASS + : CAPTEST_RUN_CASE_FAIL; cleanup: - fsm_free(fsm); - return NULL; -} + if (combined_fsm != NULL) { + fsm_free(combined_fsm); + } else { + for (size_t i = 0; i < testcase->regex_count; i++) { + if (fsms[i] != NULL) { + fsm_free(fsms[i]); + } + } + } -static struct fsm_options options; + return CAPTEST_RUN_CASE_ERROR; +} -struct fsm * -captest_fsm_with_options(void) +static struct capvm_program * +get_program_copy(const struct captest_case_program *testcase) { - struct fsm *fsm = NULL; + static struct capvm_program prog; + static struct capvm_opcode ops[MAX_PROGRAM_OPS + 1] = { 0 }; + static struct capvm_char_class cc_sets[MAX_PROGRAM_CHAR_CLASSES] = { 0 }; + + memset(&prog, 0x00, sizeof(prog)); + + memcpy(ops, testcase->ops, + MAX_PROGRAM_OPS * sizeof(testcase->ops[0])); + memcpy(cc_sets, testcase->char_class, + MAX_PROGRAM_CHAR_CLASSES * sizeof(testcase->char_class[0])); + + assert(testcase->expected.count < MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS); + prog.capture_count = testcase->expected.count; + prog.capture_base = testcase->expected.base; + + uint32_t max_cc_used = (uint32_t)-1; + + prog.used = MAX_PROGRAM_OPS; + for (size_t i = 0; i < MAX_PROGRAM_OPS; i++) { + const struct capvm_opcode *op = &testcase->ops[i]; + if (op->t == CAPVM_OP_CHAR && op->u.chr == 0x00) { + prog.used = i; + break; + } else if (op->t == CAPVM_OP_CHARCLASS) { + if (max_cc_used == (uint32_t)-1 || op->u.charclass_id > max_cc_used) { + assert(op->u.charclass_id < MAX_PROGRAM_CHAR_CLASSES); + max_cc_used = op->u.charclass_id; + } + } + } + + prog.ceil = MAX_PROGRAM_OPS; + prog.ops = ops; + + prog.char_classes.sets = cc_sets; + prog.char_classes.count = max_cc_used == (uint32_t)-1 ? 0 : max_cc_used + 1; + prog.char_classes.ceil = MAX_PROGRAM_CHAR_CLASSES; - /* We currently don't need to set anything custom on this. */ - fsm = fsm_new(&options); - return fsm; + return &prog; } -int -captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, - unsigned expected_end_id, const char **msg) +enum captest_run_case_res +captest_run_case_program(const struct captest_case_program *testcase, + int verbosity) { - fsm_end_id_t id_buf[1] = { ~0 }; - enum fsm_getendids_res gres; - size_t written; - const char *unused; + (void)verbosity; - if (msg == NULL) { - msg = &unused; + /* copy program */ + const size_t input_length = strlen(testcase->input); + struct fsm_capture capture_buf[MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS]; + const size_t capture_buf_length = MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS; + + /* Initialize with FSM_CAPTURE_NO_POS, as the caller would */ + for (size_t i = 0; i < capture_buf_length; i++) { + capture_buf[i].pos[0] = FSM_CAPTURE_NO_POS; + capture_buf[i].pos[1] = FSM_CAPTURE_NO_POS; } - if (1 != fsm_getendidcount(fsm, end_state)) { - *msg = "did not have exactly one end ID"; - return 0; + struct capvm_program *program = get_program_copy(testcase); + + if (verbosity > 2) { + fsm_capvm_program_dump(stderr, program); } - gres = fsm_getendids(fsm, end_state, 1, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { - *msg = "failed to get end IDs"; - return 0; + fsm_capvm_program_exec(program, (const uint8_t *)testcase->input, input_length, + capture_buf, capture_buf_length); + + bool dump_captures = false; + enum captest_run_case_res res = CAPTEST_RUN_CASE_PASS; + + /* check captures against expected */ + for (size_t i = 0; i < testcase->expected.count; i++) { + if (testcase->expected.captures[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected.captures[i].pos[1] != capture_buf[i].pos[1]) { + res = CAPTEST_RUN_CASE_FAIL; + dump_captures = true; + } } - if (expected_end_id != id_buf[0]) { - *msg = "failed to get expected end ID"; - return 0; + if (dump_captures) { + for (size_t i = 0; i < testcase->expected.count; i++) { + printf("exp %zd, %zd, got %zd, %zd%s\n", + testcase->expected.captures[i].pos[0], + testcase->expected.captures[i].pos[1], + capture_buf[i].pos[0], capture_buf[i].pos[1], + (testcase->expected.captures[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected.captures[i].pos[1] != capture_buf[i].pos[1]) + ? " *" : ""); + } } - return 1; + return res; } diff --git a/tests/capture/captest.h b/tests/capture/captest.h index a9debff6c..53b30cec0 100644 --- a/tests/capture/captest.h +++ b/tests/capture/captest.h @@ -1,56 +1,123 @@ /* - * Copyright 2020 Scott Vokes + * Copyright 2022 Scott Vokes * * See LICENCE for the full copyright terms. */ + #ifndef CAPTEST_H #define CAPTEST_H #include #include #include +#include +#include #include #include #include #include +#include -#define MAX_SINGLE_FSM_TEST_PATHS 8 -#define MAX_TEST_CAPTURES 8 +#include -#define CAPTEST_RUN_SINGLE_LOG 0 +/* for captest_run_case_program, to evaluate handwritten programs */ +#include "../../src/libfsm/capture_vm_program.h" +#include "../../src/libfsm/capture_vm.h" -#ifndef LOG_INTERMEDIATE_FSMS -#define LOG_INTERMEDIATE_FSMS 0 -#endif +#define MAX_CAPTEST_SINGLE_CAPTURE_PAIRS 16 +#define MAX_CAPTEST_MULTI_CAPTURE_PAIRS 16 +#define MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS 16 + +/* position representing no match */ +#define POS_NONE { (size_t)-1, (size_t)-1 } + +/* If verbosity is set to this (with -vvvvvvvvv) then dump all the + * compiled programs to 'prog_output'. */ +#define DUMP_PROGRAMS_VERBOSITY 9 -struct captest_single_fsm_test_info { - const char *string; - struct captest_single_fsm_test_path { - fsm_state_t start; - fsm_state_t end; - } paths[MAX_SINGLE_FSM_TEST_PATHS]; +enum captest_match { + SHOULD_MATCH = 0, /* implied, set by designated initializer */ + SHOULD_NOT_MATCH = 1, + SHOULD_REJECT_AS_UNSUPPORTED = 2, + SHOULD_SKIP = 3, }; -struct captest_input { - const char *string; - size_t pos; +struct captest_case_single { + const char *regex; + const char *input; + enum captest_match match; + bool no_nl; /* do not retry with trailing newline */ + + size_t count; + struct fsm_capture expected[MAX_CAPTEST_SINGLE_CAPTURE_PAIRS]; }; -int -captest_run_single(const struct captest_single_fsm_test_info *info); +/* Same as captest_case_single, but these expect multiple (possibly overlapping) + * regexes to be combined before checking the match/capture behavior. */ +#define MAX_REGEXES 4 +#define MAX_INPUTS 8 +#define MAX_CAPTEST_MULTI_EXPECTED 8 +struct captest_case_multi { + uint8_t regex_count; + const char *regexes[MAX_REGEXES]; + enum captest_match match; + bool no_nl; + + struct multi_case_input_info { + const char *input; /* first NULL input = end of list */ + struct case_multi_expected { + uint8_t regex; /* expected: ascending order */ + uint8_t capture; /* 0 is default */ + size_t pos[2]; + } expected[MAX_CAPTEST_MULTI_EXPECTED]; + } inputs[MAX_INPUTS]; +}; -int -captest_getc(void *opaque); +struct captest_case_multi_result { + size_t pass; + size_t fail; +}; + +#define MAX_PROGRAM_CHAR_CLASSES 4 +#define MAX_PROGRAM_OPS 32 +struct captest_case_program { + const char *input; + + struct capvm_char_class char_class[MAX_PROGRAM_CHAR_CLASSES]; + + struct { + uint32_t count; + uint32_t base; + struct fsm_capture captures[MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS]; + } expected; + + /* termined by 0'd record, { .t == CAPVM_OP_CHAR, .u.chr = 0x00 } */ + struct capvm_opcode ops[MAX_PROGRAM_OPS]; +}; + +enum captest_run_case_res { + CAPTEST_RUN_CASE_PASS, + CAPTEST_RUN_CASE_FAIL, + CAPTEST_RUN_CASE_ERROR, +}; +enum captest_run_case_res +captest_run_case(const struct captest_case_single *testcase, + int verbosity, bool trailing_newline, FILE *prog_output); -struct fsm * -captest_fsm_with_options(void); +enum captest_run_case_res +captest_run_case_multi(const struct captest_case_multi *testcase, + int verbosity, bool trailing_newline, FILE *prog_output, + struct captest_case_multi_result *result); -struct fsm * -captest_fsm_of_string(const char *string, unsigned end_id); +/* This should probably only be used for evaluating specific + * hand-written programs for development, because we only care + * about supporting the kinds of programs that could be produced + * by compiling from valid regexes. In other words, this is not + * a stable public interface. */ +enum captest_run_case_res +captest_run_case_program(const struct captest_case_program *testcase, + int verbosity); -int -captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, - unsigned expected_end_id, const char **msg); #endif diff --git a/tests/capture/capture_test_case_list.c b/tests/capture/capture_test_case_list.c new file mode 100644 index 000000000..6a5a36a19 --- /dev/null +++ b/tests/capture/capture_test_case_list.c @@ -0,0 +1,1867 @@ +#include "captest.h" + +#include + +#define NO_POS FSM_CAPTURE_NO_POS + +const struct captest_case_single single_cases[] = { + { + .regex = "^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$^", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "()*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^$", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^($|($)|(($))|((($))))", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(((($)))|(($))|($)|$)", + .input = "", + .count = 5, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^((((a$)))|((b$))|(c$)|d$)", + .input = "a", + .count = 5, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^((((a$)))|((b$))|(c$)|d$)", + .input = "b", + .count = 7, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^((((b$)))|((b$))|(c$)|d$)", + .input = "b", + .count = 5, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^((x?))*$", + .input = "x", + .count = 3, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^((x?)*)*$", + .input = "", + .count = 3, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^((x?)*)*$", + .input = "xxxxx", + .count = 3, .expected = { + { .pos = {0, 5}, }, + { .pos = {5, 5}, }, + { .pos = {5, 5}, }, + }, + }, + { + .regex = "xx*x", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 2}, }, + }, + }, + { + .regex = "^(x?)*$", + .input = "xx", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {2, 2}, }, + }, + }, + { + .regex = "^(x?)*$", + .input = "xxx", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {3, 3}, }, + }, + }, + { + .regex = "^(x?)+$", + .input = "xx", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {2, 2}, }, + }, + }, + { + .regex = "^(x?)+$", + .input = "xxx", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {3, 3}, }, + }, + }, + { + .regex = "^x(z?)*y$", + .input = "xy", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "()|x", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()|x", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "x|()", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|()", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$|", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = ".|$^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = ".|$^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$^|.", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$^|.", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$$$^|...", + .input = "xxx", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "x?$x?^x?|x?$x?^x?", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "[^x]", .input = "", + .no_nl = true, + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "[^x]", + .input = "\n", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = ".$()", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = ".$()", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^.$()", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^.$()", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$(x?)(y?)(z?)", + .input = "a", + .count = 4, .expected = { + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = ".$(x?)(y?)(z?)", + .input = "a", + .count = 4, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "[^y]", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = ".", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "(x)+", + .input = "xxx", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {2, 3}, }, + }, + }, + { + .regex = "^(x)*.", + .input = "xx", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^(x)*.", + .input = "xy", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "a.b(c)*", + .input = "axbc", + .count = 2, .expected = { + { .pos = {0, 4}, }, + { .pos = {3, 4}, }, + }, + }, + { + .regex = "^x?^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^x?^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$(^)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "($)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "($$$)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$x?^", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$(^)*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$(^)*", + .input = "x", + .count = 1, .expected = { + { .pos = {1, 1}, }, + }, + }, + { + .regex = "$()*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$()*", + .input = "x", + .count = 2, .expected = { + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^$^", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$^$", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$y?^x*", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "x|$^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|$^", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "x|$^$^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|$^$^", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$^|x", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$^|x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$^$^|x", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$^$^|x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^$|.", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|^$^$", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^$^$|x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$|^|a$", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "[^a]x", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "[^a]x", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 2}, }, + }, + }, + { + .regex = "a(b|c$)d", .input = "ac", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "a(^b|c)d", .input = "bd", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(a|b|)*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "xx*y$", + .input = "x_xxy", + .count = 1, .expected = { + { .pos = {2, 5}, }, + }, + }, + { + .regex = "(|.$)*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(.$)*x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(.$)*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^(|.$)*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(|.$)*$", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "x|y(^)", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(?:x*.|^$).", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(?:x|^$)x", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "()+x", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "($$)^", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$($|$a)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?i)abc$", + .input = "AbC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)ab(?-i)c$", .input = "AbC", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^(?i)ab(?-i)c$", + .input = "Abc", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)a[b]c$", + .input = "ABC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)a[^b]c$", .input = "ABC", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^(?i)a[bx]c$", + .input = "ABC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)a[b-c]c$", + .input = "ABC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "(a()b)+a", + .input = "a!aba", + .count = 3, .expected = { + { .pos = {2, 5}, }, + { .pos = {2, 4}, }, + { .pos = {3, 3}, }, + }, + }, + { + .regex = "^^[^]]", + .input = "\n", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x(x()y)*", + .input = "xxy", + .count = 3, .expected = { + { .pos = {0, 3}, }, + { .pos = {1, 3}, }, + { .pos = {2, 2}, }, + }, + }, + { + .regex = "x(()x)*", + .input = "xx", + .count = 3, .expected = { + { .pos = {0, 2}, }, + { .pos = {1, 2}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "b(x*x*a()*y)*(a)a*", + .input = "ba", + .count = 4, .expected = { + { .pos = {0, 2}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {1, 2}, }, + }, + }, + { + .regex = "a(().x)*ab", + .input = "a.a.aaxab", + .count = 3, .expected = { + { .pos = {4, 9}, }, + { .pos = {5, 7}, }, + { .pos = {5, 5}, }, + }, + }, + { + .regex = "ab(b()*()*)*()*z", + .input = "a!abz", + .count = 5, .expected = { + { .pos = {2, 5}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {4, 4}, }, + }, + }, + { + .regex = "^x(y?z*)*$", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^(y?z*)*$", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(x|$x?)*$", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(^|$x)*$", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "((x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*)*y$", + .input = "xxxxxxxxxxy", + .count = 15, .expected = { + { .pos = {0, 11}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + }, + }, + { + .regex = "^a$", + .input = "a", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^a(bcd)e$", + .input = "abcde", + .count = 2, .expected = { + { .pos = {0, 5}, }, + { .pos = {1, 4}, }, + }, + }, + { + .regex = "^(a(b((c))(d)))$", + .input = "abcd", + .count = 6, .expected = { + { .pos = {0, 4}, }, + { .pos = {0, 4}, }, + { .pos = {1, 4}, }, + { .pos = {2, 3}, }, + { .pos = {2, 3}, }, + { .pos = {3, 4}, }, + }, + }, + { + .regex = "^(a(b(c)))$", + .input = "abc", + .count = 4, .expected = { + { .pos = {0, 3}, }, + { .pos = {0, 3}, }, + { .pos = {1, 3}, }, + { .pos = {2, 3}, }, + }, + }, + { + .regex = "^a(b*)(c)$", + .input = "ac", + .count = 3, .expected = { + { .pos = {0, 2}, }, + { .pos = {1, 1}, }, + { .pos = {1, 2}, }, + }, + }, + { + .regex = "^a(b*)(c)$", + .input = "abc", + .count = 3, .expected = { + { .pos = {0, 3}, }, + { .pos = {1, 2}, }, + { .pos = {2, 3}, }, + }, + }, + { + .regex = "^a(b*)(c)$", + .input = "abbc", + .count = 3, .expected = { + { .pos = {0, 4}, }, + { .pos = {1, 3}, }, + { .pos = {3, 4}, }, + }, + }, + { + .regex = "^(ab*c)$", + .input = "ac", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 2}, }, + }, + }, + { + .regex = "^(ab*c)$", + .input = "abc", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(ab*c)$", + .input = "abbc", + .count = 2, .expected = { + { .pos = {0, 4}, }, + { .pos = {0, 4}, }, + }, + }, + { + .regex = "^(a*)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(a*)", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(a*)", + .input = "a", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^(a*)", + .input = "ax", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^a*", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^a*", + .input = "a", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^a*", + .input = "ax", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = ".|", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()*^", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(((())))*^", + .input = "", + .count = 5, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + + { + .regex = "(x|(x|))^", + .input = "", + .count = 3, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = ".*(x|())^", + .input = "", + .count = 3, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(()|(()|x)^|x)^", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + + { + .regex = "x^()()|()", + .input = "", + .count = 4, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "y^()|()^x", + .input = "x", + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()$a|()", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()$z|(x)$", + .input = "x", + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 1}, }, + }, + }, + + { + /* long enough to exercise the USE_COLLAPSED_ZERO_PREFIX optimization */ + .regex = "a*(ba*)c$", + .input = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac", + .count = 2, + .expected = { + { .pos = {101, 303}, }, + { .pos = {201, 302}, }, + }, + }, + + /* regression: losing the first character on the transition from + * the unanchored start loop to the capture */ + { + .regex = "aa+b$", + .input = "aXaXaaab", + .count = 1, + .expected = { + { .pos = {4, 8}, }, + }, + }, + { + .regex = "aa*b$", + .input = "aXaXaaab", + .count = 1, + .expected = { + { .pos = {4, 8}, }, + }, + }, + { + .regex = "!!!+$", + .input = "!\"!\"!\"!!!!", + .count = 1, + .expected = { + { .pos = {6, 10}, }, + }, + }, + + /* new fuzzer regressions */ + { + /* PCRE does not set the first capture, which is unsatisfiable */ + .regex = "^(.^)*^(a*)", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + /* similar to the previous case, but with different anchoring */ + .regex = "(a)*(^)*^", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(.a)*^(.a)", + .input = "!a", + .count = 3, + .expected = { + { .pos = {0, 2}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 2}, }, + }, + }, + { + .regex = "(A)*^()*^", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + + { + .regex = "(a(b*)*|)*bc", + .input = "b!bc", + .count = 3, + .expected = { + { .pos = {2, 4}, }, + { .pos = {2, 2}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(a(b*)*|)*bc$", + .input = "bc", + .count = 3, + .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "(|a((b*)*b*))*", + .input = "", + .count = 4, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + /* simplified version of the above */ + .regex = "^(|a(b*)*)*$", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + /* zero repetitions should not set the capture */ + .regex = "^(a)*$", + .input = "", + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(a)*(^)$", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + /* raw fuzzer output */ + .regex = "()((()(^|$|$^|^|$|$^^|$|$^|^|$|$^^^^|^|(|)($)|)+|^^|^|(|)($)|)+|)($)()+", + .input = "", + .count = 12, + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(^|())+()", + .input = "", + .count = 4, + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(?:(^|^$)+|)+", + .input = "", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^((|)($)|)+a$", + .input = "a", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 4, + .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(($)|)+a$", + .input = "a", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(|(|x))*$", + .input = "x", + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + /* same as the previous but without outer capture */ + .regex = "^(?:|(|x))*$", + .input = "x", + .count = 2, + .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "(((($)|)+|)a|)+", + .input = "", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 5, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + + { + .regex = "^(|(|(|x)))*$", + .input = "x", + .count = 4, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + + + { + .regex = "^(?:(?:(x?)^)y?)+$", + .input = "", + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?:^())+$", + .input = "", + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?:($|x))+$", + .input = "x", + + .match = SHOULD_REJECT_AS_UNSUPPORTED, + + .count = 2, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^(($)|x)+$", + .input = "x", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^(?:()?^()?)+$", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?:($|x)())+$", + .input = "x", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + + { + .regex = "()~((|)($)|%)+", + .input = "~%", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 5, + .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 0}, }, + { .pos = {2, 2}, }, + { .pos = {2, 2}, }, + { .pos = {2, 2}, }, + }, + }, + + { + /* (slightly) reduced version of the previous */ + .regex = "^(()($)|x)+$", + .input = "x", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 4, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + + { + .regex = "a|_$[^b]", + .input = "a", + .count = 1, + .expected = { + { .pos = {0, 1}, }, + }, + }, +}; + +const struct captest_case_multi multi_cases[] = { + { + .regex_count = 4, + .regexes = { + "^aa$", /* exactly two 'a's */ + "^a*", /* zero or more 'a's followed by anything */ + "^ab?$", /* 'a' and optionally 'b' */ + "a*$", /* anything ending in zero or more 'a's */ + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 0 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 0, 0 } }, + }, + }, + + { + .input = "a", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 1 } }, + { .regex = 2, .pos = { 0, 1 } }, + { .regex = 3, .pos = { 0, 1 } }, + }, + }, + + { + .input = "aa", + .expected = { + { .regex = 0, .pos = { 0, 2 } }, + { .regex = 1, .pos = { 0, 2 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 0, 2 } }, + }, + }, + + { + .input = "aaa", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 3 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 0, 3 } }, + }, + }, + + { + .input = "ba", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 0 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 1, 2 } }, + }, + }, + + { + .input = "ab", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 1 } }, + { .regex = 2, .pos = { 0, 2 } }, + { .regex = 3, .pos = { 2, 2 } }, + }, + }, + + { + .input = NULL, + }, + }, + }, + + { + .regex_count = 3, + .regexes = { + "a(b?)*c", + "(ab)(c)", + "ab+(c)", + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 2, .pos = POS_NONE }, + { .regex = 2, .capture = 0, .pos = POS_NONE }, + { .regex = 2, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "abc", + .expected = { + { .regex = 0, .capture = 0, .pos = {0, 3} }, + { .regex = 0, .capture = 1, .pos = {2, 2} }, + { .regex = 1, .capture = 0, .pos = {0, 3} }, + { .regex = 1, .capture = 1, .pos = {0, 2} }, + { .regex = 1, .capture = 2, .pos = {2, 3} }, + { .regex = 2, .capture = 0, .pos = {0, 3} }, + { .regex = 2, .capture = 1, .pos = {2, 3} }, + }, + }, + }, + }, + { + /* fuzzer regression: This led to an execution path in fsm_union_array, + * fsm_union, fsm_merge, merge that did not init or otherwise set the + * `struct fsm_combine_info`, leading to an out of range offset for + * the capture base. */ + .regex_count = 3, + .regexes = { + ".", + ".^", + "^^_", + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = POS_NONE }, + { .regex = 2, .pos = POS_NONE }, + }, + }, + { + .input = "_", + .expected = { + { .regex = 0, .pos = { 0, 1 } }, + { .regex = 1, .pos = { 0, 1 } }, + { .regex = 2, .pos = { 0, 1 } }, + }, + }, + }, + }, +}; + + +static struct captest_case_program program_cases[] = { + { + .input = "", + .char_class = { + { .octets = { ~0, ~0, ~0, ~0 }}, /* 0x00 <= x <= 0xff */ + }, + .expected = { + .count = 4, + .captures = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + + .ops = { + { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 3, .new = 1 }}, + { .t = CAPVM_OP_CHARCLASS, .u.charclass_id = 0 }, + { .t = CAPVM_OP_JMP, .u.jmp = 0 }, + { .t = CAPVM_OP_SAVE, .u.save = 0 }, + { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 5, .new = 7 }}, + { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_START }, + + { .t = CAPVM_OP_JMP, .u.jmp = 9 }, /* jump after |() */ + { .t = CAPVM_OP_SAVE, .u.save = 4 }, + { .t = CAPVM_OP_SAVE, .u.save = 5 }, + + { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 4, .new = 10 }}, + + { .t = CAPVM_OP_SAVE, .u.save = 2 }, + { .t = CAPVM_OP_SAVE, .u.save = 3 }, + { .t = CAPVM_OP_SAVE, .u.save = 6 }, + { .t = CAPVM_OP_SAVE, .u.save = 7 }, + { .t = CAPVM_OP_SAVE, .u.save = 1 }, + { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 18, .new = 16 }}, + { .t = CAPVM_OP_CHARCLASS, .u.charclass_id = 0 }, + { .t = CAPVM_OP_JMP, .u.jmp = 15 }, + { .t = CAPVM_OP_MATCH }, + }, + }, + + + { + /* correcting compilation of '^(?:($|x))+$' */ + .input = "x", + .expected = { + .count = 2, + .captures = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + + .ops = { + [0] = { .t = CAPVM_OP_SAVE, .u.save = 0 }, + [1] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_START }, + [2] = { .t = CAPVM_OP_SAVE, .u.save = 2 }, + [3] = { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 4, .new = 6 }}, + [4] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_END }, + + /* [5] = { .t = CAPVM_OP_JMP, .u.jmp = 7 }, */ + [5] = { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 7, .new = 9 }}, + + [6] = { .t = CAPVM_OP_CHAR, .u.chr = 'x' }, + [7] = { .t = CAPVM_OP_SAVE, .u.save = 3 }, + [8] = { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 2, .new = 9 }}, + [9] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_END }, + [10] = { .t = CAPVM_OP_SAVE, .u.save = 1 }, + [11] = { .t = CAPVM_OP_MATCH }, + }, + }, +}; + +#define NO_FILTER ((size_t)-1) +struct options { + size_t filter; + int verbosity; + bool track_timing; + FILE *prog_output; + enum groups { + GROUP_SINGLE = 0x01, + GROUP_MULTI = 0x02, + GROUP_PROGRAMS = 0x04, + GROUP_ALL = 0xff, + } group; +}; + +static void +print_usage(FILE *f, const char *progname) +{ + fprintf(f, "%s: [-h] [-v] [-s | -m | -p] [-f ] [-t]\n", progname); + fprintf(f, " -h: print this usage info\n"); + fprintf(f, " -v: increase verbosity (can repeat: -vvv)\n"); + fprintf(f, " -f : just run a specific test, by numeric ID\n"); + fprintf(f, " -s: only single casse\n"); + fprintf(f, " -m: only multi cases\n"); + fprintf(f, " -p: only program cases\n"); + fprintf(f, " -t: print timing info\n"); +} + +static void +get_options(struct options *opt, int argc, char **argv) +{ + const char *progname = argv[0]; + int c; + while (c = getopt(argc, argv, "hf:mpstv"), c != -1) { + switch (c) { + case 'h': + print_usage(stdout, progname); + exit(EXIT_SUCCESS); + break; + case 'v': + opt->verbosity++; + break; + case 'f': + opt->filter = atol(optarg); + break; + case 't': + opt->track_timing = true; + break; + case 'p': + opt->group = GROUP_PROGRAMS; + break; + case 's': + opt->group = GROUP_SINGLE; + break; + case 'm': + opt->group = GROUP_MULTI; + break; + case '?': + default: + print_usage(stderr, progname); + exit(EXIT_FAILURE); + } + } +} + +int main(int argc, char **argv) { + size_t pass = 0; + size_t fail = 0; + size_t skip = 0; + size_t nth = 0; + + struct options options = { + .filter = NO_FILTER, + .verbosity = 0, + .group = GROUP_ALL, + }; + get_options(&options, argc, argv); + + if (options.verbosity == DUMP_PROGRAMS_VERBOSITY) { + options.prog_output = fopen("prog_output", "w"); + assert(options.prog_output != NULL); + } + + /* avoid an extra layer of indentation here */ + if (!(options.group & GROUP_SINGLE)) { goto after_single; } + + printf("-- single cases without trailing newline\n"); + const size_t single_case_count = sizeof(single_cases)/sizeof(single_cases[0]); + for (size_t c_i = 0; c_i < single_case_count; c_i++) { + const size_t cur = nth++; + if (options.filter != NO_FILTER && options.filter != cur) { + continue; + } + + if (options.verbosity > 0) { + printf("%zu: ", cur); + if (options.verbosity > 2) { + fflush(stdout); + } + } + + if (options.verbosity == DUMP_PROGRAMS_VERBOSITY) { + fprintf(options.prog_output, "\n\n==== test_case %zu\n", c_i); + } + + const struct captest_case_single *t = &single_cases[c_i]; + + if (t->match == SHOULD_SKIP) { + printf("%zd: SKIP (regex \"%s\", input \"%s\")\n", + cur, t->regex, t->input); + skip++; + continue; + } + + enum captest_run_case_res res = captest_run_case(t, options.verbosity, false, options.prog_output); + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + pass++; + break; + case CAPTEST_RUN_CASE_FAIL: + if (options.verbosity == 0) { + printf("-- test case %zd (regex \"%s\", input \"%s\")\n", cur, t->regex, t->input); + } + fail++; + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } + + /* second pass, adding a trailing newline to input */ + printf("-- single cases with trailing newline\n"); + for (size_t c_i = 0; c_i < single_case_count; c_i++) { + const size_t cur = nth++; + if (options.filter != NO_FILTER && options.filter != cur) { + continue; + } + + const struct captest_case_single *t = &single_cases[c_i]; + if (t->no_nl) { continue; } + if (t->match == SHOULD_SKIP) { + printf("%zd: SKIP (regex \"%s\", input \"%s\\n\")\n", + cur, t->regex, t->input); + skip++; + continue; + } + + if (options.verbosity > 0) { + printf("%zu: ", cur); + if (options.verbosity > 2) { + fflush(stdout); + } + } + + enum captest_run_case_res res = captest_run_case(t, options.verbosity, true, options.prog_output); + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + pass++; + break; + case CAPTEST_RUN_CASE_FAIL: + if (options.verbosity == 0) { + printf("-- test case %zd (regex \"%s\", input \"%s\\n\")\n", cur, t->regex, t->input); + } + fail++; + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } +after_single: + + /* multi-regex tests */ + if (!(options.group & GROUP_MULTI)) { goto after_multi; } + + printf("-- multi-regex cases\n"); + const size_t multi_case_count = sizeof(multi_cases)/sizeof(multi_cases[0]); + for (size_t c_i = 0; c_i < multi_case_count; c_i++) { + const size_t cur = nth++; + if ((options.filter != NO_FILTER && options.filter != cur)) { + continue; + } + + const struct captest_case_multi *t = &multi_cases[c_i]; + if (t->match == SHOULD_SKIP) { + printf("%zu: SKIP (multi)\n", c_i); + skip++; + continue; + } + + if (options.verbosity > 0) { + printf("%zu: ", cur); + } + + struct captest_case_multi_result result; + enum captest_run_case_res res = captest_run_case_multi(t, + options.verbosity, false, options.prog_output, &result); + + pass += result.pass; + fail += result.fail; + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + if (options.verbosity > 0) { + printf("pass\n"); + } + break; + case CAPTEST_RUN_CASE_FAIL: + if (options.verbosity > 0) { + printf("FAIL\n"); + } else { + printf("-- test case %zd\n", cur); + } + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } +after_multi: + + /* hardcoded programs */ + if (!(options.group & GROUP_PROGRAMS)) { goto after_programs; } + + const size_t prog_case_count = sizeof(program_cases)/sizeof(program_cases[0]); + for (size_t c_i = 0; c_i < prog_case_count; c_i++) { + const size_t cur = nth++; + if ((options.filter != NO_FILTER && options.filter != cur)) { + continue; + } + + const struct captest_case_program *t = &program_cases[c_i]; + + if (options.verbosity > 0) { + printf("%zu: ", cur); + } + + enum captest_run_case_res res = captest_run_case_program(t, + options.verbosity); + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + if (options.verbosity > 0) { + printf("pass\n"); + } + pass++; + break; + case CAPTEST_RUN_CASE_FAIL: + fail++; + if (options.verbosity > 0) { + printf("FAIL\n"); + } else if (options.verbosity == 0) { + printf("-- test case %zd\n", cur); + } + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } +after_programs: + + printf("-- pass %zu, fail %zu, skip %zu\n", pass, fail, skip); + + return fail > 0 + ? EXIT_FAILURE + : EXIT_SUCCESS; +} diff --git a/tests/idmap/idmap_basic.c b/tests/idmap/idmap_basic.c index 19f44d56e..c7a18856b 100644 --- a/tests/idmap/idmap_basic.c +++ b/tests/idmap/idmap_basic.c @@ -19,13 +19,14 @@ #define ID_MASK ((1 << 9) - 1) #define VALUE_MASK ((1 << 10) - 1) -static void +static int dump_cb(fsm_state_t state_id, unsigned value, void *opaque) { /* fprintf(stderr, " -- state %d, value %u\n", state_id, value); */ assert(state_id <= ID_MASK); assert(value <= VALUE_MASK); (void)opaque; + return 1; } static int diff --git a/tests/minimise/minimise_test_case_list.c b/tests/minimise/minimise_test_case_list.c index 1386f0dcc..7299d2fa1 100644 --- a/tests/minimise/minimise_test_case_list.c +++ b/tests/minimise/minimise_test_case_list.c @@ -22,7 +22,6 @@ const char *test_cases[] = { "(?:a+|b)a+", "(?:a*ba)+", "(?:a|cd)+e?x", - "-> 1 'a';", "(?:abc|def)+", "(?:abc|def)*", "(?:b|a*)", @@ -81,7 +80,7 @@ check_minimisation(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI | RE_NOCAPTURE, &err); assert(fsm != NULL); if (!fsm_determinise(fsm)) { return 0; diff --git a/theft/fuzz_capture_string_set.c b/theft/fuzz_capture_string_set.c index 7326356c2..f225bb326 100644 --- a/theft/fuzz_capture_string_set.c +++ b/theft/fuzz_capture_string_set.c @@ -158,7 +158,7 @@ check_capstring_set(struct capture_env *env, return THEFT_TRIAL_ERROR; } - const size_t capture_count = fsm_countcaptures(dfa); + const size_t capture_count = fsm_capture_ceiling(dfa); if (verbosity > 2) { fprintf(stderr, "==== cs '%s'\n", cs->string); @@ -172,7 +172,7 @@ check_capstring_set(struct capture_env *env, assert(cp != NULL); fsm_copies[cs_i] = cp; - const size_t cp_capture_count = fsm_countcaptures(cp); + const size_t cp_capture_count = fsm_capture_ceiling(cp); if (verbosity > 2) { fprintf(stderr, "==== min(det(cp))\n"); fsm_print_fsm(stderr, cp); @@ -196,7 +196,7 @@ check_capstring_set(struct capture_env *env, return THEFT_TRIAL_FAIL; } - combined_capture_count = fsm_countcaptures(combined); + combined_capture_count = fsm_capture_ceiling(combined); for (size_t cs_i = 0; cs_i < css->count; cs_i++) { total_captures += capture_counts[cs_i]; } @@ -295,7 +295,7 @@ check_fsms_for_single_input(struct check_env *env, struct fsm_capture *captures, assert(exec_res >= 0); if (exec_res == 1) { if (LOG_LEVEL > 0) { - const size_t combined_capture_count = fsm_countcaptures(env->combined); + const size_t combined_capture_count = fsm_capture_ceiling(env->combined); for (size_t i = 0; i < combined_capture_count; i++) { fprintf(stderr, "capture[%zu/%zu]: (%ld, %ld)\n", i, combined_capture_count, @@ -415,7 +415,7 @@ compare_captures(const struct check_env *env, const struct fsm_capture *captures_combined, size_t nth_fsm, const struct fsm_capture *captures) { - const size_t combined_capture_count = fsm_countcaptures(env->combined); + const size_t combined_capture_count = fsm_capture_ceiling(env->combined); if (combined_capture_count == 0) { return true; /* no captures */ } @@ -639,7 +639,7 @@ build_capstring_dfa(const struct capstring *cs, uint8_t end_id) goto cleanup; } - if (fsm_countcaptures(fsm) != cs->capture_count) { + if (fsm_capture_ceiling(fsm) != cs->capture_count) { goto cleanup; } From 31ff23429fa7d1d36067e74322af73546335270e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 2 Jun 2023 11:41:28 -0400 Subject: [PATCH 008/373] Add pcre-anchor test for anchoring edge case. --- tests/pcre-anchor/in81.re | 1 + tests/pcre-anchor/out81.fsm | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/pcre-anchor/in81.re create mode 100644 tests/pcre-anchor/out81.fsm diff --git a/tests/pcre-anchor/in81.re b/tests/pcre-anchor/in81.re new file mode 100644 index 000000000..8b5fad7c3 --- /dev/null +++ b/tests/pcre-anchor/in81.re @@ -0,0 +1 @@ +($x)* \ No newline at end of file diff --git a/tests/pcre-anchor/out81.fsm b/tests/pcre-anchor/out81.fsm new file mode 100644 index 000000000..2cdc2f023 --- /dev/null +++ b/tests/pcre-anchor/out81.fsm @@ -0,0 +1,5 @@ +0 -> 0 ?; +0 -> 1 "\n"; + +start: 0; +end: 0, 1; \ No newline at end of file From 77f9260ba15b1589e288b83b82b8e2dda6c59f5c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 2 Jun 2023 14:12:15 -0400 Subject: [PATCH 009/373] fuzz/target.c: Add multi-regex and single regex cmp against PCRE. Add a couple more execution modes to the fuzzer test harness. In particular, add support for comparing match and capture behavior against PCRE's -- because this depends on linking with libpcre, it's disabled by defalut. Set `PCRE_CMP` in the Makefile to enable it, or pass it as a build argument. --- fuzz/Makefile | 11 +- fuzz/target.c | 1206 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 1203 insertions(+), 14 deletions(-) diff --git a/fuzz/Makefile b/fuzz/Makefile index 1d418cd97..2327a811b 100644 --- a/fuzz/Makefile +++ b/fuzz/Makefile @@ -7,6 +7,15 @@ ${BUILD}/fuzz/: ${BUILD} DIR += ${BUILD}/fuzz +# Uncomment to enable capture fuzzing using PCRE as a test oracle. +#PCRE_CMP=1 + +.if PCRE_CMP +PKG += libpcre2-8 +LFLAGS.fuzzer += ${LIBS.libpcre2-8} +CFLAGS.${SRC:Mfuzz/target.c} += -DCMP_PCRE=1 +.endif + .for src in ${SRC:Mfuzz/*.c} CFLAGS.${src} += -std=c99 .endfor @@ -15,7 +24,7 @@ CFLAGS.${src} += -std=c99 fuzz:: ${BUILD}/fuzz/fuzzer ${BUILD}/fuzz/fuzzer: mkdir - ${CC} -o $@ ${LFLAGS} ${.ALLSRC:M*.o} ${.ALLSRC:M*.a} + ${CC} -o $@ ${LFLAGS} ${LFLAGS.fuzzer} ${.ALLSRC:M*.o} ${.ALLSRC:M*.a} .for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} ${BUILD}/fuzz/fuzzer: ${BUILD}/lib/${lib:R}.a diff --git a/fuzz/target.c b/fuzz/target.c index cae283aca..eeda0e9d5 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -21,17 +22,47 @@ #include "../src/libfsm/minimise_test_oracle.h" +/* for fsm_capture_dump */ +/* FIXME: should this be a public interface? */ +#include "../src/libfsm/capture.h" + +/* Buffer for sanitized fuzzer input */ +#define MAX_FUZZER_DATA (64 * 1024) +static uint8_t data_buf[MAX_FUZZER_DATA + 1]; + +/* Should fuzzer harness code be built that compares behavior + * with PCRE? (Obviously, this depends on PCRE.) */ +#ifndef CMP_PCRE +#define CMP_PCRE 0 +#endif + +#if CMP_PCRE +#define PCRE2_CODE_UNIT_WIDTH 8 +#include + +static int +compare_with_pcre(const char *pattern, struct fsm *fsm); +#endif + /* 10 seconds */ #define TIMEOUT_USEC (10ULL * 1000 * 1000) +/* for TRACK_TIMES and EXPENSIVE_CHECKS */ +#include "../src/libfsm/internal.h" + enum run_mode { - MODE_DEFAULT, + MODE_REGEX, + MODE_REGEX_SINGLE_ONLY, + MODE_REGEX_MULTI_ONLY, + MODE_IDEMPOTENT_DET_MIN, MODE_SHUFFLE_MINIMISE, MODE_ALL_PRINT_FUNCTIONS, }; +static size_t +get_env_config(size_t default_value, const char *env_var_name); -/* This stuff will already exist elsewhere once other branches are merged. */ +/* TODO: These could be moved to a common file for test utils. */ #if 1 static void time_get(struct timeval *tv) @@ -73,7 +104,110 @@ scanner_next(void *opaque) } #endif -static const struct fsm_options opt; +/* This is used to track allocation during each fuzzer + * run. Note that hwm is not reduced when memory is + * free'd or realloc'd, because the size info is not + * passed to those calls. */ +#define MB(X) ((size_t)X * 1000 * 1000) +#define FH_ALLOCATOR_HWM_LIMIT (MB(50)) +struct fh_allocator_stats { + size_t hwm; /* high water mark */ +}; + +static void +fh_memory_hwm_limit_hook(const char *caller_name) +{ + /* It doesn't really help to exit here because libfuzzer will + * still treat it as a failure, but at least we can print a + * message about hitting the allocator limit and exit so we + * don't need to spend time investigating timeouts or ooms + * that are due to obvious resource exhaustion. */ + fprintf(stderr, "%s: hit FH_ALLOCATOR_HWM_LIMIT (%zu), exiting\n", + caller_name, FH_ALLOCATOR_HWM_LIMIT); + exit(EXIT_SUCCESS); +} + +static void +fh_free(void *opaque, void *p) +{ + (void)opaque; + free(p); +} + +static void * +fh_calloc(void *opaque, size_t n, size_t sz) +{ + struct fh_allocator_stats *stats = opaque; + stats->hwm += sz; + if (stats->hwm > FH_ALLOCATOR_HWM_LIMIT) { + fh_memory_hwm_limit_hook(__func__); + return NULL; + } + + (void)opaque; + return calloc(n, sz); +} + +static void * +fh_malloc(void *opaque, size_t sz) +{ + struct fh_allocator_stats *stats = opaque; + stats->hwm += sz; + if (stats->hwm > FH_ALLOCATOR_HWM_LIMIT) { + fh_memory_hwm_limit_hook(__func__); + return NULL; + } + + return malloc(sz); +} + +static void * +fh_realloc(void *opaque, void *p, size_t sz) +{ + struct fh_allocator_stats *stats = opaque; + stats->hwm += sz; + if (stats->hwm > FH_ALLOCATOR_HWM_LIMIT) { + fh_memory_hwm_limit_hook(__func__); + return NULL; + } + + return realloc(p, sz); +} + +static struct fh_allocator_stats allocator_stats; + +/* fuzzer harness allocators */ +static struct fsm_alloc custom_allocators = { + .free = fh_free, + .calloc = fh_calloc, + .malloc = fh_malloc, + .realloc = fh_realloc, + .opaque = &allocator_stats, +}; + +static const struct fsm_options fsm_options = { + .group_edges = 1, /* make output readable */ + .alloc = &custom_allocators, +}; + +static void +dump_pattern(const char *pattern) +{ + const size_t pattern_length = strlen(pattern); + fprintf(stderr, "-- Pattern: %zu bytes\n", pattern_length); + for (size_t i = 0; i < pattern_length; i++) { + fprintf(stderr, " %02x", (uint8_t)pattern[i]); + if ((i & 31) == 31) { fprintf(stderr, "\n"); } + } + if ((pattern_length & 31) != 31) { + fprintf(stderr, "\n"); + } + for (size_t i = 0; i < pattern_length; i++) { + fprintf(stderr, "%c", isprint(pattern[i]) ? pattern[i] : '.'); + if ((i & 63) == 63) { fprintf(stderr, "\n"); } + } + fprintf(stderr, "\n"); +} static struct fsm * build(const char *pattern) @@ -93,7 +227,7 @@ build(const char *pattern) }; time_get(&pre); - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, &fsm_options, RE_MULTI, &err); time_get(&post); delta_usec = time_diff_usec(&pre, &post); total_usec += delta_usec; @@ -119,12 +253,24 @@ build(const char *pattern) total_usec += delta_usec; if (total_usec > TIMEOUT_USEC) { + dump_pattern(pattern); assert(!"timeout"); } return fsm; } +static size_t +get_env_config(size_t default_value, const char *env_var_name) +{ + const char *s = getenv(env_var_name); + if (s == NULL) { + return default_value; + } else { + return strtoul(s, NULL, 10); + } +} + static int codegen(const struct fsm *fsm) { @@ -136,14 +282,46 @@ codegen(const struct fsm *fsm) } static int -build_and_codegen(const char *pattern) +build_and_check_single(const char *pattern) { + const int verbosity = get_env_config(0, "VERBOSITY"); + if (verbosity > 1) { + fprintf(stderr, "pattern: \"%s\"\n", pattern); + } + + INIT_TIMERS(); + TIME(&pre); struct fsm *fsm = build(pattern); if (fsm == NULL) { return EXIT_SUCCESS; } + TIME(&post); + DIFF_MSEC("build", pre, post, NULL); + + if (getenv("DUMP")) { + fprintf(stderr,"==================================================\n"); + fsm_print_fsm(stderr, fsm); + fprintf(stderr,"==================================================\n"); + fsm_capture_dump(stderr, "CAPTURE", fsm); + fprintf(stderr,"==================================================\n"); + } - if (!codegen(fsm)) { +#if CMP_PCRE + TIME(&pre); + const int cmp_res = compare_with_pcre(pattern, fsm); + TIME(&post); + DIFF_MSEC("cmp", pre, post, NULL); + if (!cmp_res) { + fsm_free(fsm); + return EXIT_SUCCESS; + } +#endif + + TIME(&pre); + const int codegen_res = codegen(fsm); + TIME(&post); + DIFF_MSEC("codegen", pre, post, NULL); + if (!codegen_res) { return EXIT_SUCCESS; } @@ -151,6 +329,845 @@ build_and_codegen(const char *pattern) return EXIT_SUCCESS; } +#define DEF_MAX_DEPTH 20 +#define DEF_MAX_LENGTH 10 +#define DEF_MAX_STEPS 10000 +#define DEF_MAX_MATCH_COUNT 1000 + +#if CMP_PCRE +/* These two are only used with PCRE2 */ +#define ANCHORED_PCRE 0 +#define FUZZ_RE_MATCH_LIMIT 10000 +#define FUZZ_RE_RECURSION_LIMIT 200 +#define MAX_OVEC_SIZE 512 + +static pcre2_match_context *pcre2_mc = NULL; + +struct cmp_pcre_env { + int verbosity; + const char *pattern; + const struct fsm *fsm; + pcre2_match_data *md; + pcre2_code *p; + + struct fsm_capture *captures; + size_t captures_length; + + size_t max_depth; + size_t max_steps; + size_t max_match_count; +}; + +struct test_pcre_match_info { + int res; + int pcre_error; + size_t ovector[MAX_OVEC_SIZE]; +}; + +static pcre2_code * +build_pcre2(const char *pattern, int verbosity) +{ + const uint32_t options = ANCHORED_PCRE ? PCRE2_ANCHORED : 0; + int errorcode; + PCRE2_SIZE erroffset = 0; + pcre2_compile_context *cctx = NULL; + + /* Set match limits */ + if (pcre2_mc == NULL) { + pcre2_mc = pcre2_match_context_create(NULL); + assert(pcre2_mc != NULL); + + pcre2_set_match_limit(pcre2_mc, FUZZ_RE_MATCH_LIMIT); + pcre2_set_recursion_limit(pcre2_mc, FUZZ_RE_RECURSION_LIMIT); + } + + pcre2_code *p = pcre2_compile((const unsigned char *)pattern, + PCRE2_ZERO_TERMINATED, + options, &errorcode, &erroffset, cctx); + if (verbosity > 0 && p == NULL && errorcode != 0) { +#define ERRSIZE 4096 + unsigned char errbuf[ERRSIZE] = {0}; + if (!pcre2_get_error_message(errorcode, + errbuf, ERRSIZE)) { + fprintf(stderr, "pcre2_get_error_message: failed\n"); + } + fprintf(stderr, "pcre2_compile: error: %s\n", errbuf); + } + return p; +} + +enum do_pcre_match_res { + DO_PCRE_MATCH_HIT, + DO_PCRE_MATCH_MISS, + DO_PCRE_MATCH_SKIP, /* an exceptional case we don't care about */ + DO_PCRE_MATCH_ERROR = -1, +}; +enum do_pcre_match_res +do_pcre_match(FILE *f, const pcre2_code *p, pcre2_match_data *md, int verbosity, + const char *input, struct test_pcre_match_info *match_info) +{ +#define MAX_BUF (64*1024) + const size_t input_len = strlen(input); + enum do_pcre_match_res mres; + + /* turn off the JIT because it can give inconsistent results while fuzzing */ + const uint32_t options = (ANCHORED_PCRE ? PCRE2_ANCHORED : 0) + | PCRE2_NO_JIT; + + assert(pcre2_mc != NULL); + + /* The value returned by pcre2_match() is one more than the + * highest numbered pair that has been set. */ + int res = pcre2_match(p, (const unsigned char *)input, input_len, + 0, options, md, pcre2_mc); + + if (res == PCRE2_ERROR_NOMATCH || res == PCRE2_ERROR_PARTIAL) { + if (f != NULL && verbosity > 1) { + fprintf(f, " -- no match (%s)\n", + res == PCRE2_ERROR_NOMATCH ? "NOMATCH" + : res == PCRE2_ERROR_PARTIAL ? "PARTIAL" + : ""); + } + mres = DO_PCRE_MATCH_MISS; + goto cleanup; + } else if (res == PCRE2_ERROR_MATCHLIMIT || res == PCRE2_ERROR_DEPTHLIMIT) { + /* It's possible to exhaust PCRE's internal limits with pathologically + * nested regexes like "(((((((((^.)?)*)?)?)?)*)+)+)*$" and + * "((((((((akbzaabdcOaa)|((((b*))))?|.|.|.*|.|.))+)+)+$)*)?)" , but + * as long as they don't cause it to block for excessively long or + * exhaust resources that's fine. */ + if (f != NULL) { + fprintf(f, " -- PCRE2_ERROR_MATCHLIMIT (returning SKIP)\n"); + } + mres = DO_PCRE_MATCH_SKIP; + } else if (res <= 0) { + if (f != NULL) { +#define ERR_MAX 4096 + unsigned char err_buf[ERR_MAX]; + if (pcre2_get_error_message(res, err_buf, ERR_MAX)) { + fprintf(f, " -- error %d: %s\n", res, err_buf); + } else { + fprintf(f, " -- error %d\n", res); + } +#undef ERR_MAX + } + if (match_info != NULL) { + match_info->pcre_error = res; + } + mres = DO_PCRE_MATCH_ERROR; + goto cleanup; + } else { + const uint32_t ovc = pcre2_get_ovector_count(md); + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(md); + assert(res >= 0); + size_t ures = (size_t)res; + assert(ovc > ures); + + assert(ovector[1] >= ovector[0]); + const size_t mlen = ovector[1] - ovector[0]; + if (ANCHORED_PCRE && (ovector[0] != 0 || mlen != input_len)) { + mres = DO_PCRE_MATCH_MISS; + goto cleanup; + } + mres = DO_PCRE_MATCH_HIT; + + if (f != NULL && verbosity > 1) { + for (size_t i = 0; i < ures; i++) { + char buf[MAX_BUF] = { 0 }; + memcpy(buf, &input[ovector[2*i]], + ovector[2*i + 1U] - ovector[2*i]); + fprintf(f, " -- %zu: \"%s\"\n", i, buf); + } + } + + if (match_info != NULL && res < MAX_OVEC_SIZE) { + match_info->res = res; + assert(res >= 0); + const size_t ures = (size_t)res; + + for (size_t i = 0; i < 2*ures; i++) { + match_info->ovector[i] = ovector[i]; + } + } + } + +cleanup: + return mres; +#undef MAX_BUF +} + +static bool +exec_and_compare_captures(struct cmp_pcre_env *env, + const char *input, size_t input_size, + const struct test_pcre_match_info *match_info) +{ + bool matching = true; + fsm_state_t end_state; + const uint8_t *u8_input = (const uint8_t *)input; + int res = fsm_exec_with_captures(env->fsm, u8_input, input_size, + &end_state, env->captures, env->captures_length); + + if (res < 0) { + if (env->verbosity > 1) { + fprintf(stderr, "got res of %d\n", res); + } + + return false; + } + + if (res > 0) { + assert(match_info->res >= 0); + const size_t ures = (size_t)match_info->res; + + if (env->verbosity > 1) { + fprintf(stderr, "ures %zu\n", ures); + } + + for (size_t i = 0; i < ures; i++) { + if (env->verbosity > 1) { + fprintf(stderr, "%zu/%zu: pcre [%ld, %ld] <-> libfsm [%ld, %ld]\n", + i, ures, + match_info->ovector[2*i], match_info->ovector[2*i + 1], + env->captures[i].pos[0], env->captures[i].pos[1]); + } + if ((match_info->ovector[2*i] != env->captures[i].pos[0]) + || (match_info->ovector[2*i + 1] != env->captures[i].pos[1])) { + matching = false; + } + } + + if (!matching) { + for (size_t i = 0; i < ures; i++) { + fprintf(stderr, "%zu/%zu: pcre [%ld, %ld] <-> libfsm [%ld, %ld]\n", + i, ures, + match_info->ovector[2*i], match_info->ovector[2*i + 1], + env->captures[i].pos[0], env->captures[i].pos[1]); + } + } + } + + return matching; +} + +static void +dump_pattern_and_input(const char *pattern, const char *input, size_t input_length) +{ + dump_pattern(pattern); + + fprintf(stderr, "-- Input: %zu bytes\n", input_length); + for (size_t i = 0; i < input_length; i++) { + fprintf(stderr, " %02x", (uint8_t)input[i]); + if ((i & 31) == 31) { fprintf(stderr, "\n"); } + } + if ((input_length & 31) != 31) { + fprintf(stderr, "\n"); + } + for (size_t i = 0; i < input_length; i++) { + fprintf(stderr, "%c", isprint(input[i]) ? input[i] : '.'); + if ((i & 63) == 63) { fprintf(stderr, "\n"); } + } + fprintf(stderr, "\n"); +} + +static enum fsm_generate_matches_cb_res +cmp_pcre_gen_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + struct cmp_pcre_env *env = opaque; + assert(env != NULL); + + (void)fsm; + (void)depth; + (void)end_state; + + const size_t len = strlen(input); + + if (env->verbosity > 4) { + fprintf(stderr, "%s: depth %zu/%zu, match_count %zu/%zu, steps %zu/%zu\n", + __func__, + depth, env->max_depth, + match_count, env->max_match_count, + steps, env->max_steps); + } + + if (steps > env->max_steps) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (match_count > env->max_match_count) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (depth > env->max_depth) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + /* Completely avoid exploring inputs with embedded 0x00 bytes. */ + if (input_length != len) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + if (len > 0 && input[len - 1] == '\n') { + /* These will need to be handled properly, but PCRE has + * special cases for '\n' handling. */ + /* fprintf(stderr, " -- skipping input ending with '\\n'.\n"); */ + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + struct test_pcre_match_info match_info = { .pcre_error = 0 }; + enum do_pcre_match_res mres = do_pcre_match(stderr, + env->p, env->md, env->verbosity, input, &match_info); + switch (mres) { + case DO_PCRE_MATCH_SKIP: + break; + case DO_PCRE_MATCH_MISS: + dump_pattern_and_input(env->pattern, input, input_length); + assert(!"matches libfsm but not with PCRE"); + return FSM_GENERATE_MATCHES_CB_RES_HALT; + case DO_PCRE_MATCH_ERROR: + fprintf(stderr, "FAIL: PCRE returned ERROR %d: pattern \"%s\"\n", + match_info.pcre_error, env->pattern); + return FSM_GENERATE_MATCHES_CB_RES_HALT; + case DO_PCRE_MATCH_HIT: + break; /* okay; continue below */ + } + + if (env->verbosity > 1) { + fprintf(stderr, "-- comparing captures for pattern \"%s\", input \"%s\" (len %zu)\n", + env->pattern, input, len); + } + + if (!exec_and_compare_captures(env, input, input_length, &match_info)) { + if (env->verbosity > 1 || 1) { + dump_pattern_and_input(env->pattern, input, input_length); + fsm_print_fsm(stderr, env->fsm); + fsm_capture_dump(stderr, "fsm", env->fsm); + } + assert(!"captures don't match"); + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; +} + +static int +compare_fixed_input(struct fsm *fsm, const char *pattern, const char *input, pcre2_match_data *md, pcre2_code *p) +{ + fsm_state_t end_state; + const size_t capture_ceil = fsm_capture_ceiling(fsm); + + struct fsm_capture *captures = malloc(capture_ceil * sizeof(captures[0])); + assert(captures != NULL); + for (size_t i = 0; i < capture_ceil; i++) { + /* clobber with meaningless but visually distinct values */ + captures[i].pos[0] = 88888888; + captures[i].pos[1] = 99999999; + }; + + const uint8_t *u8_input = (const uint8_t *)input; + const size_t input_len = strlen(input); + const int libfsm_res = fsm_exec_with_captures(fsm, u8_input, input_len, + &end_state, captures, capture_ceil); + + const bool libfsm_matching = libfsm_res > 0; + + int res = 1; + + struct test_pcre_match_info match_info = { .pcre_error = 0 }; + enum do_pcre_match_res mres = do_pcre_match(stderr, + p, md, 0, input, &match_info); + switch (mres) { + case DO_PCRE_MATCH_SKIP: + return 1; + case DO_PCRE_MATCH_MISS: + if (!libfsm_matching) { + goto cleanup; + } + dump_pattern_and_input(pattern, input, 0); + assert(!"matches libfsm but not with PCRE"); + return 0; + case DO_PCRE_MATCH_ERROR: + fprintf(stderr, "FAIL: PCRE returned ERROR %d: pattern \"%s\"\n", + match_info.pcre_error, pattern); + return 0; + case DO_PCRE_MATCH_HIT: + if (!libfsm_matching) { + dump_pattern_and_input(pattern, input, input_len); + assert(!"matches PCRE but not libfsm"); + res = 0; + goto cleanup; + } + + const size_t ures = (size_t)match_info.res; + if (ures > capture_ceil) { + dump_pattern_and_input(pattern, input, 0); + fprintf(stderr, "error: capture_ceil: %zu exceeded by ures: %zd\n", + capture_ceil, ures); + assert(!"both PCRE and libfsm match but with different capture counts"); + } + + bool matching = true; + for (size_t i = 0; i < ures; i++) { + if ((match_info.ovector[2*i] != captures[i].pos[0]) + || (match_info.ovector[2*i + 1] != captures[i].pos[1])) { + matching = false; + } + } + for (size_t i = 0; i < ures; i++) { + if (!matching) { + fprintf(stderr, "%zu/%zu: pcre [%ld, %ld] <-> libfsm [%ld, %ld]\n", + i, ures, + match_info.ovector[2*i], match_info.ovector[2*i + 1], + captures[i].pos[0], captures[i].pos[1]); + } + } + + if (!matching) { + dump_pattern_and_input(pattern, input, 0); + assert(!"both PCRE and libfsm match but with different captures"); + } + + goto cleanup; /* ok, both matched */ + } + + assert(!"unreachable"); + +cleanup: + free(captures); + return res; + +} + +static int +compare_with_pcre(const char *pattern, struct fsm *fsm) +{ + size_t verbosity = get_env_config(0, "VERBOSITY"); + size_t max_length = get_env_config(DEF_MAX_LENGTH, "MAX_LENGTH"); + size_t max_steps = get_env_config(DEF_MAX_STEPS, "MAX_STEPS"); + size_t max_depth = get_env_config(DEF_MAX_DEPTH, "MAX_DEPTH"); + size_t max_match_count = get_env_config(DEF_MAX_MATCH_COUNT, "MAX_MATCH_COUNT"); + int res = 1; + + pcre2_match_data *md; + + pcre2_code *p = build_pcre2(pattern, 0); + if (p == NULL) { + return 1; + } + + md = pcre2_match_data_create(MAX_OVEC_SIZE, NULL); + assert(md != NULL); + + /* Check the empty string and "\n", because PCRE has an awkward + * special case for "\n" that has complicated interactions + * with start and end anchoring. */ + if (!compare_fixed_input(fsm, pattern, "", md, p) + || !compare_fixed_input(fsm, pattern, "\n", md, p)) { + pcre2_match_data_free(md); + pcre2_code_free(p); + return res; + } + + struct fsm_capture captures[MAX_OVEC_SIZE/2] = { 0 }; + + const size_t pattern_length = strlen(pattern); + if (pattern_length >= max_length) { + max_length = pattern_length + 1; + static size_t max_max_length; + if (max_length > max_max_length) { + fprintf(stderr, "Note: increasing max_length to %zu\n", + pattern_length + 1); + max_max_length = max_length; + if (max_depth < max_length) { + max_depth = max_length + 1; + } + } + } + + struct cmp_pcre_env env = { + .verbosity = (int)verbosity, + .pattern = pattern, + .fsm = fsm, + .captures = captures, + .captures_length = MAX_OVEC_SIZE/2, + .md = md, + .p = p, + .max_steps = max_steps, + .max_depth = max_depth, + .max_match_count = max_match_count, + }; + + if (!fsm_generate_matches(fsm, max_length, cmp_pcre_gen_cb, &env)) { + res = 0; + } + + pcre2_match_data_free(md); + pcre2_code_free(p); + return res; +} +#endif + +/* Note: combined_fsm and fsms[] are non-const because fsm_generate_matches + * calls fsm_trim on them. */ +static int +compare_combined_and_separate(int verbosity, size_t max_length, size_t count, + struct fsm *combined_fsm, const struct fsm_combined_base_pair *bases, + struct fsm **fsms); + +static enum fsm_generate_matches_cb_res +cmp_combined_and_separate_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +static int +build_and_check_multi(const char *input) +{ + int res = EXIT_FAILURE; + const int verbosity = get_env_config(0, "VERBOSITY"); +#define MAX_PATTERNS 8 +#define MAX_PATTERN_LEN 256 + char patterns[MAX_PATTERNS][MAX_PATTERN_LEN] = { 0 }; + size_t count = 0; + const size_t len = strlen(input); + size_t max_length = get_env_config(DEF_MAX_LENGTH, "MAX_LENGTH"); + INIT_TIMERS(); + + /* if nonzero, apply a timeout to the combined FSM det/min below */ + const size_t timeout = get_env_config(0, "TIMEOUT"); + + if (timeout > 0) { + if (TRACK_TIMES == 0) { + fprintf(stderr, "\n\n\n\n\nError: src/libfsm/internal.h:TRACK_TIMES needs to be nonzero for this use case, exiting.\n\n\n\n\n"); + exit(EXIT_FAILURE); + } else { + static bool printed_timeout_msg; + if (!printed_timeout_msg) { + fprintf(stderr, "Using timeout of %zu msec for fsm_determinise/fsm_minimise on combined FSM.\n", + timeout); + printed_timeout_msg = true; + } + } + } + + size_t i, j; + for (i = 0, j = 0; i < len; i++) { + const char c = input[i]; + if (c == '\n' || c == '\r') { + if (j > max_length) { + max_length = j; + } + count++; + if (count == MAX_PATTERNS) { + /* ignore: too many patterns */ + return EXIT_SUCCESS; + } + j = 0; + } else { + patterns[count][j] = c; + j++; + if (j == MAX_PATTERN_LEN) { + /* ignore: pattern too long */ + return EXIT_SUCCESS; + } + } + } + if (j > 0) { count++; } + + struct re_err err; + const enum re_flags flags = 0; + + /* build each regex, combining them and keeping track of capture offsets */ + struct fsm *fsms[count]; + struct fsm *fsms_cp[count]; + struct fsm_combined_base_pair bases[count]; + struct fsm *combined_fsm = NULL; + for (size_t i = 0; i < count; i++) { + fsms[i] = NULL; + fsms_cp[i] = NULL; + + bases[i].state = 0; + bases[i].capture = 0; + } + + /* compile each individually */ + for (size_t i = 0; i < count; i++) { + if (verbosity > 1) { + fprintf(stderr, "%s: compiling \"%s\"\n", + __func__, patterns[i]); + } + + struct scanner s = { + .str = (const uint8_t *)patterns[i], + .size = strlen(patterns[i]), + }; + + struct fsm *fsm = re_comp(RE_PCRE, scanner_next, &s, &fsm_options, flags, &err); + if (fsm == NULL) { + res = EXIT_SUCCESS; /* invalid regex, so skip this batch */ + goto cleanup; + } + + char label_buf[100]; + snprintf(label_buf, 100, "single_determisise_%zu", i); + + TIME(&pre); + if (!fsm_determinise(fsm)) { + goto cleanup; + } + TIME(&post); + DIFF_MSEC(label_buf, pre, post, NULL); + + snprintf(label_buf, 100, "single_minimise_%zu", i); + TIME(&pre); + if (!fsm_minimise(fsm)) { + goto cleanup; + } + TIME(&post); + DIFF_MSEC(label_buf, pre, post, NULL); + + if (verbosity > 4) { + char tag_buf[16] = { 0 }; + snprintf(tag_buf, sizeof(tag_buf), "fsm[%zu]", i); + + fprintf(stderr, "==== fsm[%zu]\n", i); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, tag_buf, fsm); + } + + fsms[i] = fsm; + fsms_cp[i] = fsm_clone(fsm); /* save a copy for comparison */ + } + + combined_fsm = fsm_union_array(count, fsms, bases); + assert(combined_fsm != NULL); + if (verbosity > 1) { + fprintf(stderr, "%s: combined_fsm: %d states after fsm_union_array\n", + __func__, fsm_countstates(combined_fsm)); + } + if (verbosity > 1) { + for (size_t i = 0; i < count; i++) { + fprintf(stderr, "%s: base[%zu]: state %d, capture %u\n", + __func__, i, bases[i].state, bases[i].capture); + } + } + + TIME(&pre); + if (!fsm_determinise(combined_fsm)) { + goto cleanup; + } + TIME(&post); + size_t timeout_accum = 0; + if (timeout != 0) { + if (verbosity > 1) { + DIFF_MSEC_ALWAYS("combined_determinise", pre, post, &timeout_accum); + } else { + DIFF_MSEC("combined_determinise", pre, post, &timeout_accum); + } + assert(timeout_accum < timeout); + timeout_accum = 0; + } + + const unsigned states_after_determinise = fsm_countstates(combined_fsm); + if (verbosity > 1) { + fprintf(stderr, "%s: combined_fsm: %d states after determinise\n", + __func__, states_after_determinise); + } + + TIME(&pre); + if (!fsm_minimise(combined_fsm)) { + goto cleanup; + } + TIME(&post); + if (timeout != 0) { + if (verbosity > 1) { + DIFF_MSEC_ALWAYS("combined_minimise", pre, post, &timeout_accum); + } else { + DIFF_MSEC("combined_minimise", pre, post, &timeout_accum); + } + assert(timeout_accum < timeout); + timeout_accum = 0; + } + + const unsigned states_after_minimise = fsm_countstates(combined_fsm); + if (verbosity > 1) { + fprintf(stderr, "%s: combined_fsm: %d states after minimise\n", + __func__, states_after_minimise); + } + + if (verbosity > 4) { + fprintf(stderr, "==== combined\n"); + fsm_print_fsm(stderr, combined_fsm); + fsm_capture_dump(stderr, "combined", combined_fsm); + } + + res = compare_combined_and_separate(verbosity, max_length, + count, combined_fsm, bases, (struct fsm **)fsms_cp); + + for (i = 0; i < count; i++) { + fsm_free(fsms_cp[i]); + } + fsm_free(combined_fsm); + + if (res == EXIT_SUCCESS) { + static size_t pass_count; + if (verbosity == 1) { + fprintf(stderr, "%s: pass: %zu, %zu patterns\n", + __func__, ++pass_count, count); + } else if (verbosity > 1) { + fprintf(stderr, "%s: pass: %zu, %zu patterns\n", + __func__, ++pass_count, count); + for (i = 0; i < count; i++) { + fprintf(stderr, " -- %zu: \"%s\"\n", + i, patterns[i]); + } + } + } + + return res; + +cleanup: + for (i = 0; i < count; i++) { + if (fsms[i] != NULL) { + fsm_free(fsms[i]); + } + if (fsms_cp[i] != NULL) { + fsm_free(fsms_cp[i]); + } + } + if (combined_fsm != NULL) { + fsm_free(combined_fsm); + } + return res; +} + +struct cmp_combined_env { + bool ok; + int verbosity; + size_t count; + struct fsm *combined_fsm; + const struct fsm_combined_base_pair *bases; + size_t current_i; + struct fsm **fsms; + size_t max_depth; + size_t max_steps; + size_t max_match_count; +}; + +static int +compare_combined_and_separate(int verbosity, size_t max_length, size_t count, + struct fsm *combined_fsm, const struct fsm_combined_base_pair *bases, + struct fsm **fsms) +{ + const size_t max_steps = get_env_config(DEF_MAX_STEPS, "MAX_STEPS"); + const size_t max_depth = get_env_config(DEF_MAX_DEPTH, "MAX_DEPTH"); + const size_t max_match_count = get_env_config(DEF_MAX_MATCH_COUNT, "MAX_MATCH_COUNT"); + + struct cmp_combined_env env = { + .ok = true, + .verbosity = verbosity, + .count = count, + .combined_fsm = combined_fsm, + .bases = bases, + .fsms = fsms, + .max_steps = max_steps, + .max_depth = max_depth, + .max_match_count = max_match_count, + }; + + /* For each individual fsm, generate matching inputs and check that + * they match with the same captures in the combined fsm. */ + for (env.current_i = 0; env.current_i < count; env.current_i++) { + if (!fsm_generate_matches(env.fsms[env.current_i], max_length, + cmp_combined_and_separate_cb, &env)) { + env.ok = false; + } + if (!env.ok) { + break; + } + } + + /* TODO: also generate matches with combined and check the individual ones */ + + return env.ok ? EXIT_SUCCESS : EXIT_FAILURE; +} + +static enum fsm_generate_matches_cb_res +cmp_combined_and_separate_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + struct cmp_combined_env *env = opaque; + + if (steps > env->max_steps) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (depth > env->max_depth) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + if (match_count > env->max_match_count) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + +#define MAX_CAPTURES 256 + struct fsm_capture captures_single[MAX_CAPTURES]; + struct fsm_capture captures_combined[MAX_CAPTURES]; + + const uint8_t *u8_input = (const uint8_t *)input; + const int res_combined = fsm_exec_with_captures(env->combined_fsm, u8_input, input_length, + &end_state, captures_combined, MAX_CAPTURES); + const int res_single = fsm_exec_with_captures(fsm, u8_input, input_length, + &end_state, captures_single, MAX_CAPTURES); + + if (res_combined != res_single) { + env->ok = false; + if (env->verbosity > 0) { + fprintf(stderr, "%s: res_combined %d != res_single %d\n", + __func__, res_combined, res_single); + } + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (res_single > 0) { + if (env->verbosity > 3) { + fprintf(stderr, "%s: res %d (single and combined)\n", __func__, res_single); + } + + bool matching = true; + const unsigned base = env->bases[env->current_i].capture; + assert(base < MAX_CAPTURES); + for (int i = 0; i < res_single; i++) { + if (env->verbosity > 3) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + if ((captures_single[i].pos[0] != captures_combined[i + base].pos[0]) || + (captures_single[i].pos[1] != captures_combined[i + base].pos[1])) { + matching = false; + } + } + + if (!matching) { + for (int i = 0; i < res_single; i++) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; +} + #define DEF_MAX_SHUFFLE 10 #define DEF_MAX_MINIMISE_ORACLE_STATE_COUNT 1000 @@ -169,7 +1186,7 @@ shuffle_minimise(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, &fsm_options, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ @@ -324,8 +1341,81 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const return EXIT_SUCCESS; } -#define MAX_FUZZER_DATA (64 * 1024) -static uint8_t data_buf[MAX_FUZZER_DATA + 1]; +static int +build_and_test_idempotent_det_and_min(const char *pattern) +{ + const int verbosity = get_env_config(0, "VERBOSITY"); + assert(pattern != NULL); + + struct re_err err; + struct fsm *fsm; + const size_t length = strlen(pattern); + + struct scanner s = { + .str = (const uint8_t *)pattern, + .size = length, + }; + + fsm = re_comp(RE_PCRE, scanner_next, &s, &fsm_options, RE_MULTI, &err); + if (fsm == NULL) { + return EXIT_SUCCESS; + } + + if (!fsm_determinise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_det_a\n"); + fsm_print_fsm(stderr, fsm); + } + const size_t post_det_a = fsm_countstates(fsm); + + if (!fsm_determinise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_det_b\n"); + fsm_print_fsm(stderr, fsm); + } + const size_t post_det_b = fsm_countstates(fsm); + assert(post_det_b == post_det_a); + + if (!fsm_minimise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_min_a\n"); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "post_a", fsm); + } + const size_t post_min_a = fsm_countstates(fsm); + + if (!fsm_minimise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_min_b\n"); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "post_b", fsm); + } + const size_t post_min_b = fsm_countstates(fsm); + assert(post_min_b == post_min_a); + + if (!fsm_determinise(fsm)) { + return EXIT_FAILURE; + } + const size_t post_det_c = fsm_countstates(fsm); + assert(post_det_c == post_min_b); + + if (!fsm_minimise(fsm)) { + return EXIT_FAILURE; + } + const size_t post_min_c = fsm_countstates(fsm); + assert(post_min_c == post_det_c); + + fsm_free(fsm); + return EXIT_SUCCESS; +} static enum run_mode get_run_mode(void) @@ -333,14 +1423,26 @@ get_run_mode(void) const char *mode = getenv("MODE"); if (mode != NULL) { switch (mode[0]) { - case 'm': return MODE_SHUFFLE_MINIMISE; + case 'r': return MODE_REGEX; + case 's': return MODE_REGEX_SINGLE_ONLY; + case 'm': return MODE_REGEX_MULTI_ONLY; + case 'i': return MODE_IDEMPOTENT_DET_MIN; + case 'M': return MODE_SHUFFLE_MINIMISE; case 'p': return MODE_ALL_PRINT_FUNCTIONS; default: + fprintf(stderr, "Unrecognized mode '%c', expect one of:\n", mode[0]); + fprintf(stderr, " - r.egex (default)\n"); + fprintf(stderr, " - s.ingle regex only\n"); + fprintf(stderr, " - m.ulti regex only\n"); + fprintf(stderr, " - M.inimisation shuffling\n"); + fprintf(stderr, " - i.dempotent determinise/minimise\n"); + fprintf(stderr, " - p.rint functions\n"); + exit(EXIT_FAILURE); break; } } - return MODE_DEFAULT; + return MODE_REGEX; /* default */ } static FILE *dev_null = NULL; @@ -359,12 +1461,87 @@ harness_fuzzer_target(const uint8_t *data, size_t size) size = MAX_FUZZER_DATA; } memcpy(data_buf, data, size); + /* ensure the buffer is 0-terminated */ + data_buf[size] = 0; + + /* truncate to a valid c string */ + size = strlen((const char *)data_buf); + data_buf[size] = 0; + + /* reset for each run */ + allocator_stats.hwm = 0; + + size_t dot_count = 0; + bool has_newline = false; + size_t first_newline; + + for (size_t i = 0; i < size; i++) { + const uint8_t c = data_buf[i]; + if (c == '.') { + dot_count++; + if (dot_count >= 4) { + /* Too many '.'s can lead to a regex that is + * very slow to determinise/minimise, but that + * failure mode is not interesting to this + * particular fuzzer. */ + return EXIT_SUCCESS; + } + } + + if (c == '(') { + /* This triggers an "unreached" assertion in the parser. + * It's already been reported (issue #386), but once the + * fuzzer finds it, it will report it over and over. + * Exit here so that the fuzzer considers it uninteresting. */ + if (size - i >= 3 && 0 == memcmp("(*:", &data_buf[i], 3)) { + return EXIT_SUCCESS; + } + } + + if (c == '\\') { + /* Not supported yet. */ + return EXIT_SUCCESS; + } + + if (c == '\r' || c == '\n') { + if (!has_newline) { + first_newline = i; + } + has_newline = true; + } + } const char *pattern = (const char *)data_buf; switch (run_mode) { - case MODE_DEFAULT: - return build_and_codegen(pattern); + case MODE_REGEX: + if (has_newline) { + return build_and_check_multi(pattern); + } else { + return build_and_check_single(pattern); + } + + case MODE_REGEX_SINGLE_ONLY: + if (has_newline) { + return EXIT_SUCCESS; /* ignore */ + } else { + return build_and_check_single(pattern); + } + + case MODE_REGEX_MULTI_ONLY: + if (has_newline) { + return build_and_check_multi(pattern); + } else { + return EXIT_SUCCESS; /* ignore */ + } + + case MODE_IDEMPOTENT_DET_MIN: + if (has_newline) { + assert(data_buf[first_newline] == '\n' + || data_buf[first_newline] == '\r'); + data_buf[first_newline] = '\0'; + } + return build_and_test_idempotent_det_and_min(pattern); case MODE_SHUFFLE_MINIMISE: return shuffle_minimise(pattern); @@ -385,5 +1562,8 @@ harness_fuzzer_target(const uint8_t *data, size_t size) int res = fuzz_all_print_functions(dev_null, shifted_pattern, det, min, io_mode); return res; } + +default: + assert(!"match fail"); } } From 318c9ad41380d6ef340a7cd41703cf7ba067f3a0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 2 Jun 2023 14:24:16 -0400 Subject: [PATCH 010/373] re: Add -FC flag to disable captures. I'm adding this because many of the existing tests under `tests/pcre-anchor/` and so on contain regexes that would now be rejected as unsupported in combination with captures, but they are testing cases unrelated to capturing. --- src/re/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/re/main.c b/src/re/main.c index 226267196..fd9ea2424 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -618,6 +618,10 @@ parse_flags(const char *arg, enum re_flags *flags) *flags = *flags | RE_EXTENDED; break; + case 'C': + *flags = *flags | RE_NOCAPTURE; + break; + /* others? */ default: From 5e4971354677b9478fe7f38f3f5d4a23fdaf55bf Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 16 Feb 2023 18:39:42 -0500 Subject: [PATCH 011/373] tests/*/Makefile: Add `-FC` (no captures) for some calls to RE. There are several tests that have nothing to do with captures, capture behavior is tested directly with `tests/captures/`. --- tests/aho_corasick/Makefile | 8 ++++---- tests/ir/Makefile | 2 +- tests/native/Makefile | 6 +++--- tests/pcre-anchor/Makefile | 4 ++-- tests/pcre-classes/Makefile | 10 +++++----- tests/pcre-flags/Makefile | 6 +++--- tests/pcre-repeat/Makefile | 6 +++--- tests/pcre/Makefile | 10 +++++----- tests/re_literal/Makefile | 2 +- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/aho_corasick/Makefile b/tests/aho_corasick/Makefile index 5748ddd5c..6eef421bf 100644 --- a/tests/aho_corasick/Makefile +++ b/tests/aho_corasick/Makefile @@ -33,7 +33,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}a.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}a.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}a.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}a.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt @@ -48,7 +48,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}l.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}l.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}l.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}l.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt @@ -63,7 +63,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}r.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}r.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}r.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}r.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt @@ -79,7 +79,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}u.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}u.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}u.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}u.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt diff --git a/tests/ir/Makefile b/tests/ir/Makefile index 0009c45ec..566d1add8 100755 --- a/tests/ir/Makefile +++ b/tests/ir/Makefile @@ -9,7 +9,7 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/ir:T:Mout*.json:R:C/^out//} ${TEST_OUTDIR.tests/ir}/got${n}.json: ${TEST_SRCDIR.tests/ir}/in${n}.re - ${RE} -pl irjson -y ${.ALLSRC:M*.re} \ + ${RE} -FC -pl irjson -y ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/ir}/res${n}: \ diff --git a/tests/native/Makefile b/tests/native/Makefile index 8712e1588..fbca0ca69 100755 --- a/tests/native/Makefile +++ b/tests/native/Makefile @@ -9,11 +9,11 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/native:T:Mout*.fsm:R:C/^out//} ${TEST_OUTDIR.tests/native}/got${n}.fsm: ${TEST_SRCDIR.tests/native}/in${n}.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/native}/nfa${n}.fsm: ${TEST_SRCDIR.tests/native}/in${n}.re - ${RE} -r native -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/native}/res${n}: \ @@ -27,7 +27,7 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/native}/res${n} .for n in ${TEST.tests/native:T:Mout*.err:R:C/^out//} ${TEST_OUTDIR.tests/native}/got${n}.err: ${TEST_SRCDIR.tests/native}/in${n}.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ 2> $@; [ $$? -ne 0 ] ${TEST_OUTDIR.tests/native}/res${n}: \ diff --git a/tests/pcre-anchor/Makefile b/tests/pcre-anchor/Makefile index bb9954554..1dc4a77bc 100644 --- a/tests/pcre-anchor/Makefile +++ b/tests/pcre-anchor/Makefile @@ -9,11 +9,11 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/pcre-anchor:T:Mout*.fsm:R:C/^out//} ${TEST_OUTDIR.tests/pcre-anchor}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-anchor}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-anchor}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-anchor}/in${n}.re - ${RE} -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-anchor}/res${n}: \ diff --git a/tests/pcre-classes/Makefile b/tests/pcre-classes/Makefile index 0d9809d76..0d459e256 100755 --- a/tests/pcre-classes/Makefile +++ b/tests/pcre-classes/Makefile @@ -16,7 +16,7 @@ RE=${BUILD}/bin/re FSM=${BUILD}/bin/fsm ${TEST_OUTDIR.tests/pcre-classes}/dot-all.fsm: - ${RE} -r pcre -p '^[\x00-\xff]$$' | ${FSM} -pm \ + ${RE} -FC -r pcre -p '^[\x00-\xff]$$' | ${FSM} -pm \ > $@ # compl.re tests @@ -32,7 +32,7 @@ ${TEST_OUTDIR.tests/pcre-classes}/dot-all.fsm: .for n in ${TEST.tests/pcre-classes:M*/compl*.re:T:R:C/^compl//} ${TEST_OUTDIR.tests/pcre-classes}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/got-compl${n}.fsm: ${TEST_OUTDIR.tests/pcre-classes}/got${n}.fsm ${TEST_OUTDIR.tests/pcre-classes}/dot-all.fsm @@ -40,7 +40,7 @@ ${TEST_OUTDIR.tests/pcre-classes}/got-compl${n}.fsm: ${TEST_OUTDIR.tests/pcre-cl > $@ ${TEST_OUTDIR.tests/pcre-classes}/expect-compl${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/compl${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/compl*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/compl*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/res${n}: \ @@ -66,11 +66,11 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/pcre-classes}/res${n} @echo x: ${n} ${TEST_OUTDIR.tests/pcre-classes}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/out${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/equal${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/equal*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/equal*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/res${n}: \ diff --git a/tests/pcre-flags/Makefile b/tests/pcre-flags/Makefile index 67e70fbbb..308571395 100755 --- a/tests/pcre-flags/Makefile +++ b/tests/pcre-flags/Makefile @@ -13,17 +13,17 @@ RE=${BUILD}/bin/re TEST_OUTDIR.tests/pcre-flags/mode${n} != cat ${TEST_SRCDIR.tests/pcre-flags}/mode${n} ${TEST_OUTDIR.tests/pcre-flags}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re - ${RE} -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-flags}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re - ${RE} -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ .else ${TEST_OUTDIR.tests/pcre-flags}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re - ${RE} -b -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -b -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-flags}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re diff --git a/tests/pcre-repeat/Makefile b/tests/pcre-repeat/Makefile index c325d2f8e..97535b2c3 100755 --- a/tests/pcre-repeat/Makefile +++ b/tests/pcre-repeat/Makefile @@ -12,11 +12,11 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/pcre-repeat:T:Mout*.fsm:R:C/^out//} ${TEST_OUTDIR.tests/pcre-repeat}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-repeat}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-repeat}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-repeat}/in${n}.re - ${RE} -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-repeat}/res${n}: \ @@ -30,7 +30,7 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/pcre-repeat}/res${n} .for n in ${TEST.tests/pcre-repeat:T:Mout*.err:R:C/^out//} ${TEST_OUTDIR.tests/pcre-repeat}/got${n}.err: ${TEST_SRCDIR.tests/pcre-repeat}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ 2> $@; [ $$? -ne 0 ] ${TEST_OUTDIR.tests/pcre-repeat}/res${n}: \ diff --git a/tests/pcre/Makefile b/tests/pcre/Makefile index 23f879a04..239d2c93b 100755 --- a/tests/pcre/Makefile +++ b/tests/pcre/Makefile @@ -21,7 +21,7 @@ PCREGREP ?= pcregrep # bit of a hack. # 2) removes any trailing \n at the end of the input ${TEST_OUTDIR.tests/pcre-pcregrep}/in${n}.txt: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -mr pcre -y ${.ALLSRC:M*.re} \ + ${RE} -FC -mr pcre -y ${.ALLSRC:M*.re} \ | perl -0pe 's/\\x([0-9a-zA-z]{2})/chr(hex($$1))/ge;' -e 's/\n\Z//' \ > $@ @@ -41,16 +41,16 @@ test:: ${TEST_OUTDIR.tests/pcre-pcregrep}/res${n} .if exists(${TEST_SRCDIR.tests/pcre}/mode${n}) TEST_OUTDIR.tests/pcre/mode${n} != cat ${TEST_SRCDIR.tests/pcre}/mode$n ${TEST_OUTDIR.tests/pcre}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -F "${TEST_OUTDIR.tests/pcre/mode${n}}" -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -F "${TEST_OUTDIR.tests/pcre/mode${n}}" -r pcre -py ${.ALLSRC:M*.re} \ > $@ .else ${TEST_OUTDIR.tests/pcre}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ > $@ .endif ${TEST_OUTDIR.tests/pcre}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre}/res${n}: \ @@ -64,7 +64,7 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/pcre}/res${n} .for n in ${TEST.tests/pcre:T:Mout*.err:R:C/^out//} ${TEST_OUTDIR.tests/pcre}/got${n}.err: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ 2> $@; [ $$? -ne 0 ] ${TEST_OUTDIR.tests/pcre}/res${n}: \ diff --git a/tests/re_literal/Makefile b/tests/re_literal/Makefile index 27941b094..41680e4df 100755 --- a/tests/re_literal/Makefile +++ b/tests/re_literal/Makefile @@ -9,7 +9,7 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/re_literal:T:R:C/^out//} ${TEST_OUTDIR.tests/re_literal}/got${n}.txt: ${TEST_SRCDIR.tests/re_literal}/in${n}.re - ( ${RE} -r pcre -t -y ${.ALLSRC:M*.re} || echo non-literal ) \ + ( ${RE} -FC -r pcre -t -y ${.ALLSRC:M*.re} || echo non-literal ) \ > $@ ${TEST_OUTDIR.tests/re_literal}/res${n}: \ From 6e410b3b4d1ac1ae7997678c9c56401f7c0278d4 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 2 Jun 2023 14:49:05 -0400 Subject: [PATCH 012/373] ast_analysis: Reject '\z' as unsupported. We currently get the wrong capture result for it. --- src/libre/ast_analysis.c | 6 ++++++ tests/capture/capture_test_case_list.c | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 70e65cad4..5e811bdb0 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1836,7 +1836,13 @@ analysis_iter_captures(struct capture_env *env, struct ast_expr *n) switch (n->type) { case AST_EXPR_EMPTY: case AST_EXPR_TOMBSTONE: + break; + case AST_EXPR_ANCHOR: + if (env->use_captures && n->u.anchor.type == AST_ANCHOR_END && !n->u.anchor.is_end_nl) { + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } break; case AST_EXPR_LITERAL: diff --git a/tests/capture/capture_test_case_list.c b/tests/capture/capture_test_case_list.c index 6a5a36a19..ea82e82de 100644 --- a/tests/capture/capture_test_case_list.c +++ b/tests/capture/capture_test_case_list.c @@ -1376,6 +1376,13 @@ const struct captest_case_single single_cases[] = { { .pos = {0, 1}, }, }, }, + + { + .regex = "\\z", + .input = "", + .count = 1, + .match = SHOULD_REJECT_AS_UNSUPPORTED, + }, }; const struct captest_case_multi multi_cases[] = { From 86ec9e4757ba9d22090f3bbbb4c5e7e1229b7e49 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 15 Jun 2023 12:26:18 -0400 Subject: [PATCH 013/373] union.c: Remove EXPENSIVE_CHECKS based on removed interface. Now instead of exposing exactly how many captures the fsm has, we keep track of the ceiling of the count, to track how large the capture buffer needs to be. We could add this back if fsm_capture_count gets re-added. --- src/libfsm/union.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 860cdc2c8..736292a8b 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -104,15 +104,8 @@ fsm_union_array(size_t fsm_count, fsms[0] = NULL; memset(bases, 0x00, fsm_count * sizeof(bases[0])); -#if EXPENSIVE_CHECKS - size_t capture_count = fsm_capture_count(res); -#endif - for (i = 1; i < fsm_count; i++) { struct fsm_combine_info ci; -#if EXPENSIVE_CHECKS - capture_count += fsm_capture_count(fsms[i]); -#endif struct fsm *combined = fsm_union(res, fsms[i], &ci); fsms[i] = NULL; @@ -149,8 +142,5 @@ fsm_union_array(size_t fsm_count, } #endif -#if EXPENSIVE_CHECKS - assert(fsm_capture_count(res) == capture_count); -#endif return res; } From d54ba0d5db6cab12069a9e8e249d6801824e703d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 15 Jun 2023 12:27:43 -0400 Subject: [PATCH 014/373] bugfix: resize endid buffer for carry_end_metadata properly. Previously we doubled the buffer if it wasn't large enough, but the next endid array may be > twice the size of the old buffer, so we need to keep expanding until it's large enough. We weren't saving the updated array size, so this could potentially lead to repeated doubling and eventually allocation failures. Also, change the assert for fsm_getendids's result -- if we get to that point, it should always be found, not just not an insufficient space error. --- src/libfsm/epsilons.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index e63ae50ce..52b73db6c 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -265,20 +265,26 @@ carry_end_metadata(struct carry_end_metadata_env *env, enum fsm_getendids_res id_res; size_t written; if (id_count > env->end.ceil) { /* grow buffer */ - const size_t nceil = (env->end.ceil == 0) + size_t nceil = (env->end.ceil == 0) ? DEF_END_METADATA_ENDIDS_CEIL : 2*env->end.ceil; + while (nceil < id_count) { + nceil *= 2; + } assert(nceil > 0); - env->end.ids = f_realloc(env->alloc, + fsm_end_id_t *nids = f_realloc(env->alloc, env->end.ids, nceil * sizeof(env->end.ids[0])); - if (env->end.ids == NULL) { + if (nids == NULL) { return 0; } + env->end.ids = nids; + env->end.ceil = nceil; } id_res = fsm_getendids(env->fsm, end_state, id_count, env->end.ids, &written); - assert(id_res != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); + assert(id_res == FSM_GETENDIDS_FOUND); + assert(written == id_count); for (i = 0; i < id_count; i++) { #if LOG_COPYING From e2ac4d7ba4630b52eb39a89abedc1f81b9b5a039 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 15 Jun 2023 14:00:25 -0400 Subject: [PATCH 015/373] Address a couple warnings from scan-build. determinise: It's not possible to find a cached result in the hash table without allocating a to-set buffer first, so assert that it will be non-NULL. fsm_findmode: This should never be used on a state without edges. vm/v1.c and vm/v2.c: Free allocated return value on error. --- src/libfsm/determinise.c | 2 ++ src/libfsm/mode.c | 4 ++++ src/libfsm/vm/v1.c | 4 +++- src/libfsm/vm/v2.c | 5 ++++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 817fd335b..3dc1a4268 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1260,6 +1260,7 @@ to_set_htab_check(struct analyze_closures_env *env, if (b->count == 0) { return 0; /* empty bucket -> not found */ } else if (b->count == count) { + assert(env->to_sets.buf != NULL); assert(b->offset + count <= env->to_sets.used); const fsm_state_t *ids = &env->to_sets.buf[b->offset]; if (0 == memcmp(ids, dst, count * sizeof(dst[0]))) { @@ -1386,6 +1387,7 @@ save_to_set(struct analyze_closures_env *env, env->to_sets.ceil = nceil; env->to_sets.buf = nbuf; } + assert(env->to_sets.buf != NULL); #if LOG_TO_SET static size_t to_set_id; diff --git a/src/libfsm/mode.c b/src/libfsm/mode.c index 76c60b8ad..87af0bdf9 100644 --- a/src/libfsm/mode.c +++ b/src/libfsm/mode.c @@ -28,6 +28,7 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) } mode; mode.freq = 1; + mode.state = (fsm_state_t)-1; edge_set_group_iter_reset(fsm->states[state].edges, EDGE_GROUP_ITER_ALL, &iter); while (edge_set_group_iter_next(&iter, &info)) { @@ -46,6 +47,9 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) *freq = mode.freq; } + /* It's not meaningful to call this on a state without edges. */ + assert(mode.state != (fsm_state_t)-1); + assert(mode.freq >= 1); return mode.state; } diff --git a/src/libfsm/vm/v1.c b/src/libfsm/vm/v1.c index a326b88d8..de1f6ea93 100644 --- a/src/libfsm/vm/v1.c +++ b/src/libfsm/vm/v1.c @@ -217,7 +217,9 @@ encode_opasm_v1(const struct dfavm_vm_op *instr, size_t ninstr, size_t total_byt return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } return NULL; } diff --git a/src/libfsm/vm/v2.c b/src/libfsm/vm/v2.c index c85edff98..07eb12ef4 100644 --- a/src/libfsm/vm/v2.c +++ b/src/libfsm/vm/v2.c @@ -155,7 +155,10 @@ encode_opasm_v2(const struct dfavm_vm_op *instr, size_t ninstr) return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } + return NULL; } From c11f372ae5656e8a7e59e2d421bfb8bc17593a99 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 15 Jun 2023 15:59:26 -0400 Subject: [PATCH 016/373] fuzz/target.c: In MULTI mode, check endid behavior. For each pattern 0..n that will be combined, set an endid on them. Then, generate inputs that match, and check that the endid result on the single and combined FSMs are consistent. --- fuzz/target.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 1b5b30e7e..21d6589de 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -877,6 +877,11 @@ build_and_check_multi(const char *input) } if (j > 0) { count++; } + if (count == 1) { + /* multi mode with only one pattern is pointless */ + return EXIT_SUCCESS; + } + struct re_err err; const enum re_flags flags = 0; @@ -894,6 +899,7 @@ build_and_check_multi(const char *input) } /* compile each individually */ + /* FIXME: apply and check endids */ for (size_t i = 0; i < count; i++) { if (verbosity > 1) { fprintf(stderr, "%s: compiling \"%s\"\n", @@ -911,6 +917,11 @@ build_and_check_multi(const char *input) goto cleanup; } + /* set endid to associate each FSM with its pattern */ + if (!fsm_setendid(fsm, (fsm_end_id_t)i)) { + goto cleanup; + } + char label_buf[100]; snprintf(label_buf, 100, "single_determisise_%zu", i); @@ -1102,6 +1113,7 @@ cmp_combined_and_separate_cb(const struct fsm *fsm, fsm_state_t end_state, void *opaque) { struct cmp_combined_env *env = opaque; + (void)end_state; if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; @@ -1119,11 +1131,15 @@ cmp_combined_and_separate_cb(const struct fsm *fsm, struct fsm_capture captures_single[MAX_CAPTURES]; struct fsm_capture captures_combined[MAX_CAPTURES]; + const fsm_end_id_t expected_end_id = (fsm_end_id_t)env->current_i; + const uint8_t *u8_input = (const uint8_t *)input; + fsm_state_t end_state_combined, end_state_single; + const int res_combined = fsm_exec_with_captures(env->combined_fsm, u8_input, input_length, - &end_state, captures_combined, MAX_CAPTURES); + &end_state_combined, captures_combined, MAX_CAPTURES); const int res_single = fsm_exec_with_captures(fsm, u8_input, input_length, - &end_state, captures_single, MAX_CAPTURES); + &end_state_single, captures_single, MAX_CAPTURES); if (res_combined != res_single) { env->ok = false; @@ -1134,11 +1150,44 @@ cmp_combined_and_separate_cb(const struct fsm *fsm, return FSM_GENERATE_MATCHES_CB_RES_HALT; } + fsm_end_id_t id_buf_combined[MAX_PATTERNS]; + size_t written_combined = 0; + if (res_combined > 0) { + const size_t exp_written = fsm_getendidcount(env->combined_fsm, end_state_combined); + assert(exp_written <= env->count); + const enum fsm_getendids_res gres = fsm_getendids(env->combined_fsm, + end_state_combined, MAX_PATTERNS, id_buf_combined, &written_combined); + assert(gres == FSM_GETENDIDS_FOUND); + assert(written_combined == exp_written); + } + if (res_single > 0) { if (env->verbosity > 3) { fprintf(stderr, "%s: res %d (single and combined)\n", __func__, res_single); } + /* Check that the end state's endid for the single DFA is among the + * endids for the combined DFA's end state. */ + assert(fsm_getendidcount(fsm, end_state_single) == 1); + assert(fsm_getendidcount(env->combined_fsm, end_state_combined) <= env->count); + + fsm_end_id_t id_buf_single[1]; + size_t written; + const enum fsm_getendids_res gres = fsm_getendids(fsm, + end_state_single, 1, id_buf_single, &written); + assert(gres == FSM_GETENDIDS_FOUND); + assert(written == 1); + assert(id_buf_single[0] == expected_end_id); + + bool found_single_id_in_combined = false; + for (size_t i = 0; i < written_combined; i++) { + if (id_buf_combined[i] == expected_end_id) { + found_single_id_in_combined = true; + break; + } + } + assert(found_single_id_in_combined); + bool matching = true; const unsigned base = env->bases[env->current_i].capture; assert(base < MAX_CAPTURES); @@ -1165,6 +1214,18 @@ cmp_combined_and_separate_cb(const struct fsm *fsm, env->ok = false; return FSM_GENERATE_MATCHES_CB_RES_HALT; } + } else if (res_combined > 0) { + /* This matched the combined DFA but not the single one, + * so check that the single DFA's end id is *absent* + * from the combined DFA's end state. */ + bool found_single_id_in_combined = false; + for (size_t i = 0; i < written_combined; i++) { + if (id_buf_combined[i] == expected_end_id) { + found_single_id_in_combined = true; + break; + } + } + assert(!found_single_id_in_combined); } return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; From 8b49a1a1071df47107dbb370c0490bb06f861310 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 10 Jul 2023 13:50:27 -0400 Subject: [PATCH 017/373] minimisation: Distinct sets of end IDs should not split ECs. In minimise.c, `split_ecs_by_end_metadata` does a first pass splitting ECs based on their end data (like the name says). This sets which end metadata should prevent states from starting out in the same EC, effectively which states can/cannot be combined once minimisation's analysis is done. Previously, distinct sets of end IDs would keep states from merging, but if epsilon removal and determinisation have led to end states with distinct sets of end IDs, that alone shouldn't prevent them from merging later -- the same end state just becomes associated with all those end IDs. We do prevent states with distinct capture ID sets from merging, but that's because of a few special cases like "^a(b)c$" and "^a(b*)c$", where combining partially overlapping regexes' end states could lead to false positives in the capture result. Note: I added checking the program IDs (which was missing) to `same_end_metadata`. This seems correct to me, but at the moment I can't think of any test inputs that would lead to the same sets of capture IDs but distinct sets of program IDs. I will see if fuzzing can find any. This is tested by tests/endids/endids2_union_many_endids.c and the new multi test in tests/capture/capture_test_case_list.c. --- src/libfsm/minimise.c | 76 ++------------------------ tests/capture/capture_test_case_list.c | 70 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 70 deletions(-) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 39929b4ac..6cfd7968e 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -53,11 +53,6 @@ all_end_states_are_currently_together(const struct min_env *env); #define DEF_CAPTURE_ID_CEIL 4 struct end_metadata { - struct end_metadata_end { - unsigned count; - fsm_end_id_t *ids; - } end; - struct end_metadata_capture { unsigned count; unsigned ceil; @@ -71,10 +66,6 @@ struct end_metadata { } program; }; -static int -collect_end_ids(const struct fsm *fsm, fsm_state_t s, - struct end_metadata_end *e); - static int collect_capture_ids(const struct fsm *fsm, fsm_state_t s, struct end_metadata_capture *c); @@ -732,24 +723,23 @@ incremental_hash_of_ids(uint64_t *accum, fsm_end_id_t id) static int same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) { - if (a->end.count != b->end.count) { + if (a->capture.count != b->capture.count) { return 0; } - if (a->capture.count != b->capture.count) { + if (a->program.count != b->program.count) { return 0; } /* compare -- these must be sorted */ - for (size_t i = 0; i < a->end.count; i++) { - if (a->end.ids[i] != b->end.ids[i]) { + for (size_t i = 0; i < a->capture.count; i++) { + if (a->capture.ids[i] != b->capture.ids[i]) { return 0; } } - - for (size_t i = 0; i < a->capture.count; i++) { - if (a->capture.ids[i] != b->capture.ids[i]) { + for (size_t i = 0; i < a->program.count; i++) { + if (a->program.ids[i] != b->program.ids[i]) { return 0; } } @@ -806,13 +796,6 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) break; /* this EC has non-end states, skip */ } - /* FIXME: should distinct end IDs partition here? - * Disabled to make tests/endids/endids2_union_many_endids.c pass. */ - if (0 && - !collect_end_ids(fsm, s, &e->end)) { - goto cleanup; - } - if (!collect_capture_ids(fsm, s, &e->capture)) { goto cleanup; } @@ -849,9 +832,6 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) uint64_t hash = 0; const fsm_state_t next = env->jump[s]; - for (size_t eid_i = 0; eid_i < s_md->end.count; eid_i++) { - incremental_hash_of_ids(&hash, s_md->end.ids[eid_i]); - } for (size_t pid_i = 0; pid_i < s_md->program.count; pid_i++) { incremental_hash_of_ids(&hash, s_md->program.ids[pid_i]); } @@ -996,9 +976,6 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) size_t i; for (i = 0; i < state_count; i++) { struct end_metadata *e = &end_md[i]; - if (e->end.ids != NULL) { - f_free(fsm->opt->alloc, e->end.ids); - } if (e->capture.ids != NULL) { f_free(fsm->opt->alloc, e->capture.ids); } @@ -1012,14 +989,6 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) return res; } -static int -cmp_end_ids(const void *pa, const void *pb) -{ - const fsm_end_id_t a = *(fsm_end_id_t *)pa; - const fsm_end_id_t b = *(fsm_end_id_t *)pb; - return a < b ? -1 : a > b ? 1 : 0; -} - static int cmp_unsigned(const void *pa, const void *pb) { @@ -1028,39 +997,6 @@ cmp_unsigned(const void *pa, const void *pb) return a < b ? -1 : a > b ? 1 : 0; } -static int -collect_end_ids(const struct fsm *fsm, fsm_state_t s, - struct end_metadata_end *e) -{ - e->count = fsm_getendidcount(fsm, s); - - if (e->count > 0) { - e->ids = f_malloc(fsm->opt->alloc, - e->count * sizeof(e->ids[0])); - if (e->ids == NULL) { - return 0; - } - -#if LOG_ECS - fprintf(stderr, "%d:", s); -#endif - - size_t written; - enum fsm_getendids_res gres = fsm_getendids(fsm, - s, e->count, e->ids, &written); - assert(gres == FSM_GETENDIDS_FOUND); - assert(written == e->count); - -#if LOG_ECS - fprintf(stderr, "\n"); -#endif - /* sort, to make comparison easier later */ - qsort(e->ids, e->count, - sizeof(e->ids[0]), cmp_end_ids); - } - return 1; -} - struct collect_capture_env { int ok; const struct fsm_alloc *alloc; diff --git a/tests/capture/capture_test_case_list.c b/tests/capture/capture_test_case_list.c index ea82e82de..d05210d2c 100644 --- a/tests/capture/capture_test_case_list.c +++ b/tests/capture/capture_test_case_list.c @@ -1525,6 +1525,76 @@ const struct captest_case_multi multi_cases[] = { }, }, }, + + { + /* This checks that minimisation doesn't incorrectly + * merge these and lead to capture false positives. */ + .regex_count = 2, + .regexes = { + "^a(b)c$", /* exactly one 'b' */ + "^a(b*)c$", /* any number of 'b's */ + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "a", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "ab", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "ac", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = { 0, 2 } }, + { .regex = 1, .capture = 1, .pos = { 1, 1 } }, + }, + }, + { + .input = "abc", + .expected = { + { .regex = 0, .capture = 0, .pos = {0, 3 } }, + { .regex = 0, .capture = 1, .pos = {1, 2 } }, + { .regex = 1, .capture = 0, .pos = { 0, 3 } }, + { .regex = 1, .capture = 1, .pos = { 1, 2 } }, + }, + }, + { + .input = "abbc", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = { 0, 4 } }, + { .regex = 1, .capture = 1, .pos = { 1, 3 } }, + }, + }, + + { + .input = NULL, + }, + }, + } }; From a3a79cac1b31bef478ce885017fd99f7ec7f1850 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 10 Jul 2023 14:02:31 -0400 Subject: [PATCH 018/373] Remove theft test harness for deleted ADT (ipriq). --- theft/Makefile | 1 - theft/fuzz_adt_ipriq.c | 197 ----------------------------------------- 2 files changed, 198 deletions(-) delete mode 100644 theft/fuzz_adt_ipriq.c diff --git a/theft/Makefile b/theft/Makefile index 0d38d8cfc..921c482a9 100644 --- a/theft/Makefile +++ b/theft/Makefile @@ -6,7 +6,6 @@ SRC += theft/util.c SRC += theft/wrap.c SRC += theft/fuzz_adt_edge_set.c -SRC += theft/fuzz_adt_ipriq.c SRC += theft/fuzz_adt_priq.c SRC += theft/fuzz_capture_string_set.c SRC += theft/fuzz_literals.c diff --git a/theft/fuzz_adt_ipriq.c b/theft/fuzz_adt_ipriq.c deleted file mode 100644 index 1847ef6ce..000000000 --- a/theft/fuzz_adt_ipriq.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2021 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include "type_info_adt_ipriq.h" - -#include -#include - -struct model { - size_t used; - size_t entries[]; -}; - -static enum ipriq_cmp_res -cmp_size_t(size_t a, size_t b, void *opaque) -{ - (void)opaque; - return a < b ? IPRIQ_CMP_LT : - a > b ? IPRIQ_CMP_GT : IPRIQ_CMP_EQ; -} - -static int exec_add(size_t x, struct model *m, struct ipriq *pq) -{ - if (!ipriq_add(pq, x)) { - return 0; - } - - m->entries[m->used] = x; - m->used++; - return 1; -} - -static int find_min_pos(const struct model *m, size_t *pos) -{ - size_t i; - if (m->used == 0) { - return 0; - } - - size_t res, min; - res = 0; - min = m->entries[0]; - - for (i = 1; i < m->used; i++) { - if (m->entries[i] < min) { - res = i; - min = m->entries[i]; - } - } - *pos = res; - return 1; -} - -static int exec_peek(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_peek(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (peek)"); - } - - return res == m->entries[pos]; -} - -static int exec_pop(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_pop(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (pop)"); - } - - if (res != m->entries[pos]) { - return 0; - } - - assert(m->used > 0); - if (pos < m->used - 1) { - m->entries[pos] = m->entries[m->used - 1]; - } - m->used--; - return 1; -} - -static enum theft_trial_res -compare_against_model(const struct ipriq_scenario *scen) -{ - enum theft_trial_res res = THEFT_TRIAL_FAIL; - size_t i; - - struct model *m = malloc(sizeof(*m) - + scen->count * sizeof(m->entries[0])); - if (m == NULL) { - return THEFT_TRIAL_ERROR; - } - m->used = 0; - - struct ipriq *pq = ipriq_new(NULL, cmp_size_t, NULL); - if (pq == NULL) { - return THEFT_TRIAL_ERROR; - } - - for (i = 0; i < scen->count; i++) { - const struct ipriq_op *op = &scen->ops[i]; - - switch (op->t) { - case IPRIQ_OP_ADD: - if (!exec_add(op->u.add.x, m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_PEEK: - if (!exec_peek(m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_POP: - if (!exec_pop(m, pq)) { - goto cleanup; - } - break; - - default: - assert(false); break; - } - } - - res = THEFT_TRIAL_PASS; - -cleanup: - free(m); - - return res; -} - -static enum theft_trial_res -prop_ipriq_model(struct theft *t, void *arg1) -{ - const struct ipriq_scenario *scen = arg1; - (void)t; - return compare_against_model(scen); -} - -static bool -test_ipriq(theft_seed seed, uintptr_t limit) -{ - enum theft_run_res res; - - struct ipriq_hook_env env = { - .tag = 'I', - .limit = limit, - }; - - struct theft_run_config config = { - .name = __func__, - .prop1 = prop_ipriq_model, - .type_info = { &type_info_adt_ipriq }, - .trials = 1000, - .hooks = { - .trial_pre = theft_hook_first_fail_halt, - .env = &env, - }, - .fork = { - .enable = true, - }, - - .seed = seed, - }; - - (void)limit; - - res = theft_run(&config); - printf("%s: %s\n", __func__, theft_run_res_str(res)); - - return res == THEFT_RUN_PASS; -} - -void -register_test_adt_ipriq(void) -{ - reg_test1("adt_ipriq", test_ipriq, 10000); -} From bfc9a804509c237894fd371c2e498740bee79736 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 11 Jul 2023 10:47:09 -0400 Subject: [PATCH 019/373] capture_vm_exec: Address scan-build warnings. There were several unused store warnings about values that were only set for logging, either `(void)x` them out or restructure the code so that their scope is more limited. Remove `next_offset` from `populate_solution`, since that isn't being used at all. IIRC it was added before some of the path sharing details made it unnecessary. --- src/libfsm/capture_vm_exec.c | 48 ++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/libfsm/capture_vm_exec.c b/src/libfsm/capture_vm_exec.c index 5e33f159c..f2d0b522e 100644 --- a/src/libfsm/capture_vm_exec.c +++ b/src/libfsm/capture_vm_exec.c @@ -659,6 +659,10 @@ cmp_paths(struct capvm *vm, uint32_t p_a, uint32_t p_b) const uint32_t backlink_a = get_path_node_backlink(vm, link_a); const uint32_t backlink_b = get_path_node_backlink(vm, link_b); + /* These are only used for logging, which may compile out. */ + (void)backlink_a; + (void)backlink_b; + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: link_a %d (offset %u, prev %d), link_b %d (offset %u, prev %d)\n", __func__, link_a, offset_a, prev_a, link_b, offset_b, prev_b); @@ -1506,6 +1510,7 @@ extend_path_info(struct capvm *vm, uint32_t pi_id, bool greedy, uint32_t uniq_id if (LOG_CAPVM >= 4 || 1) { const uint32_t refcount = get_path_node_refcount(vm, epi_id); + (void)refcount; LOG(4 - LOG_EPI, "%s: pi[%u] refcount %u -> %u (reusing identical path backlink %u instead of %u)\n", __func__, epi_id, refcount, refcount + 1, epi_id, pi_id); @@ -1593,7 +1598,6 @@ populate_solution(struct capvm *vm) uint32_t path_link = vm->solution.best_path_id; uint32_t next_link = NO_ID; - uint32_t next_offset = NO_POS; uint32_t first_link = NO_ID; size_t split_count = 0; @@ -1648,33 +1652,34 @@ populate_solution(struct capvm *vm) prev = prev_link; } - next_offset = get_path_node_offset(vm, path_link); next_link = path_link; assert(path_link != prev_link); path_link = prev_link; } while (path_link != NO_ID && path_link != COLLAPSED_ZERO_PREFIX_ID); /* iter forward */ - uint32_t cur = first_link; - if (LOG_CAPVM >= 3) do { - struct capvm_path_info *pi = &vm->paths.pool[cur]; + if (LOG_CAPVM >= 3) { + uint32_t cur = first_link; + do { + struct capvm_path_info *pi = &vm->paths.pool[cur]; - assert(IS_PATH_NODE(pi)); - LOG(3, "%s (moving fwd): node %u: refcount %u, used %u, offset %u, fwdlink %d, bits '", - __func__, cur, get_path_node_refcount(vm, cur), - pi->u.path.used, - get_path_node_offset(vm, cur), - get_path_node_backlink(vm, cur)); - for (uint8_t i = 0; i < pi->u.path.used; i++) { - const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); - LOG(3, "%c", bit ? '1' : '0'); - } - LOG(3, "'\n"); + assert(IS_PATH_NODE(pi)); + LOG(3, "%s (moving fwd): node %u: refcount %u, used %u, offset %u, fwdlink %d, bits '", + __func__, cur, get_path_node_refcount(vm, cur), + pi->u.path.used, + get_path_node_offset(vm, cur), + get_path_node_backlink(vm, cur)); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", bit ? '1' : '0'); + } + LOG(3, "'\n"); - const uint32_t next_cur = get_path_node_backlink(vm, cur); - assert(cur != next_cur); - cur = next_cur; /* fwd link */ - } while (cur != NO_ID); + const uint32_t next_cur = get_path_node_backlink(vm, cur); + assert(cur != next_cur); + cur = next_cur; /* fwd link */ + } while (cur != NO_ID); + } /* evaluate program with forward path */ LOG(3, "%s: split_count %zu\n", __func__, split_count); @@ -1693,7 +1698,7 @@ populate_solution(struct capvm *vm) * as (0,1). */ bool explicitly_matched_nl_at_end = false; - cur = first_link; + uint32_t cur = first_link; while (split_i < split_count || !done) { assert(prog_i < vm->p->used); const uint32_t cur_prog_i = prog_i; @@ -1901,6 +1906,7 @@ populate_solution(struct capvm *vm) for (uint8_t i = 0; i < pi->u.path.used; i++) { const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); LOG(3, "%c", (pi->u.path.bits & bit) ? '1' : '0'); + (void)bit; } LOG(3, "'\n"); From f43585ab012a2d922fb47ff48e6b0d9c3a390667 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 17 Jul 2023 13:39:15 -0400 Subject: [PATCH 020/373] fuzz/target.c: Expand build_and_check_multi. Also compare in the other direction, generating matching inputs from the combined DFA and then check that individual regexes's captures still match as expected. --- fuzz/target.c | 236 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 176 insertions(+), 60 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 21d6589de..0166f7cf2 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -813,12 +813,12 @@ compare_with_pcre(const char *pattern, struct fsm *fsm) /* Note: combined_fsm and fsms[] are non-const because fsm_generate_matches * calls fsm_trim on them. */ static int -compare_combined_and_separate(int verbosity, size_t max_length, size_t count, +compare_separate_and_combined(int verbosity, size_t max_length, size_t count, struct fsm *combined_fsm, const struct fsm_combined_base_pair *bases, struct fsm **fsms); static enum fsm_generate_matches_cb_res -cmp_combined_and_separate_cb(const struct fsm *fsm, +cmp_separate_and_combined_cb(const struct fsm *fsm, size_t depth, size_t match_count, size_t steps, const char *input, size_t input_length, fsm_state_t end_state, void *opaque); @@ -1015,7 +1015,7 @@ build_and_check_multi(const char *input) fsm_capture_dump(stderr, "combined", combined_fsm); } - res = compare_combined_and_separate(verbosity, max_length, + res = compare_separate_and_combined(verbosity, max_length, count, combined_fsm, bases, (struct fsm **)fsms_cp); for (i = 0; i < count; i++) { @@ -1068,8 +1068,14 @@ struct cmp_combined_env { size_t max_match_count; }; +static enum fsm_generate_matches_cb_res +cmp_combined_with_separate_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + static int -compare_combined_and_separate(int verbosity, size_t max_length, size_t count, +compare_separate_and_combined(int verbosity, size_t max_length, size_t count, struct fsm *combined_fsm, const struct fsm_combined_base_pair *bases, struct fsm **fsms) { @@ -1093,21 +1099,29 @@ compare_combined_and_separate(int verbosity, size_t max_length, size_t count, * they match with the same captures in the combined fsm. */ for (env.current_i = 0; env.current_i < count; env.current_i++) { if (!fsm_generate_matches(env.fsms[env.current_i], max_length, - cmp_combined_and_separate_cb, &env)) { + cmp_separate_and_combined_cb, &env)) { env.ok = false; } if (!env.ok) { break; } } + env.current_i = (size_t)-1; - /* TODO: also generate matches with combined and check the individual ones */ + /* Also go in the other direction, generating matches with + * combined and check the individual ones match as expected. */ + if (env.ok) { + if (!fsm_generate_matches(env.combined_fsm, max_length, + cmp_combined_with_separate_cb, &env)) { + env.ok = false; + } + } return env.ok ? EXIT_SUCCESS : EXIT_FAILURE; } static enum fsm_generate_matches_cb_res -cmp_combined_and_separate_cb(const struct fsm *fsm, +cmp_separate_and_combined_cb(const struct fsm *fsm, size_t depth, size_t match_count, size_t steps, const char *input, size_t input_length, fsm_state_t end_state, void *opaque) @@ -1161,76 +1175,178 @@ cmp_combined_and_separate_cb(const struct fsm *fsm, assert(written_combined == exp_written); } - if (res_single > 0) { + /* we got here, so we have a match */ + assert(res_single > 0); + + if (env->verbosity > 3) { + fprintf(stderr, "%s: res %d (single and combined)\n", __func__, res_single); + } + + /* Check that the end state's endid for the single DFA is among the + * endids for the combined DFA's end state. */ + assert(fsm_getendidcount(fsm, end_state_single) == 1); + assert(fsm_getendidcount(env->combined_fsm, end_state_combined) <= env->count); + + fsm_end_id_t id_buf_single[1]; + size_t written; + const enum fsm_getendids_res gres = fsm_getendids(fsm, + end_state_single, 1, id_buf_single, &written); + assert(gres == FSM_GETENDIDS_FOUND); + assert(written == 1); + assert(id_buf_single[0] == expected_end_id); + + bool found_single_id_in_combined = false; + for (size_t i = 0; i < written_combined; i++) { + if (id_buf_combined[i] == expected_end_id) { + found_single_id_in_combined = true; + break; + } + } + assert(found_single_id_in_combined); + + bool matching = true; + const unsigned base = env->bases[env->current_i].capture; + assert(base < MAX_CAPTURES); + for (int i = 0; i < res_single; i++) { if (env->verbosity > 3) { - fprintf(stderr, "%s: res %d (single and combined)\n", __func__, res_single); + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + if ((captures_single[i].pos[0] != captures_combined[i + base].pos[0]) || + (captures_single[i].pos[1] != captures_combined[i + base].pos[1])) { + matching = false; } + } + + if (!matching) { + for (int i = 0; i < res_single; i++) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; +} + +static enum fsm_generate_matches_cb_res +cmp_combined_with_separate_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + /* We have an input that matched the combined DFA, + * use the set of end IDs to check which of the + * single DFAs it should/should not match, and check + * the endid behavior. */ + + struct cmp_combined_env *env = opaque; + + if (steps > env->max_steps) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } - /* Check that the end state's endid for the single DFA is among the - * endids for the combined DFA's end state. */ - assert(fsm_getendidcount(fsm, end_state_single) == 1); - assert(fsm_getendidcount(env->combined_fsm, end_state_combined) <= env->count); + if (depth > env->max_depth) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + if (match_count > env->max_match_count) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + +#define MAX_CAPTURES 256 + struct fsm_capture captures_single[MAX_CAPTURES]; + struct fsm_capture captures_combined[MAX_CAPTURES]; - fsm_end_id_t id_buf_single[1]; - size_t written; - const enum fsm_getendids_res gres = fsm_getendids(fsm, - end_state_single, 1, id_buf_single, &written); + const uint8_t *u8_input = (const uint8_t *)input; + + fsm_state_t end_state_combined; + assert(fsm == env->combined_fsm); + const int res_combined = fsm_exec_with_captures(env->combined_fsm, u8_input, input_length, + &end_state_combined, captures_combined, MAX_CAPTURES); + assert(res_combined > 0); /* we got here, so we have a match */ + assert(end_state_combined == end_state); + + fsm_end_id_t id_buf_combined[MAX_PATTERNS]; + size_t written_combined = 0; + { + const size_t exp_written = fsm_getendidcount(env->combined_fsm, end_state_combined); + assert(exp_written <= env->count); + const enum fsm_getendids_res gres = fsm_getendids(env->combined_fsm, + end_state_combined, MAX_PATTERNS, id_buf_combined, &written_combined); assert(gres == FSM_GETENDIDS_FOUND); - assert(written == 1); - assert(id_buf_single[0] == expected_end_id); + assert(written_combined == exp_written); + } - bool found_single_id_in_combined = false; - for (size_t i = 0; i < written_combined; i++) { - if (id_buf_combined[i] == expected_end_id) { - found_single_id_in_combined = true; + /* For each pattern, check if its endid is in the combined DFA's end state + * endids. If so, it should match, otherwise it should not. */ + for (size_t pattern_i = 0; pattern_i < env->count; pattern_i++) { + const struct fsm *single_fsm = env->fsms[pattern_i]; + bool found = false; + for (size_t endid_i = 0; endid_i < written_combined; endid_i++) { + const fsm_end_id_t endid = id_buf_combined[endid_i]; + if (endid == pattern_i) { + found = true; break; } } - assert(found_single_id_in_combined); - - bool matching = true; - const unsigned base = env->bases[env->current_i].capture; - assert(base < MAX_CAPTURES); - for (int i = 0; i < res_single; i++) { - if (env->verbosity > 3) { - fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", - i, res_single, - captures_single[i].pos[0], captures_single[i].pos[1], - captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); - } - if ((captures_single[i].pos[0] != captures_combined[i + base].pos[0]) || - (captures_single[i].pos[1] != captures_combined[i + base].pos[1])) { - matching = false; - } - } - - if (!matching) { + fsm_state_t end_state_single; + + const int res_single = fsm_exec_with_captures(single_fsm, + u8_input, input_length, + &end_state_single, captures_single, MAX_CAPTURES); + + if (found) { + assert(res_single > 0); + fsm_end_id_t id_buf_single[1]; + size_t written; + const enum fsm_getendids_res gres = fsm_getendids(single_fsm, + end_state_single, 1, id_buf_single, &written); + assert(gres == FSM_GETENDIDS_FOUND); + assert(written == 1); + assert(id_buf_single[0] == pattern_i); + + /* check captures */ + bool matching = true; + const unsigned base = env->bases[pattern_i].capture; + assert(base < MAX_CAPTURES); for (int i = 0; i < res_single; i++) { - fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", - i, res_single, - captures_single[i].pos[0], captures_single[i].pos[1], - captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + if (env->verbosity > 3) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + if ((captures_single[i].pos[0] != captures_combined[i + base].pos[0]) || + (captures_single[i].pos[1] != captures_combined[i + base].pos[1])) { + matching = false; + } } - env->ok = false; - return FSM_GENERATE_MATCHES_CB_RES_HALT; - } - } else if (res_combined > 0) { - /* This matched the combined DFA but not the single one, - * so check that the single DFA's end id is *absent* - * from the combined DFA's end state. */ - bool found_single_id_in_combined = false; - for (size_t i = 0; i < written_combined; i++) { - if (id_buf_combined[i] == expected_end_id) { - found_single_id_in_combined = true; - break; + + if (!matching) { + for (int i = 0; i < res_single; i++) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; } + } else { + assert(res_single == 0); /* no match */ } - assert(!found_single_id_in_combined); } return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; } + #define DEF_MAX_SHUFFLE 10 #define DEF_MAX_MINIMISE_ORACLE_STATE_COUNT 1000 @@ -1622,7 +1738,7 @@ harness_fuzzer_target(const uint8_t *data, size_t size) const bool det = b0 & 0x1; const bool min = b0 & 0x2; const enum fsm_io io_mode = (b0 >> 2) % 3; - + const char *shifted_pattern = (const char *)&data_buf[1]; int res = fuzz_all_print_functions(dev_null, shifted_pattern, det, min, io_mode); return res; From 3a93f6d15e710b593ad9fd6515f341649bec4f3b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 17 Jul 2023 14:03:29 -0400 Subject: [PATCH 021/373] re_capvm_compile.c: Update #include for EXPENSIVE_CHECKS. --- src/libre/re_capvm_compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index c67db2669..c1d856618 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -10,7 +10,7 @@ #include "../libfsm/capture_vm_log.h" /* for EXPENSIVE_CHECKS */ -#include "../libfsm/internal.h" +#include "adt/common.h" #include #include From da51d9e237fbbfde9cd96caf2800dcc6571fb0d9 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 17 Jul 2023 14:04:31 -0400 Subject: [PATCH 022/373] Remove nullable ALT backpatching, it's now unreachable. This was set up to handle edge cases like `^(($)|x)+$` where there is an alt with some nullable cases with anchors, surrounded by + repetition, which turned up during fuzzing. This is an obscure edge case, it is proving very difficult to handle correctly, and there is probably little value in actually doing so. Now we are flagging it as an unsupported PCRE construct in ast_analysis.c's analysis_iter_repetition, so the code using repeated_alt_backpatches is unreachable. Just remove it. --- src/libre/ast.h | 1 - src/libre/ast_analysis.c | 1 - src/libre/re_capvm_compile.c | 122 ----------------------------------- 3 files changed, 124 deletions(-) diff --git a/src/libre/ast.h b/src/libre/ast.h index 20cc3cdb7..3ef0c1f5f 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -183,7 +183,6 @@ struct ast_expr { unsigned min; unsigned max; /* can be AST_COUNT_UNBOUNDED */ int contains_empty_groups; - int contains_nullable_alt; } repeat; struct { diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 3ae19841d..548d9f60c 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -392,7 +392,6 @@ analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_p assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT); assert(repeat_plus_ancestor->u.repeat.min == 1); assert(repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); - repeat_plus_ancestor->u.repeat.contains_nullable_alt = 1; /* Repetition of an alt subtree which has a capture group child that * only contains only* anchors is not handled properly yet. This diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index c1d856618..15f5bbfa5 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -88,19 +88,6 @@ struct capvm_compile_env { /* linked list */ struct repeated_group_info *prev; } *repeated_groups; - - /* Linked list of nodes used for regexes like '^(($)|x)+$', - * which need to special-case the JMP instruction after the - * nullable '($)' case to replace it with a SPLIT to before - * and after the + repetition. */ - struct repeated_alt_backpatch_info { - const struct ast_expr *repeat; /* must be a + repeat */ - size_t ceil; - size_t used; - unsigned *opcode_offsets; - /* linked list */ - struct repeated_alt_backpatch_info *prev; - } *repeated_alt_backpatches; }; static bool @@ -512,74 +499,6 @@ pop_repeated_group_info(struct capvm_compile_env *env, const struct ast_expr *ex f_free(env->alloc, rgi); } -static bool -push_repeated_alt_backpatch_info(struct capvm_compile_env *env, const struct ast_expr *expr) -{ - assert(expr - && expr->type == AST_EXPR_REPEAT - && expr->u.repeat.min == 1 - && expr->u.repeat.max == AST_COUNT_UNBOUNDED); - struct repeated_alt_backpatch_info *rabi = f_calloc(env->alloc, - 1, sizeof(*rabi)); - if (rabi == NULL) { - return false; - } - rabi->repeat = expr; - rabi->prev = env->repeated_alt_backpatches; - LOG(3 - LOG_REPETITION_CASES, - "%s: pushing node %p onto %p, prev link %p\n", - __func__, (void *)expr, (void *)rabi, (void *)rabi->prev); - env->repeated_alt_backpatches = rabi; - return true; -} - -static bool -append_repeated_alt_backpatch_offset(struct capvm_compile_env *env, unsigned offset) -{ - struct repeated_alt_backpatch_info *rabi = env->repeated_alt_backpatches; - assert(rabi != NULL); - if (rabi->used == rabi->ceil) { - const size_t nceil = (rabi->ceil == 0 - ? DEF_REPEATED_ALT_BACKPATCH_CEIL - : 2*rabi->ceil); - LOG(3 - LOG_REPETITION_CASES, - "%s: growing %zu -> %zu\n", __func__, rabi->ceil, nceil); - - unsigned *noffsets = f_realloc(env->alloc, - rabi->opcode_offsets, nceil * sizeof(noffsets[0])); - if (noffsets == NULL) { - return false; - } - rabi->ceil = nceil; - rabi->opcode_offsets = noffsets; - } - - LOG(3 - LOG_REPETITION_CASES, - "%s: pushing offset %u\n", __func__, offset); - rabi->opcode_offsets[rabi->used] = offset; - rabi->used++; - return true; -} - -static void -pop_repeated_alt_backpatch_info(struct capvm_compile_env *env, const struct ast_expr *expr) -{ - struct repeated_alt_backpatch_info *rabi = env->repeated_alt_backpatches; - assert(rabi != NULL); - assert(rabi->repeat == expr); - struct repeated_alt_backpatch_info *prev = rabi->prev; - LOG(3 - LOG_REPETITION_CASES, - "%s: popping %p, prev link %p\n", - __func__, (void *)rabi, (void *)prev); - f_free(env->alloc, rabi->opcode_offsets); - f_free(env->alloc, rabi); - env->repeated_alt_backpatches = prev; -} - -static void -backpatch_repeated_nullable_alt_split(struct capvm_compile_env *env, - const struct ast_expr *expr, struct capvm_program *p, unsigned split_new_dst); - static bool emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p); @@ -845,9 +764,6 @@ capvm_compile_iter(struct capvm_compile_env *env, op_split_after->t = CAPVM_OP_SPLIT; op_split_after->u.split.cont = PENDING_OFFSET_ALT_BACKPATCH_JMP; op_split_after->u.split.new = PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS; - if (!append_repeated_alt_backpatch_offset(env, pos_split_after)) { - return false; - } } else { const uint32_t pos_jmp_after = reserve_program_opcode(p); flow_info[c_i].backpatch = pos_jmp_after; @@ -974,12 +890,6 @@ capvm_compile_iter(struct capvm_compile_env *env, return false; } } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* + */ - if (expr->u.repeat.contains_nullable_alt) { - if (!push_repeated_alt_backpatch_info(env, expr)) { - return false; - } - } - /* l1: * split l1, l2 * l2: */ @@ -1007,16 +917,6 @@ capvm_compile_iter(struct capvm_compile_env *env, op_split->t = CAPVM_OP_SPLIT; op_split->u.split.cont = pos_l1; op_split->u.split.new = pos_l2; - - /* Update any ALT nodes in the subtree whose SPLIT instructions - * are awaiting backpatching with pos_l2. */ - if (expr->u.repeat.contains_nullable_alt) { - backpatch_repeated_nullable_alt_split(env, expr, p, pos_l2); - } - } - - if (expr->u.repeat.contains_nullable_alt) { - pop_repeated_alt_backpatch_info(env, expr); } } else if (min == 0 && max == 0) { /* {0,0} */ /* ignored, except any groups contained within that could match @@ -1384,28 +1284,6 @@ emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p) return true; } -static void -backpatch_repeated_nullable_alt_split(struct capvm_compile_env *env, - const struct ast_expr *expr, struct capvm_program *p, unsigned split_new_dst) -{ - struct repeated_alt_backpatch_info *rabi = env->repeated_alt_backpatches; - assert(rabi != NULL && rabi->repeat == expr); - - for (size_t op_i = 0; op_i < rabi->used; op_i++) { - const unsigned offset = rabi->opcode_offsets[op_i]; - assert(offset < p->used); - LOG(3 - LOG_REPETITION_CASES, - "%s: backpatching SPLIT instruction %u's .new to %u\n", - __func__, offset, split_new_dst); - struct capvm_opcode *op = &p->ops[offset]; - assert(op->t == CAPVM_OP_SPLIT); - assert(op->u.split.new == PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS); - op->u.split.new = split_new_dst; - } - - rabi->used = 0; -} - static bool capvm_compile_iter_save_groups_in_skipped_subtree(struct capvm_compile_env *env, struct capvm_program *p, const struct ast_expr *expr) From 9f3ae29c2c7433218920560a3da6193b47114f6e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 17 Jul 2023 16:40:21 -0400 Subject: [PATCH 023/373] capture vm: Cleanup & clarifying comments throughout. --- src/libfsm/capture_vm.c | 17 ++++++------- src/libfsm/capture_vm.h | 3 +-- src/libfsm/capture_vm_exec.c | 48 ++++++++++++++++++++++-------------- src/libre/re_capvm_compile.c | 12 ++++++--- 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/libfsm/capture_vm.c b/src/libfsm/capture_vm.c index aacae7f9c..7dd1599cd 100644 --- a/src/libfsm/capture_vm.c +++ b/src/libfsm/capture_vm.c @@ -13,9 +13,8 @@ * with PCRE's behavior for libfsm's supported subset of PCRE. * * Instead of giving each green thread its own copy of the capture - * buffers, which uses a prohibitive amount of memory when matching DFAs - * that combine several regexes with several captures each, operate in - * two passes. + * buffers, which uses a prohibitive amount of memory when combining DFAs + * with several captures each, operate in two passes: * * In the first pass, each thread keeps track of its execution path, * appending a bit for each branch: 1 for the greedy option, 0 for the @@ -32,13 +31,13 @@ * a greedier search path (since PCRE's results match the greediest). * * In the second pass, replay the execution path for just the single - * greediest thread, which represents the "best" match, and write - * capture offsets into buffers passed in by the caller. + * greediest thread, which represents the "correct" match (according to + * PCRE semantics), and write capture offsets into buffers passed in by + * the caller. * - * Most of the other differences have to do with matching PCRE's quirky - * behaviors, particularly interactions between newlines and start/end - * anchors. - * */ + * Most of the other differences have to do with matching PCRE + * edge cases, particularly interactions between newlines and start/end + * anchors. */ #include "capture_vm.h" #include "capture_vm_program.h" diff --git a/src/libfsm/capture_vm.h b/src/libfsm/capture_vm.h index 74e958eb8..34d4c73df 100644 --- a/src/libfsm/capture_vm.h +++ b/src/libfsm/capture_vm.h @@ -13,7 +13,7 @@ #include #include -/* Interface the virtual machine used to resolve captures. +/* Interface for the virtual machine used to resolve captures. * These interfaces are exposed to libre but should not be * used directly. */ @@ -43,7 +43,6 @@ enum fsm_capvm_program_exec_res { FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND, FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC = -1, - /* FIXME: what other ways can this fail? */ }; /* Execute a capture program with the given input and populate diff --git a/src/libfsm/capture_vm_exec.c b/src/libfsm/capture_vm_exec.c index f2d0b522e..579a67fa4 100644 --- a/src/libfsm/capture_vm_exec.c +++ b/src/libfsm/capture_vm_exec.c @@ -36,7 +36,7 @@ * This value cannot be changed without reworking the data structures. */ #define PATH_LINK_BITS 32 -/* This enables extra debugging/testing output */ +/* This enables extra debugging/testing output in an easily scraped format */ #ifndef TESTING_OPTIONS #define TESTING_OPTIONS 0 #endif @@ -49,10 +49,12 @@ #define CAPVM_STATS (0 || TESTING_OPTIONS) #define CAPVM_PATH_STATS (0 && CAPVM_STATS) -/* This may no longer be necessary after further work on path handling. */ -#define ALLOW_TABLE_RESIZING 1 -#define ALLOW_PATH_TABLE_RESIZING (1 || ALLOW_TABLE_RESIZING) -#define ALLOW_THREAD_TABLE_RESIZING (0 || ALLOW_TABLE_RESIZING) +/* Allow the path table to grow on demand. + * In theory it should be possible to determine the worst case + * based on compile-time analysis and the input length; if an + * appropriately sized buffer was passed in capture resolution + * would not need dynamic allocation at all. */ +#define ALLOW_PATH_TABLE_RESIZING 1 /* Set to non-zero to trap runaway path table growth */ #define PATH_TABLE_CEIL_LIMIT 0 @@ -135,7 +137,8 @@ struct capvm { * be advanced next. The current stack is * run_stacks[PAIR_ID_CURRENT], run_stacks[PAIR_ID_NEXT] is the * stack for the next input position, and when the current stack - * is completed the next stack is copied over (and reversed). + * is completed the next stack is copied over (and reversed, so + * the greediest threads end up on top and resume first). * Same with run_stacks_h, the height for each stack, and the * other fields with [2] below. */ uint32_t *run_stacks[2]; @@ -143,9 +146,12 @@ struct capvm { /* Similarly, two columns of bits and two arrays of path_info * node IDs and uniq_ids for the execution at a particular - * opcode. */ + * opcode. + * + * evaluated bit array[]: Has the instruction n already been + * evaluated at the current input position? */ uint32_t *evaluated[2]; - uint32_t *path_info_heads[2]; + uint32_t *path_info_heads[2]; /* path for thread on instruction */ #if CAPVM_STATS uint32_t *uniq_ids[2]; #endif @@ -164,14 +170,19 @@ struct capvm { struct capvm_path_info { union { struct capvm_path_freelist_link { - uint16_t refcount; /* == 0 */ + uint16_t refcount; /* == 0: tag for freelist node */ uint32_t freelist; } freelist_node; struct capvm_path_info_link { - uint16_t refcount; /* > 0, sticky at UINT16_MAX? */ + /* refcount: When > 0 this is a path node. + * This could be sticky at UINT16_MAX, but in order + * to get there it would need a regex whose compiled + * program has well over 2**16 instructions that all + * share the same path info node. */ + uint16_t refcount; uint8_t used; /* .bits used, <= PATH_LINK_BITS */ - uint32_t bits; - uint32_t offset; + uint32_t bits; /* buffer for this link's path bits */ + uint32_t offset; /* offset into the path bit array */ /* Linked list to earlier path nodes, with common * nodes shared until paths diverge. * @@ -667,8 +678,8 @@ cmp_paths(struct capvm *vm, uint32_t p_a, uint32_t p_b) "%s: backward loop: link_a %d (offset %u, prev %d), link_b %d (offset %u, prev %d)\n", __func__, link_a, offset_a, prev_a, link_b, offset_b, prev_b); - assert((offset_a & 31) == 0); /* multiple of 32 */ - assert((offset_b & 31) == 0); /* multiple of 32 */ + assert((offset_a & (PATH_LINK_BITS - 1)) == 0); /* multiple of 32 */ + assert((offset_b & (PATH_LINK_BITS - 1)) == 0); /* multiple of 32 */ if (offset_a > offset_b) { LOG(3 - LOG_CMP_PATHS, "%s: backward loop: a longer than b\n", __func__); set_path_node_backlink(vm, link_a, fwd_a); @@ -894,7 +905,7 @@ schedule_possible_next_step(struct capvm *vm, enum pair_id pair_id, const size_t h = *stack_h; for (size_t i = 0; i < h; i++) { if (stack[i] == op_id) { - stack[i] = NO_ID; + stack[i] = NO_ID; /* cancel thread */ vm->threads.live--; } } @@ -1093,8 +1104,8 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, case CAPVM_OP_ANCHOR: if (op->u.anchor == CAPVM_ANCHOR_START) { LOG(3, "%s: ^ anchor\n", __func__); - /* ignore a trailing newline, because PCRE does, - * even after a $ anchor. */ + /* ignore a single trailing newline, because PCRE does. + * For ^ this affects the capture position. */ if (input_pos == 0 && vm->input_len == 1 && vm->input[0] == '\n') { @@ -1113,7 +1124,7 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, LOG(3, "%s: $ anchor: input_len %u, input_pos %u\n", __func__, vm->input_len, input_pos); - /* ignore a trailing newline, because PCRE does */ + /* ignore a single trailing newline, because PCRE does */ if (vm->input_len > 0 && input_pos == vm->input_len - 1) { if (vm->input[input_pos] != '\n') { goto halt_thread; @@ -1140,7 +1151,6 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, check_path_table(vm); } - /* FIXME: Check the cleanup logic here. */ return; halt_thread: diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index 15f5bbfa5..6b810b246 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -605,9 +605,11 @@ capvm_compile_iter(struct capvm_compile_env *env, pop_repeated_group_info(env, expr); } - /* FIXME: may need distinct error case to not leak. - * There is currently no test reaching this, try - * using the fuzzer to trigger it. */ + /* FIXME: May need distinct error case to not + * leak. There is currently no test reaching + * this and the fuzzer has not produced an input + * that reaches it -- unsatisfiability has probably + * already pruned subtrees that would get here. */ return true; } else if (active_count == 1) { /* even if one of the later subtrees is active, an earlier @@ -938,7 +940,9 @@ capvm_compile_iter(struct capvm_compile_env *env, if (max == AST_COUNT_UNBOUNDED) { /* A repeat of {x,inf} should be treated like - * (?:subtree){x} (?:subtree)* . */ + * (?:subtree){x} (?:subtree)* , where any numbered + * capture groups inside have the same group ID in + * both copies of the subtree. */ if (!compile_kleene_star(env, p, expr)) { return false; } From 26483cf76567488d55f4f63a66443f53cdd14aa6 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 18 Jul 2023 14:21:35 -0400 Subject: [PATCH 024/373] re_capvm_compile: Change "active_node" categorization. Strictly speaking this shouldn't include nodes that have been flagged with `AST_FLAG_UNSATISFIABLE`. I have re-fuzzed with this changed and it does not seem to have introduced any new issues. `active_node` is only used in one place. --- src/libre/re_capvm_compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index 6b810b246..248cd0e29 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -334,7 +334,7 @@ active_node(const struct ast_expr *n) case AST_EXPR_TOMBSTONE: return false; default: - return true; + return !(n->flags & AST_FLAG_UNSATISFIABLE); } } From e9353afd02c0370231d7ac9499e84aa5fe925508 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 21 Jul 2023 11:29:59 -0400 Subject: [PATCH 025/373] capture_vm: rename split's .cont and .new to .greedy and .nongreedy. Rather than commenting throughout to note that .cont is the greedy branch and .new is non-greedy, just rename them. --- src/libfsm/capture_vm.c | 2 +- src/libfsm/capture_vm_exec.c | 48 ++++++++++---------- src/libfsm/capture_vm_program.h | 4 +- src/libre/re_capvm_compile.c | 62 +++++++++++++------------- tests/capture/capture_test_case_list.c | 14 +++--- 5 files changed, 64 insertions(+), 66 deletions(-) diff --git a/src/libfsm/capture_vm.c b/src/libfsm/capture_vm.c index 7dd1599cd..bc5073df9 100644 --- a/src/libfsm/capture_vm.c +++ b/src/libfsm/capture_vm.c @@ -153,7 +153,7 @@ fsm_capvm_program_dump(FILE *f, fprintf(f, "%zu: jmp_once %u\n", i, op->u.jmp_once); break; case CAPVM_OP_SPLIT: - fprintf(f, "%zu: split cont %u new %u\n", i, op->u.split.cont, op->u.split.new); + fprintf(f, "%zu: split greedy %u nongreedy %u\n", i, op->u.split.greedy, op->u.split.nongreedy); break; case CAPVM_OP_SAVE: fprintf(f, "%zu: save %u (cap %u, %s)\n", diff --git a/src/libfsm/capture_vm_exec.c b/src/libfsm/capture_vm_exec.c index 579a67fa4..5fc477ca8 100644 --- a/src/libfsm/capture_vm_exec.c +++ b/src/libfsm/capture_vm_exec.c @@ -1045,36 +1045,35 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, case CAPVM_OP_SPLIT: { - const uint32_t dst_cont = op->u.split.cont; - const uint32_t dst_new = op->u.split.new; + const uint32_t dst_greedy = op->u.split.greedy; + const uint32_t dst_nongreedy = op->u.split.nongreedy; /* destinations must be in range and not self-referential */ - assert(dst_cont < vm->p->used); - assert(dst_new < vm->p->used); - assert(dst_cont != op_id); - assert(dst_new != op_id); + assert(dst_greedy < vm->p->used); + assert(dst_nongreedy < vm->p->used); + assert(dst_greedy != op_id); + assert(dst_nongreedy != op_id); - uint32_t new_path_info_head; - if (!copy_path_info(vm, path_info_head, &new_path_info_head)) { + uint32_t nongreedy_path_info_head; + if (!copy_path_info(vm, path_info_head, &nongreedy_path_info_head)) { goto alloc_error; } - /* cont is the greedy branch */ if (!extend_path_info(vm, path_info_head, 1, uniq_id, &path_info_head)) { release_path_info_link(vm, &path_info_head); goto alloc_error; } - /* new is the non-greedy branch */ - if (!extend_path_info(vm, new_path_info_head, 0, uniq_id, &new_path_info_head)) { + /* nongreedy is the non-greedy branch */ + if (!extend_path_info(vm, nongreedy_path_info_head, 0, uniq_id, &nongreedy_path_info_head)) { release_path_info_link(vm, &path_info_head); goto alloc_error; } #if CAPVM_STATS - const uint32_t new_uniq_id = ++vm->uniq_id_counter; + const uint32_t nongreedy_uniq_id = ++vm->uniq_id_counter; #else - const uint32_t new_uniq_id = 0; + const uint32_t nongreedy_uniq_id = 0; #endif vm->threads.live++; @@ -1082,15 +1081,14 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, set_max_threads_live(vm, vm->threads.live); } - /* Push the split.new destination, and then the - * split.cont destination on top of it, so that the - * greedier .cont branch will be fully evaluated - * first. */ - schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_new, - new_path_info_head, new_uniq_id); - schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_cont, + /* Push the split.nongreedy destination, and then the + * split.greedy destination on top of it, so that the + * greedier branch will be fully evaluated first. */ + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_nongreedy, + nongreedy_path_info_head, nongreedy_uniq_id); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_greedy, path_info_head, uniq_id); - LOG_EXEC_SPLIT(uniq_id, new_uniq_id); + LOG_EXEC_SPLIT(uniq_id, nongreedy_uniq_id); break; } @@ -1792,8 +1790,8 @@ populate_solution(struct capvm *vm) const uint32_t offset = get_path_node_offset(vm, cur); const struct capvm_path_info *pi = &vm->paths.pool[cur]; - const uint32_t dst_cont = op->u.split.cont; - const uint32_t dst_new = op->u.split.new; + const uint32_t dst_greedy = op->u.split.greedy; + const uint32_t dst_nongreedy = op->u.split.nongreedy; assert(IS_PATH_NODE(pi)); bool next_bit; @@ -1811,9 +1809,9 @@ populate_solution(struct capvm *vm) LOG(3, "split: next_bit %d\n", next_bit); LOG_EXEC_PATH_SAVE_CAPTURES(uniq_id, next_bit); if (next_bit) { /* greedy edge */ - prog_i = dst_cont; + prog_i = dst_greedy; } else { /* non-greedy edge */ - prog_i = dst_new; + prog_i = dst_nongreedy; } split_i++; if (split_i >= offset && diff --git a/src/libfsm/capture_vm_program.h b/src/libfsm/capture_vm_program.h index 15e585761..0b24ffb5b 100644 --- a/src/libfsm/capture_vm_program.h +++ b/src/libfsm/capture_vm_program.h @@ -45,8 +45,8 @@ struct capvm_program { uint32_t jmp; /* absolute */ uint32_t jmp_once; /* absolute */ struct { - uint32_t cont; /* greedy branch */ - uint32_t new; /* non-greedy branch */ + uint32_t greedy; + uint32_t nongreedy; } split; /* (save >> 1): capture ID, * (save & 0x01): save pos to start (0b0) or end (0b1). */ diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index 248cd0e29..8a4c25562 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -144,10 +144,10 @@ check_program_for_invalid_labels(const struct capvm_program *p) assert(op->u.jmp_once != op_i); break; case CAPVM_OP_SPLIT: - assert(op->u.split.cont < p->used); - assert(op->u.split.cont != op_i); - assert(op->u.split.new < p->used); - assert(op->u.split.new != op_i); + assert(op->u.split.greedy < p->used); + assert(op->u.split.greedy != op_i); + assert(op->u.split.nongreedy < p->used); + assert(op->u.split.nongreedy != op_i); break; case CAPVM_OP_CHAR: @@ -724,11 +724,11 @@ capvm_compile_iter(struct capvm_compile_env *env, op_split_before->t = CAPVM_OP_SPLIT; /* greedier branch: trying the next case, in order */ - op_split_before->u.split.cont = get_program_offset(p); + op_split_before->u.split.greedy = get_program_offset(p); /* less greedy branch: moving on to the next case. * will backpatch .new to after this case's JMP later */ - op_split_before->u.split.new = PENDING_OFFSET_ALT_BACKPATCH_NEW; + op_split_before->u.split.nongreedy = PENDING_OFFSET_ALT_BACKPATCH_NEW; const struct ast_expr *n = expr->u.alt.n[c_i]; LOG(3, "%s: %p recursing...\n", __func__, (void *)expr); @@ -764,8 +764,8 @@ capvm_compile_iter(struct capvm_compile_env *env, flow_info[c_i].backpatch = pos_split_after; struct capvm_opcode *op_split_after = &p->ops[pos_split_after]; op_split_after->t = CAPVM_OP_SPLIT; - op_split_after->u.split.cont = PENDING_OFFSET_ALT_BACKPATCH_JMP; - op_split_after->u.split.new = PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS; + op_split_after->u.split.greedy = PENDING_OFFSET_ALT_BACKPATCH_JMP; + op_split_after->u.split.nongreedy = PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS; } else { const uint32_t pos_jmp_after = reserve_program_opcode(p); flow_info[c_i].backpatch = pos_jmp_after; @@ -779,7 +779,7 @@ capvm_compile_iter(struct capvm_compile_env *env, /* and the original split jumps to after * this case's JMP */ - op_split_before->u.split.new = get_program_offset(p); + op_split_before->u.split.nongreedy = get_program_offset(p); } } @@ -802,10 +802,10 @@ capvm_compile_iter(struct capvm_compile_env *env, assert(op_patch->u.jmp == PENDING_OFFSET_ALT_BACKPATCH_JMP); op_patch->u.jmp = pos_after_all; } else if (op_patch->t == CAPVM_OP_SPLIT) { - assert(op_patch->u.split.cont == PENDING_OFFSET_ALT_BACKPATCH_JMP); - op_patch->u.split.cont = pos_after_all; + assert(op_patch->u.split.greedy == PENDING_OFFSET_ALT_BACKPATCH_JMP); + op_patch->u.split.greedy = pos_after_all; /* This will be patched by an ancestor repeat node after returning. */ - assert(op_patch->u.split.cont == PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS); + assert(op_patch->u.split.greedy == PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS); } else { assert(!"type mismatch"); } @@ -875,8 +875,8 @@ capvm_compile_iter(struct capvm_compile_env *env, struct capvm_opcode *op_split = &p->ops[pos_split]; op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = pos_l1; - op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; if (!capvm_compile_iter(env, p, e)) { return false; } @@ -886,7 +886,7 @@ capvm_compile_iter(struct capvm_compile_env *env, op_split = &p->ops[pos_split]; /* refresh pointer */ const uint32_t after_expr = get_program_offset(p); - op_split->u.split.new = after_expr; + op_split->u.split.nongreedy = after_expr; } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* * */ if (!compile_kleene_star(env, p, expr)) { return false; @@ -917,8 +917,8 @@ capvm_compile_iter(struct capvm_compile_env *env, struct capvm_opcode *op_split = &p->ops[pos_split]; op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = pos_l1; - op_split->u.split.new = pos_l2; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = pos_l2; } } else if (min == 0 && max == 0) { /* {0,0} */ /* ignored, except any groups contained within that could match @@ -963,8 +963,8 @@ capvm_compile_iter(struct capvm_compile_env *env, struct capvm_opcode *op_split = &p->ops[pos_split]; op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = pos_l1; - op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; if (!capvm_compile_iter(env, p, e)) { return false; } @@ -974,7 +974,7 @@ capvm_compile_iter(struct capvm_compile_env *env, op_split = &p->ops[pos_split]; /* refresh pointer */ const uint32_t after_expr = get_program_offset(p); - op_split->u.split.new = after_expr; + op_split->u.split.nongreedy = after_expr; } } } @@ -1171,8 +1171,8 @@ compile_kleene_star(struct capvm_compile_env *env, struct capvm_opcode *op_split = &p->ops[pos_l1]; op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = PENDING_OFFSET_REPEAT_OPTIONAL_CONT; - op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + op_split->u.split.greedy = PENDING_OFFSET_REPEAT_OPTIONAL_CONT; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; if (!capvm_compile_iter(env, p, expr->u.repeat.e)) { return false; } @@ -1197,8 +1197,8 @@ compile_kleene_star(struct capvm_compile_env *env, const uint32_t pos_l3 = get_program_offset(p); op_split = &p->ops[pos_l1]; /* refresh pointer */ - op_split->u.split.cont = pos_l2; - op_split->u.split.new = pos_l3; + op_split->u.split.greedy = pos_l2; + op_split->u.split.nongreedy = pos_l3; return true; } @@ -1233,8 +1233,8 @@ emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p) struct capvm_opcode *op_split = &p->ops[pos_split]; op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = pos_l1; - op_split->u.split.new = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; if (group->flags & AST_FLAG_ANCHORED_START) { const uint32_t pos_start = reserve_program_opcode(p); @@ -1263,7 +1263,7 @@ emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p) const uint32_t after_expr = get_program_offset(p); op_split = &p->ops[pos_split]; /* refresh pointer */ - op_split->u.split.new = after_expr; + op_split->u.split.nongreedy = after_expr; } else { /* simple case, emit SAVE pair */ if (!ensure_program_capacity(env->alloc, p, 2)) { @@ -1404,8 +1404,8 @@ capvm_compile(struct capvm_compile_env *env, const uint32_t l3 = get_program_offset(p); op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = l3; /* greedy */ - op_split->u.split.new = l2; /* non-greedy */ + op_split->u.split.greedy = l3; + op_split->u.split.nongreedy = l2; op_cc->t = CAPVM_OP_CHARCLASS; uint64_t any[4]; @@ -1447,8 +1447,8 @@ capvm_compile(struct capvm_compile_env *env, struct capvm_opcode *op_jmp = &p->ops[l_jmp]; op_split->t = CAPVM_OP_SPLIT; - op_split->u.split.cont = l3; /* greedy */ - op_split->u.split.new = l2; /* non-greedy */ + op_split->u.split.greedy = l3; + op_split->u.split.nongreedy = l2; op_any->t = CAPVM_OP_CHARCLASS; uint64_t any[4]; diff --git a/tests/capture/capture_test_case_list.c b/tests/capture/capture_test_case_list.c index d05210d2c..6b72d1018 100644 --- a/tests/capture/capture_test_case_list.c +++ b/tests/capture/capture_test_case_list.c @@ -1615,25 +1615,25 @@ static struct captest_case_program program_cases[] = { }, .ops = { - { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 3, .new = 1 }}, + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 3, .nongreedy = 1 }}, { .t = CAPVM_OP_CHARCLASS, .u.charclass_id = 0 }, { .t = CAPVM_OP_JMP, .u.jmp = 0 }, { .t = CAPVM_OP_SAVE, .u.save = 0 }, - { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 5, .new = 7 }}, + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 5, .nongreedy = 7 }}, { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_START }, { .t = CAPVM_OP_JMP, .u.jmp = 9 }, /* jump after |() */ { .t = CAPVM_OP_SAVE, .u.save = 4 }, { .t = CAPVM_OP_SAVE, .u.save = 5 }, - { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 4, .new = 10 }}, + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 4, .nongreedy = 10 }}, { .t = CAPVM_OP_SAVE, .u.save = 2 }, { .t = CAPVM_OP_SAVE, .u.save = 3 }, { .t = CAPVM_OP_SAVE, .u.save = 6 }, { .t = CAPVM_OP_SAVE, .u.save = 7 }, { .t = CAPVM_OP_SAVE, .u.save = 1 }, - { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 18, .new = 16 }}, + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 18, .nongreedy = 16 }}, { .t = CAPVM_OP_CHARCLASS, .u.charclass_id = 0 }, { .t = CAPVM_OP_JMP, .u.jmp = 15 }, { .t = CAPVM_OP_MATCH }, @@ -1656,15 +1656,15 @@ static struct captest_case_program program_cases[] = { [0] = { .t = CAPVM_OP_SAVE, .u.save = 0 }, [1] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_START }, [2] = { .t = CAPVM_OP_SAVE, .u.save = 2 }, - [3] = { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 4, .new = 6 }}, + [3] = { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 4, .nongreedy = 6 }}, [4] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_END }, /* [5] = { .t = CAPVM_OP_JMP, .u.jmp = 7 }, */ - [5] = { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 7, .new = 9 }}, + [5] = { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 7, .nongreedy = 9 }}, [6] = { .t = CAPVM_OP_CHAR, .u.chr = 'x' }, [7] = { .t = CAPVM_OP_SAVE, .u.save = 3 }, - [8] = { .t = CAPVM_OP_SPLIT, .u.split = { .cont = 2, .new = 9 }}, + [8] = { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 2, .nongreedy = 9 }}, [9] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_END }, [10] = { .t = CAPVM_OP_SAVE, .u.save = 1 }, [11] = { .t = CAPVM_OP_MATCH }, From 1e552204934797be29c35b8d00bcc2c7b9542e18 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 21 Jul 2023 12:02:00 -0400 Subject: [PATCH 026/373] capture_vm: Add error state for step limit reached. --- src/libfsm/capture_vm.h | 2 +- src/libfsm/capture_vm_exec.c | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/libfsm/capture_vm.h b/src/libfsm/capture_vm.h index 34d4c73df..02c198dab 100644 --- a/src/libfsm/capture_vm.h +++ b/src/libfsm/capture_vm.h @@ -41,7 +41,7 @@ fsm_capvm_program_dump(FILE *f, enum fsm_capvm_program_exec_res { FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN, FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND, - + FSM_CAPVM_PROGRAM_EXEC_STEP_LIMIT_REACHED, FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC = -1, }; diff --git a/src/libfsm/capture_vm_exec.c b/src/libfsm/capture_vm_exec.c index 5fc477ca8..9d4be066a 100644 --- a/src/libfsm/capture_vm_exec.c +++ b/src/libfsm/capture_vm_exec.c @@ -926,33 +926,29 @@ schedule_possible_next_step(struct capvm *vm, enum pair_id pair_id, } } -static void +/* returns whether the vm should continue. */ +static bool eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, uint32_t path_info_head, uint32_t uniq_id, uint32_t op_id) { LOG(5, "%s: input_pos %u, input_len %u, op_id %u, threads_live %u\n", __func__, input_pos, vm->input_len, op_id, vm->threads.live); + assert(op_id < vm->p->used); + if (vm->stats.steps == vm->step_limit) { LOG(1, "%s: halting, steps == step_limit %zu\n", __func__, vm->step_limit); - return; + vm->res = FSM_CAPVM_PROGRAM_EXEC_STEP_LIMIT_REACHED; + return false; } - - assert(op_id < vm->p->used); + vm->stats.steps++; const struct capvm_opcode *op = &vm->p->ops[op_id]; - LOG(2, "%s: op_id[%u]: input_pos %u, path_info_head %u, uniq_id %u, op %s\n", __func__, op_id, input_pos, path_info_head, uniq_id, op_name[op->t]); LOG_EXEC_OP(uniq_id, input_pos, op_id, op_name[op->t]); - vm->stats.steps++; - if (vm->stats.steps == vm->step_limit) { - /* TODO: Set some sort of STEP_LIMIT_REACHED error. */ - return; - } - switch (op->t) { case CAPVM_OP_CHAR: if (input_pos == vm->input_len) { @@ -1142,14 +1138,14 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, default: assert(!"unreachable"); - return; + return false; } if (EXPENSIVE_CHECKS) { /* postcondition */ check_path_table(vm); } - return; + return true; halt_thread: /* do not push further execution on the run stack */ @@ -1158,11 +1154,12 @@ eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, release_path_info_link(vm, &path_info_head); assert(vm->threads.live > 0); vm->threads.live--; - return; + return true; alloc_error: release_path_info_link(vm, &path_info_head); vm->res = FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC; + return false; } static void @@ -1351,7 +1348,9 @@ eval_vm(struct capvm *vm) #else const uint32_t uniq_id = 0; #endif - eval_vm_advance_greediest(vm, i_i, path_info_head, uniq_id, op_id); + if (!eval_vm_advance_greediest(vm, i_i, path_info_head, uniq_id, op_id)) { + return false; + } } @@ -2054,7 +2053,7 @@ fsm_capvm_program_exec(const struct capvm_program *program, * flag, which would skip this assertion. */ assert(vm.paths.live == 0); } else { - assert(vm.res == FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND); + assert(vm.res != FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN); } TIME(&post); From 937c9f0bce0378d47fcc6eb655fd928f8d40b7ef Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 24 Jul 2023 17:14:15 -0400 Subject: [PATCH 027/373] ast_analysis: Remove variables that are no longer used. --- src/libre/ast_analysis.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 548d9f60c..df9e8ce54 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1048,8 +1048,6 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) /* flow ANCHORED_START and ANCHORED_END flags upward */ { - int after_always_consumes = 0; - for (i = 0; i < n->u.concat.count; i++) { struct ast_expr *child = n->u.concat.n[i]; if (child->flags & AST_FLAG_ANCHORED_START) { @@ -1062,7 +1060,6 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (always_consumes_input(child)) { LOG(3 - LOG_ANCHORING, "%s: child %zd always consumes input\n", __func__, i); - after_always_consumes = 1; } } } @@ -1525,7 +1522,6 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_ALT: { int any_sat = 0; int all_set_followed_by_consuming = 1; - int all_set_followed_by_consuming_newline = 1; int any_set_followed_by_consuming_newline = 0; int all_set_before_start_anchor = 1; @@ -1554,7 +1550,6 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) all_set_before_start_anchor &= child_env.before_start_anchor; any_set_followed_by_consuming_newline |= child_env.followed_by_consuming_newline; - all_set_followed_by_consuming_newline &= child_env.followed_by_consuming_newline; any_sat = 1; } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE From 5e54a8bb0af2a75826edecb25b6930798c94c261 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 24 Jul 2023 17:20:07 -0400 Subject: [PATCH 028/373] src/adt/stateset.c: Reference `adt/common.h`'s `EXPENSIVE_CHECKS`. --- src/adt/stateset.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index ab3542161..65bbf173a 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -15,10 +15,9 @@ #include #include -/* FIXME: This should use the same define as currently exists in - * src/fsm/internal.h. This is used here because the calls to +/* This is used here because the calls to * state_set_contains change the order of growth. */ -#define EXPENSIVE_CHECKS 0 +#include /* From 94d6e7f97b182202179dbb242ccb0f935319bff8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 25 Jul 2023 09:55:39 -0400 Subject: [PATCH 029/373] fuzz/target.c: Use default mode for the empty string. --- fuzz/target.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzz/target.c b/fuzz/target.c index e7a981b02..736c2d889 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -1615,6 +1615,7 @@ get_run_mode(void) } switch (mode[0]) { + case '\0': return MODE_REGEX; /* default */ case 'r': return MODE_REGEX; case 's': return MODE_REGEX_SINGLE_ONLY; case 'm': return MODE_REGEX_MULTI_ONLY; From dced7c77ef3cc83d58d6743792052c79b7536fce Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 25 Jul 2023 09:55:52 -0400 Subject: [PATCH 030/373] re_capvm_compile.c: Avoid potential sign-extension bug. `expr->u.literal.c` is signed, so when negative casting it to `uint64_t` can lead to a 64-bit value > 255. Clamp it to 8 bits. --- src/libre/re_capvm_compile.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index 8a4c25562..06c48fc28 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -362,7 +362,7 @@ subtree_represents_character_class(const struct ast_expr *expr, uint64_t cc[4]) || t->type != AST_ENDPOINT_LITERAL) { return false; } - for (uint64_t c = (uint64_t)f->u.literal.c; c <= (uint64_t)t->u.literal.c; c++) { + for (uint64_t c = (uint8_t)f->u.literal.c; c <= (uint8_t)t->u.literal.c; c++) { u64bitset_set(cc, (uint8_t)c); } return true; @@ -404,7 +404,7 @@ subtree_represents_character_class(const struct ast_expr *expr, uint64_t cc[4]) } for (uint64_t i = 0; i < 256; i++) { - if (i >= (uint64_t)f->u.literal.c && i <= (uint64_t)f->u.literal.c) { + if (i >= (uint8_t)f->u.literal.c && i <= (uint8_t)f->u.literal.c) { u64bitset_set(cc, i); } } @@ -827,7 +827,7 @@ capvm_compile_iter(struct capvm_compile_env *env, if (expr->re_flags & RE_ICASE) { uint64_t cc[4] = { 0 }; - u64bitset_set(cc, (uint64_t)expr->u.literal.c); + u64bitset_set(cc, (uint8_t)expr->u.literal.c); op->t = CAPVM_OP_CHARCLASS; make_charclass_case_insensitive(cc); From 8f0a683758ef5ebeca2f4fc56c9ec697a12e207e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 25 Jul 2023 10:19:26 -0400 Subject: [PATCH 031/373] Do non-zero allocations to silence EFENCE. EFENCE breaks CI builds that (correctly) do a zero-size allocation when calling `calloc` with a count of zero, reporting that it's a likely bug. Use a size of 1 instead of 0 to silence it. --- src/libfsm/capture_vm.c | 4 +++- src/libre/ast_compile.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libfsm/capture_vm.c b/src/libfsm/capture_vm.c index bc5073df9..e6a1f0539 100644 --- a/src/libfsm/capture_vm.c +++ b/src/libfsm/capture_vm.c @@ -77,7 +77,9 @@ capvm_program_copy(const struct fsm_alloc *alloc, ops = f_calloc(alloc, src->used, sizeof(ops[0])); if (ops == NULL) { goto cleanup; } - sets = f_calloc(alloc, src->char_classes.count, + sets = f_calloc(alloc, + /* do non-zero allocation to silence EFENCE */ + src->char_classes.count == 0 ? 1 : src->char_classes.count, sizeof(src->char_classes.sets[0])); if (sets == NULL) { goto cleanup; } diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index d13f742aa..b376aa144 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -765,7 +765,7 @@ comp_iter(struct comp_env *env, { const size_t capture_id_words = (env->max_capture_id == AST_NO_MAX_CAPTURE_ID) - ? 0 + ? 1 /* do non-zero allocation to silence EFENCE */ : ((env->max_capture_id)/64 + 1); active_capture_ids = f_calloc(env->alloc, capture_id_words, sizeof(active_capture_ids[0])); From 722260ad34d28d13e692879e71b1f93f1b580a94 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 25 Jul 2023 13:02:58 -0400 Subject: [PATCH 032/373] re_capvm_compile: subtree_represents_character_class: handle empty b. The comment says this handles a .b that is EMPTY, but the recursive call rejects it. Something in PCRE suite in CI is triggering this. --- src/libre/re_capvm_compile.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index 06c48fc28..068c00d32 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -414,7 +414,11 @@ subtree_represents_character_class(const struct ast_expr *expr, uint64_t cc[4]) } uint64_t neg_cc[4]; - if (subtree_represents_character_class(expr->u.subtract.b, neg_cc)) { + if (expr->u.subtract.b->type == AST_EXPR_EMPTY) { + for (size_t cc_i = 0; cc_i < 4; cc_i++) { + neg_cc[cc_i] = (uint64_t)0; + } + } else if (subtree_represents_character_class(expr->u.subtract.b, neg_cc)) { for (size_t cc_i = 0; cc_i < 4; cc_i++) { cc[cc_i] &=~ neg_cc[cc_i]; } From 75bb88d95bc25117d79ff7cc349bce5aa94b9999 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 16:52:35 -0400 Subject: [PATCH 033/373] state_set: Improve `state_set_search` performance, correct result. The description says "Return where an item would be, if it were inserted", but it was returning the last element <= rather than the first element >=, then the call to `state_set_cmpval` later was shifting i by 1 for that specific case. Handle it correctly inside the search function instead. Two other all call sites need to check whether the result refers to the append position (one past the end of the array) before checking `set->a[i] == state`, update them. Add a fast path upfront: It's VERY common to append states in order to the state array, so before we binary search each first compare against the last entry (unless empty). --- src/adt/stateset.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index c1cff9933..4d4de9438 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -138,7 +138,8 @@ state_set_cmp(const struct state_set *a, const struct state_set *b) } /* - * Return where an item would be, if it were inserted + * Return where an item would be, if it were inserted. + * When insertion would append this returns one past the array. */ static size_t state_set_search(const struct state_set *set, fsm_state_t state) @@ -150,6 +151,11 @@ state_set_search(const struct state_set *set, fsm_state_t state) assert(!IS_SINGLETON(set)); assert(set->a != NULL); + /* fast path: append case */ + if (set->i > 0 && state > set->a[set->i - 1]) { + return set->i; + } + start = mid = 0; end = set->i; @@ -161,6 +167,12 @@ state_set_search(const struct state_set *set, fsm_state_t state) end = mid; } else if (r > 0) { start = mid + 1; + /* update mid if we're about to halt, because + * we're looking for the first position >= state, + * not the last position <= */ + if (start == end) { + mid = start; + } } else { return mid; } @@ -242,7 +254,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, */ if (!state_set_empty(set)) { i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } } @@ -261,9 +273,6 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - if (state_set_cmpval(state, set->a[i]) > 0) { - i++; - } if (i <= set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); @@ -470,7 +479,7 @@ state_set_remove(struct state_set **setp, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { if (i < set->i) { memmove(&set->a[i], &set->a[i + 1], (set->i - i - 1) * (sizeof *set->a)); } @@ -524,7 +533,7 @@ state_set_contains(const struct state_set *set, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } From 709b8cc11c5f425e6962386cd43ad36ae6eb2d68 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 16:57:13 -0400 Subject: [PATCH 034/373] stateset: Avoid memmove of size 0. --- src/adt/stateset.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 4d4de9438..645424555 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -273,8 +273,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - - if (i <= set->i) { + if (i < set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); } From cead0d99915faad077e3e22dd09761a61556c8e0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:09:01 -0400 Subject: [PATCH 035/373] stateset: Add note about potentially expensive assertion. In -O0 this can become pretty expensive (~25% of overall runtime for `time ./re -rpcre -C '^[ab]{0,2000}$'`), but when built with -O3 very little overhead remains. I'm adding this comment because every time I see this it seems to me like it should have `EXPENSIVE_CHECKS` around it, but profiling is telling me it really doesn't matter. --- src/adt/stateset.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 645424555..7839234d2 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -284,6 +284,8 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->i = 1; } + /* This assert can be pretty expensive in -O0 but in -O3 it has very + * little impact on the overall runtime. */ assert(state_set_contains(set, state)); return 1; From cbfeddd9c54178fe34aae0606a619e9592c6dd69 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:11:07 -0400 Subject: [PATCH 036/373] stateset: Comment struct fields. --- src/adt/stateset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 7839234d2..8d73a9b5f 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -44,8 +44,8 @@ struct state_set { const struct fsm_alloc *alloc; fsm_state_t *a; - size_t i; - size_t n; + size_t i; /* used */ + size_t n; /* ceil */ }; int From c3dab77fc7785e3ec4df9e4fb60c6c760ca66b0c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:12:04 -0400 Subject: [PATCH 037/373] edgeset: Fix indentation for `#if`'d block. --- src/adt/edgeset.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index c718727ca..f1a9e9f31 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -223,9 +223,9 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, assert(set->count <= set->ceil); #if LOG_BITSET - fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", - symbols[0], symbols[1], symbols[2], symbols[3], - state, (void *)set); + fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", + symbols[0], symbols[1], symbols[2], symbols[3], + state, (void *)set); #endif /* Linear search for a group with the same destination From 1ada07cce13fd2982056963e5b1e2bae59b235e9 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:12:31 -0400 Subject: [PATCH 038/373] edgeset: Switch from linear to binary searching in edge_set_add_bulk. This is a major hotspot when doing epsilon removal over large runs of potentially skipped states (as might appear from `^[ab]{0,2000}$`). Add a fast path for appending, which is also very common. Extract the edge set destination search into its own function, `find_state_position`, and add a `#define` to switch between linear search, binary search, or calling both and comparing the result. I will remove linear search in the next commit, but am checking this in as an intermediate step for checking & benchmarking. --- src/adt/edgeset.c | 182 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 17 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index f1a9e9f31..818b78493 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -11,6 +11,7 @@ #include #define LOG_BITSET 0 +#define LOG_BSEARCH 0 #include "libfsm/internal.h" /* XXX: for allocating struct fsm_edge, and the edges array */ @@ -184,6 +185,157 @@ edge_set_advise_growth(struct edge_set **pset, const struct fsm_alloc *alloc, return 1; } +enum fsp_res { + FSP_FOUND_INSERT_POSITION, + FSP_FOUND_VALUE_PRESENT, +}; + +/* Use binary search to find the first position N where set->groups[N].to >= state, + * which includes the position immediately following the last entry. Return an enum + * which indicates whether state is already present. */ +static enum fsp_res +find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + size_t lo = 0, hi = set->count; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: looking for %d in %p (count %zu)\n", + __func__, state, (void *)set, set->count); + } + +#if EXPENSIVE_CHECKS + /* invariant: input is unique and sorted */ + for (size_t i = 1; i < set->count; i++) { + assert(set->groups[i - 1].to < set->groups[i].to); + } +#endif + + if (set->count == 0) { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: empty, returning 0\n", __func__); + } + *dst = 0; + return FSP_FOUND_INSERT_POSITION; + } else { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: fast path: looking for %d, set->groups[last].to %d\n", + __func__, state, set->groups[hi - 1].to); + } + + /* Check the last entry so we can append in constant time. */ + const fsm_state_t last = set->groups[hi - 1].to; + if (state > last) { + *dst = hi; + return FSP_FOUND_INSERT_POSITION; + } else if (state == last) { + *dst = hi - 1; + return FSP_FOUND_VALUE_PRESENT; + } + } + + size_t mid; + while (lo < hi) { /* lo <= mid < hi */ + mid = lo + (hi - lo)/2; /* avoid overflow */ + const struct edge_group *eg = &set->groups[mid]; + const fsm_state_t cur = eg->to; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: lo %zu, hi %zu, mid %zu, cur %d, looking for %d\n", + __func__, lo, hi, mid, cur, state); + } + + if (state == cur) { + *dst = mid; + return FSP_FOUND_VALUE_PRESENT; + } else if (state > cur) { + lo = mid + 1; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new lo %zd\n", __func__, lo); + } + + /* Update mid if we're about to halt, because we're looking + * for the first position >= state, not the last position <=. */ + if (lo == hi) { + mid = lo; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: special case, updating mid to %zd\n", __func__, mid); + } + } + } else if (state < cur) { + hi = mid; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new hi %zd\n", __func__, hi); + } + } + } + + if (LOG_BSEARCH) { + fprintf(stderr, "%s: halting at %zd (looking for %d, cur %d)\n", + __func__, mid, state, set->groups[mid].to); + } + + /* dst is now the first position > state (== case is handled above), + * which may be one past the end of the array. */ + assert(mid == set->count || set->groups[mid].to > state); + *dst = mid; + return FSP_FOUND_INSERT_POSITION; +} + +static enum fsp_res +find_state_position_linear(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + /* Linear search for a group with the same destination + * state, or the position where that group would go. */ + size_t i; + for (i = 0; i < set->count; i++) { + const struct edge_group *eg = &set->groups[i]; + if (eg->to == state) { + *dst = i; + return FSP_FOUND_VALUE_PRESENT; + } else if (eg->to > state) { + break; /* will shift down and insert below */ + } else { + continue; + } + } + + *dst = i; + return FSP_FOUND_INSERT_POSITION; +} + +/* Find the state in the edge set, or where it would be inserted if not present. */ +static enum fsp_res +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + /* 0: linear, 1: bsearch, -1: call both, to check result */ +#define USE_BSEARCH 1 + + switch (USE_BSEARCH) { + case 0: + return find_state_position_linear(set, state, dst); + case 1: + return find_state_position_bsearch(set, state, dst); + case -1: + { + size_t dst_linear, dst_bsearch; + enum fsp_res res_linear = find_state_position_linear(set, state, &dst_linear); + enum fsp_res res_bsearch = find_state_position_bsearch(set, state, &dst_bsearch); + + if (res_linear != res_bsearch || dst_linear != dst_bsearch) { + fprintf(stderr, "%s: disagreement for state %d: linear res %d, dst %zu, bsearch res %d, dst %zu\n", + __func__, state, + res_linear, dst_linear, + res_bsearch, dst_bsearch); + for (size_t i = 0; i < set->count; i++) { + fprintf(stderr, "set->groups[%zu].to: %d\n", i, set->groups[i].to); + } + } + assert(res_linear == res_bsearch); + assert(dst_linear == dst_bsearch); + *dst = dst_linear; + return res_linear; + } + } +} + int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) @@ -228,25 +380,21 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, state, (void *)set); #endif - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - for (i = 0; i < set->count; i++) { + switch (find_state_position(set, state, &i)) { + case FSP_FOUND_VALUE_PRESENT: + assert(i < set->count); + /* This API does not indicate whether that + * symbol -> to edge was already present. */ eg = &set->groups[i]; - - if (eg->to == state) { - /* This API does not indicate whether that - * symbol -> to edge was already present. */ - size_t i; - for (i = 0; i < 256/64; i++) { - eg->symbols[i] |= symbols[i]; - } - dump_edge_set(set); - return 1; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; + for (i = 0; i < 256/64; i++) { + eg->symbols[i] |= symbols[i]; } + dump_edge_set(set); + return 1; + + break; + case FSP_FOUND_INSERT_POSITION: + break; /* continue below */ } /* insert/append at i */ From 7122d2fe739e106be8b422206b15e681027acd11 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Aug 2023 17:19:24 -0400 Subject: [PATCH 039/373] edgeset: Commit to using binary search. When I run `time ./re -rpcre -C '^[ab]{0,2000}$'` locally for -O3: - linear search: 2.991s - binary search: 1.521s --- src/adt/edgeset.c | 59 +---------------------------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index 818b78493..dd3b8853c 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -194,7 +194,7 @@ enum fsp_res { * which includes the position immediately following the last entry. Return an enum * which indicates whether state is already present. */ static enum fsp_res -find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_t *dst) +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) { size_t lo = 0, hi = set->count; if (LOG_BSEARCH) { @@ -279,63 +279,6 @@ find_state_position_bsearch(const struct edge_set *set, fsm_state_t state, size_ return FSP_FOUND_INSERT_POSITION; } -static enum fsp_res -find_state_position_linear(const struct edge_set *set, fsm_state_t state, size_t *dst) -{ - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - size_t i; - for (i = 0; i < set->count; i++) { - const struct edge_group *eg = &set->groups[i]; - if (eg->to == state) { - *dst = i; - return FSP_FOUND_VALUE_PRESENT; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; - } - } - - *dst = i; - return FSP_FOUND_INSERT_POSITION; -} - -/* Find the state in the edge set, or where it would be inserted if not present. */ -static enum fsp_res -find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) -{ - /* 0: linear, 1: bsearch, -1: call both, to check result */ -#define USE_BSEARCH 1 - - switch (USE_BSEARCH) { - case 0: - return find_state_position_linear(set, state, dst); - case 1: - return find_state_position_bsearch(set, state, dst); - case -1: - { - size_t dst_linear, dst_bsearch; - enum fsp_res res_linear = find_state_position_linear(set, state, &dst_linear); - enum fsp_res res_bsearch = find_state_position_bsearch(set, state, &dst_bsearch); - - if (res_linear != res_bsearch || dst_linear != dst_bsearch) { - fprintf(stderr, "%s: disagreement for state %d: linear res %d, dst %zu, bsearch res %d, dst %zu\n", - __func__, state, - res_linear, dst_linear, - res_bsearch, dst_bsearch); - for (size_t i = 0; i < set->count; i++) { - fprintf(stderr, "set->groups[%zu].to: %d\n", i, set->groups[i].to); - } - } - assert(res_linear == res_bsearch); - assert(dst_linear == dst_bsearch); - *dst = dst_linear; - return res_linear; - } - } -} - int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) From 937585a955d277231d558fe5ae20b0cc40c87eda Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:08:44 -0400 Subject: [PATCH 040/373] determinise: Drastically reduce calls to qsort. After the other changes in this PR, calls to qsort from `sort_and_dedup_dst_buf` are one of the largest remaining hotspots in the profile. We can often avoid calling qsort, though: - If there is <= 1 entry, just return, it's sorted. - Otherwise, first do a sweep through the array noting the min and max values. Unless there is a huge range between them, it's much faster to build a bitset from them in a small (max 10KB) stack-allocated array and then unpack the bitset (now sorted and unique). Only the needed portion of the array is initialized. I have not done a lot of experimentation to find a cutoff point where the bitset becomes slower than qsort (it may be much larger), I picked 10KB because it's likely to be safe to stack-allocate. I tried changing the bitset unpacking to use an 8 or 16 bit mask and jump forward faster through large sub-word ranges of 0 bits, but any improvement was lost among random variation, so I decided it wasn't worth the extra complexity. We already skip whole words that are 0. --- src/libfsm/determinise.c | 95 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 56e135afd..ac15f05a8 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2016,28 +2016,87 @@ static void sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) { const size_t orig_used = *used; - qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); - - /* squash out duplicates */ - size_t rd = 1; - size_t wr = 1; - while (rd < orig_used) { - if (buf[rd - 1] == buf[rd]) { - rd++; /* skip */ - } else { - buf[wr] = buf[rd]; - rd++; - wr++; - } + + if (orig_used <= 1) { + return; /* no change */ } - *used = wr; -#if EXPENSIVE_CHECKS - assert(wr <= orig_used); - for (size_t i = 1; i < *used; i++) { - assert(buf[i - 1] < buf[i]); + /* Figure out what the min and max values are, because + * when the difference between them is not too large it + * can be significantly faster to avoid qsort here. */ + fsm_state_t min = (fsm_state_t)-1; + fsm_state_t max = 0; + for (size_t i = 0; i < orig_used; i++) { + const fsm_state_t cur = buf[i]; + if (cur < min) { min = cur; } + if (cur > max) { max = cur; } } + + /* If there's only one unique value, then we're done. */ + if (min == max) { + buf[0] = min; + *used = 1; + return; + } + +/* 81920 = 10 KB buffer on the stack. This must be divisible by 64. + * Set to 0 to disable. */ +#define QSORT_CUTOFF 81920 + + if (QSORT_CUTOFF == 0 || max - min > QSORT_CUTOFF) { + /* If the bitset would be very large but sparse due to + * extreme values, then fall back on using qsort and + * then sweeping over the array to squash out + * duplicates. */ + qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); + + /* squash out duplicates */ + size_t rd = 1; + size_t wr = 1; + while (rd < orig_used) { + if (buf[rd - 1] == buf[rd]) { + rd++; /* skip */ + } else { + buf[wr] = buf[rd]; + rd++; + wr++; + } + } + + *used = wr; +#if EXPENSIVE_CHECKS + assert(wr <= orig_used); + for (size_t i = 1; i < *used; i++) { + assert(buf[i - 1] < buf[i]); + } #endif + } else { + /* Convert the array into a bitset and back, which sorts + * and deduplicates in the process. Add 1 to avoid a zero- + * zero-length array error if QSORT_CUTOFF is 0. */ + uint64_t bitset[QSORT_CUTOFF/64 + 1]; + const size_t words = u64bitset_words(max - min); + memset(bitset, 0x00, words * sizeof(bitset[0])); + + for (size_t i = 0; i < orig_used; i++) { + u64bitset_set(bitset, buf[i] - min); + } + + size_t dst = 0; + for (size_t i = 0; i < words; i++) { + const uint64_t w = bitset[i]; + if (w != 0) { /* skip empty words */ + uint64_t bit = 0x1; + for (size_t b_i = 0; b_i < 64; b_i++, bit <<= 1) { + if (w & bit) { + buf[dst] = 64*i + b_i + min; + dst++; + } + } + } + } + *used = dst; + } } static int From 30e34ef9a739c78f788753919d68d6925171a0d5 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:17:56 -0400 Subject: [PATCH 041/373] edgeset: Remove stale comment. --- src/adt/edgeset.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index dd3b8853c..9658213c8 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -326,8 +326,6 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, switch (find_state_position(set, state, &i)) { case FSP_FOUND_VALUE_PRESENT: assert(i < set->count); - /* This API does not indicate whether that - * symbol -> to edge was already present. */ eg = &set->groups[i]; for (i = 0; i < 256/64; i++) { eg->symbols[i] |= symbols[i]; From cf6051f26135802784795465232369e5dfc66c66 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:46:35 -0400 Subject: [PATCH 042/373] UBSan: Avoid implicit signed/unsigned conversion. --- src/libre/ast_compile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 502faf8b4..aa0902d55 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -208,11 +208,11 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, assert(to < env->fsm->statecount); if (re_flags & RE_ICASE) { - if (!fsm_addedge_literal(fsm, from, to, tolower((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)tolower((unsigned char) c))) { return 0; } - if (!fsm_addedge_literal(fsm, from, to, toupper((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)toupper((unsigned char) c))) { return 0; } } else { From c1e12828b2e79a11b61978d809a0e020f5d8a6e6 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 12:49:22 -0400 Subject: [PATCH 043/373] UBSan: Avoid implicit signed/unsigned conversion. --- src/retest/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/main.c b/src/retest/main.c index b6b4c52f7..e01c93e7c 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -393,7 +393,7 @@ parse_escapes(char *s, char **errpos, int *lenp) ndig++; } else { - s[j++] = ccode; + s[j++] = (char)ccode; st = ST_BARE; if (!hexcurly) { From 6eff0f98ce625e93ba025f0f4c7358153708cc92 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Aug 2023 13:17:12 -0400 Subject: [PATCH 044/373] bugfix: The range is min..max inclusive, so add 1. If min and max are exactly 64 states apart the upper value was getting silently dropped due to an incorrect `words` value here. One of the patterns in the PCRE suite triggers this: ./re -rpcre '(?:c|d)(?:)(?:aaaaaaaa(?:)(?:bbbbbbbb)(?:bbbbbbbb(?:))(?:bbbbbbbb(?:)(?:bbbbbbbb)))' "caaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" This should match, but did not. --- src/libfsm/determinise.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index ac15f05a8..3dbdc429e 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2075,7 +2075,7 @@ sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) * and deduplicates in the process. Add 1 to avoid a zero- * zero-length array error if QSORT_CUTOFF is 0. */ uint64_t bitset[QSORT_CUTOFF/64 + 1]; - const size_t words = u64bitset_words(max - min); + const size_t words = u64bitset_words(max - min + 1); memset(bitset, 0x00, words * sizeof(bitset[0])); for (size_t i = 0; i < orig_used; i++) { From 98ee906d15426150020070db975778bf5f97a565 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 15 Jun 2023 14:00:25 -0400 Subject: [PATCH 045/373] Address a couple warnings from scan-build. determinise: It's not possible to find a cached result in the hash table without allocating a to-set buffer first, so assert that it will be non-NULL. fsm_findmode: This should never be used on a state without edges. vm/v1.c and vm/v2.c: Free allocated return value on error. --- src/libfsm/determinise.c | 2 ++ src/libfsm/mode.c | 4 ++++ src/libfsm/vm/v1.c | 4 +++- src/libfsm/vm/v2.c | 5 ++++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 3dbdc429e..9c77f3a42 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1339,6 +1339,7 @@ to_set_htab_check(struct analyze_closures_env *env, if (b->count == 0) { return 0; /* empty bucket -> not found */ } else if (b->count == count) { + assert(env->to_sets.buf != NULL); assert(b->offset + count <= env->to_sets.used); const fsm_state_t *ids = &env->to_sets.buf[b->offset]; if (0 == memcmp(ids, dst, count * sizeof(dst[0]))) { @@ -1465,6 +1466,7 @@ save_to_set(struct analyze_closures_env *env, env->to_sets.ceil = nceil; env->to_sets.buf = nbuf; } + assert(env->to_sets.buf != NULL); #if LOG_TO_SET static size_t to_set_id; diff --git a/src/libfsm/mode.c b/src/libfsm/mode.c index 76c60b8ad..87af0bdf9 100644 --- a/src/libfsm/mode.c +++ b/src/libfsm/mode.c @@ -28,6 +28,7 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) } mode; mode.freq = 1; + mode.state = (fsm_state_t)-1; edge_set_group_iter_reset(fsm->states[state].edges, EDGE_GROUP_ITER_ALL, &iter); while (edge_set_group_iter_next(&iter, &info)) { @@ -46,6 +47,9 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) *freq = mode.freq; } + /* It's not meaningful to call this on a state without edges. */ + assert(mode.state != (fsm_state_t)-1); + assert(mode.freq >= 1); return mode.state; } diff --git a/src/libfsm/vm/v1.c b/src/libfsm/vm/v1.c index a326b88d8..de1f6ea93 100644 --- a/src/libfsm/vm/v1.c +++ b/src/libfsm/vm/v1.c @@ -217,7 +217,9 @@ encode_opasm_v1(const struct dfavm_vm_op *instr, size_t ninstr, size_t total_byt return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } return NULL; } diff --git a/src/libfsm/vm/v2.c b/src/libfsm/vm/v2.c index c85edff98..07eb12ef4 100644 --- a/src/libfsm/vm/v2.c +++ b/src/libfsm/vm/v2.c @@ -155,7 +155,10 @@ encode_opasm_v2(const struct dfavm_vm_op *instr, size_t ninstr) return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } + return NULL; } From 51892e3745fd642f6bb8a6acd31b69cf065a43b3 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 16 Feb 2023 13:52:31 -0500 Subject: [PATCH 046/373] Add src/adt/idmap.c, a state -> ID set map. --- Makefile | 1 + include/adt/idmap.h | 58 ++++++ src/adt/Makefile | 1 + src/adt/idmap.c | 392 ++++++++++++++++++++++++++++++++++++++ tests/idmap/Makefile | 19 ++ tests/idmap/idmap_basic.c | 136 +++++++++++++ 6 files changed, 607 insertions(+) create mode 100644 include/adt/idmap.h create mode 100644 src/adt/idmap.c create mode 100644 tests/idmap/Makefile create mode 100644 tests/idmap/idmap_basic.c diff --git a/Makefile b/Makefile index 8d742883e..f1f4f1396 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,7 @@ SUBDIR += src SUBDIR += tests/capture SUBDIR += tests/complement SUBDIR += tests/gen +SUBDIR += tests/idmap SUBDIR += tests/intersect #SUBDIR += tests/ir # XXX: fragile due to state numbering SUBDIR += tests/eclosure diff --git a/include/adt/idmap.h b/include/adt/idmap.h new file mode 100644 index 000000000..064fd15d1 --- /dev/null +++ b/include/adt/idmap.h @@ -0,0 +1,58 @@ +#ifndef IDMAP_H +#define IDMAP_H + +/* Mapping between one fsm_state_t and a set of + * unsigned IDs. The implementation assumes that both + * IDs are sequentially assigned and don't need a sparse + * mapping -- it will handle 10 -> [1, 3, 47] well, but + * not 1000000 -> [14, 524288, 1073741823]. */ + +#include + +#include "fsm/fsm.h" +#include "fsm/alloc.h" + +struct idmap; /* Opaque handle. */ + +struct idmap * +idmap_new(const struct fsm_alloc *alloc); + +void +idmap_free(struct idmap *m); + +/* Associate a value with a state (if not already present.) + * Returns 1 on success, or 0 on allocation failure. */ +int +idmap_set(struct idmap *m, fsm_state_t state_id, unsigned value); + +/* How many values are associated with an ID? */ +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id); + +/* Get the values associated with an ID. + * + * Returns 1 on success and writes them into the buffer, in ascending + * order, with the count in *written (if non-NULL). + * + * Returns 0 on error (insufficient buffer space). */ +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written); + +/* Iterator callback. */ +typedef void +idmap_iter_fun(fsm_state_t state_id, unsigned value, void *opaque); + +/* Iterate over the ID map. State IDs may be yielded out of order, + * values will be in ascending order. */ +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque); + +/* Iterate over the values associated with a single state + * (in ascending order). */ +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque); + +#endif diff --git a/src/adt/Makefile b/src/adt/Makefile index 05199f2dc..64ad7429f 100644 --- a/src/adt/Makefile +++ b/src/adt/Makefile @@ -2,6 +2,7 @@ SRC += src/adt/alloc.c SRC += src/adt/bitmap.c +SRC += src/adt/idmap.c SRC += src/adt/internedstateset.c SRC += src/adt/priq.c SRC += src/adt/path.c diff --git a/src/adt/idmap.c b/src/adt/idmap.c new file mode 100644 index 000000000..ca169b71e --- /dev/null +++ b/src/adt/idmap.c @@ -0,0 +1,392 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "adt/idmap.h" + +#include "adt/alloc.h" +#include "adt/hash.h" +#include "adt/u64bitset.h" + +#include +#include +#include + +#define NO_STATE ((fsm_state_t)-1) + +#define DEF_BUCKET_COUNT 4 + +struct idmap { + const struct fsm_alloc *alloc; + unsigned bucket_count; + unsigned buckets_used; + + /* All buckets' values are assumed to be large + * enough to store this value, and they will all + * grow as necessary. */ + unsigned max_value; + + /* Basic linear-probing, add-only hash table. */ + struct idmap_bucket { + fsm_state_t state; /* Key. NO_STATE when empty. */ + + /* values[] is always either NULL or has at least + * max_value + 1 bits; all grow on demand. */ + uint64_t *values; + } *buckets; +}; + +static unsigned +value_words(unsigned max_value) { + if (max_value == 0) { + /* Still allocate one word, for storing 0. */ + return 1; + } else { + return u64bitset_words(max_value); + } +} + +struct idmap * +idmap_new(const struct fsm_alloc *alloc) +{ + struct idmap *res = NULL; + struct idmap_bucket *buckets = NULL; + + res = f_malloc(alloc, sizeof(*res)); + if (res == NULL) { + goto cleanup; + } + + buckets = f_calloc(alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + goto cleanup; + } + + for (size_t i = 0; i < DEF_BUCKET_COUNT; i++) { + buckets[i].state = NO_STATE; + } + + res->alloc = alloc; + res->buckets_used = 0; + res->bucket_count = DEF_BUCKET_COUNT; + res->max_value = 0; + res->buckets = buckets; + + return res; + +cleanup: + f_free(alloc, res); + f_free(alloc, buckets); + return NULL; +} + +void +idmap_free(struct idmap *m) +{ + if (m == NULL) { + return; + } + + for (size_t i = 0; i < m->bucket_count; i++) { + if (m->buckets[i].state == NO_STATE) { + continue; + } + f_free(m->alloc, m->buckets[i].values); + } + + f_free(m->alloc, m->buckets); + f_free(m->alloc, m); +} + +static int +grow_bucket_values(struct idmap *m, unsigned old_words, unsigned new_words) +{ + assert(new_words > old_words); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + assert(b->values == NULL); + continue; + } + + uint64_t *nv = f_calloc(m->alloc, + new_words, sizeof(nv[0])); + if (nv == NULL) { + return 0; + } + + for (size_t w_i = 0; w_i < old_words; w_i++) { + nv[w_i] = b->values[w_i]; + } + f_free(m->alloc, b->values); + b->values = nv; + } + return 1; +} + +static int +grow_buckets(struct idmap *m) +{ + const size_t ocount = m->bucket_count; + const size_t ncount = 2*ocount; + assert(ncount > m->bucket_count); + + struct idmap_bucket *nbuckets = f_calloc(m->alloc, + ncount, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + nbuckets[nb_i].state = NO_STATE; + } + + const size_t nmask = ncount - 1; + + for (size_t ob_i = 0; ob_i < ocount; ob_i++) { + const struct idmap_bucket *ob = &m->buckets[ob_i]; + if (ob->state == NO_STATE) { + continue; + } + + const uint64_t h = hash_id(ob->state); + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + struct idmap_bucket *nb = &nbuckets[(h + nb_i) & nmask]; + if (nb->state == NO_STATE) { + nb->state = ob->state; + nb->values = ob->values; + break; + } else { + assert(nb->state != ob->state); + /* collision */ + continue; + } + } + } + + f_free(m->alloc, m->buckets); + + m->buckets = nbuckets; + m->bucket_count = ncount; + + return 1; +} + +int +idmap_set(struct idmap *m, fsm_state_t state_id, + unsigned value) +{ + assert(state_id != NO_STATE); + + const uint64_t h = hash_id(state_id); + if (value > m->max_value) { + const unsigned ovw = value_words(m->max_value); + const unsigned nvw = value_words(value); + /* If this value won't fit in the existing value + * arrays, then grow them all. We do not track the + * number of bits in each individual array. */ + if (nvw > ovw && !grow_bucket_values(m, ovw, nvw)) { + return 0; + } + m->max_value = value; + } + + assert(m->max_value >= value); + + if (m->buckets_used >= m->bucket_count/2) { + if (!grow_buckets(m)) { + return 0; + } + } + + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == state_id) { + assert(b->values != NULL); + u64bitset_set(b->values, value); + return 1; + } else if (b->state == NO_STATE) { + b->state = state_id; + assert(b->values == NULL); + + const unsigned vw = value_words(m->max_value); + b->values = f_calloc(m->alloc, + vw, sizeof(b->values[0])); + if (b->values == NULL) { + return 0; + } + m->buckets_used++; + + u64bitset_set(b->values, value); + return 1; + } else { + continue; /* collision */ + } + + } + + assert(!"unreachable"); + return 0; +} + +static const struct idmap_bucket * +get_bucket(const struct idmap *m, fsm_state_t state_id) +{ + const uint64_t h = hash_id(state_id); + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == NO_STATE) { + return NULL; + } else if (b->state == state_id) { + return b; + } + } + + return NULL; +} + +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return 0; + } + assert(b->values != NULL); + + size_t res = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + /* This could use popcount64(w). */ + if (w == 0) { + continue; + } + for (uint64_t bit = 1; bit; bit <<= 1) { + if (w & bit) { + res++; + } + } + } + + return res; +} + +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + if (written != NULL) { + *written = 0; + } + return 1; + } + + size_t buf_offset = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + if (buf_offset * sizeof(buf[0]) >= buf_size) { + return 0; + } + buf[buf_offset] = 64*w_i + b_i; + buf_offset++; + } + } + } + + if (written != NULL) { + *written = buf_offset; + } + return 1; +} + +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + continue; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + cb(b->state, v, opaque); + } + } + } + } +} + +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + /* if N contiguous bits are all zero, skip them all at once */ +#define BLOCK_BITS 16 + uint64_t block = ((uint64_t)1 << BLOCK_BITS) - 1; + size_t block_count = 0; + + uint64_t b_i = 0; + while (b_i < 64) { + if ((w & block) == 0) { + block <<= BLOCK_BITS; + b_i += BLOCK_BITS; + continue; + } + + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + cb(b->state, v, opaque); + block_count++; + } + b_i++; + block <<= 1; + } + +#define CHECK 0 +#if CHECK + size_t check_count = 0; + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + check_count++; + } + } + assert(block_count == check_count); +#endif + } +} diff --git a/tests/idmap/Makefile b/tests/idmap/Makefile new file mode 100644 index 000000000..aee01f565 --- /dev/null +++ b/tests/idmap/Makefile @@ -0,0 +1,19 @@ +.include "../../share/mk/top.mk" + +TEST.tests/idmap != ls -1 tests/idmap/idmap*.c +TEST_SRCDIR.tests/idmap = tests/idmap +TEST_OUTDIR.tests/idmap = ${BUILD}/tests/idmap + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +INCDIR.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += src/adt +.endfor + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +test:: ${TEST_OUTDIR.tests/idmap}/res${n} +SRC += ${TEST_SRCDIR.tests/idmap}/idmap${n}.c +CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += -UNDEBUG -D_DEFAULT_SOURCE -std=c99 +${TEST_OUTDIR.tests/idmap}/run${n}: ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c} -o ${TEST_OUTDIR.tests/idmap}/run${n} ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o +${TEST_OUTDIR.tests/idmap}/res${n}: ${TEST_OUTDIR.tests/idmap}/run${n} + ( ${TEST_OUTDIR.tests/idmap}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/idmap}/res${n} +.endfor diff --git a/tests/idmap/idmap_basic.c b/tests/idmap/idmap_basic.c new file mode 100644 index 000000000..19f44d56e --- /dev/null +++ b/tests/idmap/idmap_basic.c @@ -0,0 +1,136 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#define DEF_LIMIT 10 +#define DEF_SEED 0 + +/* Thes numbers were chose to get a reasonable variety, + * but also some duplicated values as the input grows. */ +#define MAX_GEN_VALUES 23 +#define ID_MASK ((1 << 9) - 1) +#define VALUE_MASK ((1 << 10) - 1) + +static void +dump_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + /* fprintf(stderr, " -- state %d, value %u\n", state_id, value); */ + assert(state_id <= ID_MASK); + assert(value <= VALUE_MASK); + (void)opaque; +} + +static int +cmp_u(const void *pa, const void *pb) +{ + const unsigned a = *(unsigned *)pa; + const unsigned b = *(unsigned *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +int main(int argc, char **argv) { + const size_t limit = (argc > 1 ? atoi(argv[1]) : DEF_LIMIT); + const unsigned seed = (argc > 2 ? atoi(argv[2]) : DEF_SEED); + + (void)argc; + (void)argv; + struct idmap *m = idmap_new(NULL); + + srandom(seed); + + /* Fill the table with random data */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t value_count = random() % MAX_GEN_VALUES; + + for (size_t v_i = 0; v_i < value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + if (!idmap_set(m, id, v)) { + assert(!"failed to set"); + } + } + } + + idmap_iter(m, dump_cb, NULL); + + srandom(seed); + + size_t got_buf_ceil = MAX_GEN_VALUES; + unsigned *got_buf = malloc(got_buf_ceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + + /* Reset the PRNG and read back the same data. */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t generated_value_count = random() % MAX_GEN_VALUES; + + /* Note: This can occasionally differ from + * generated_value_count, because the same id or values + * may have been generated more than once. As long as + * all the values match, it's fine. */ + const size_t value_count = idmap_get_value_count(m, id); + + if (value_count > got_buf_ceil) { + size_t nceil = got_buf_ceil; + while (nceil <= value_count) { + nceil *= 2; + } + free(got_buf); + got_buf = malloc(nceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + got_buf_ceil = nceil; + } + + size_t written; + if (!idmap_get(m, id, + got_buf_ceil * sizeof(got_buf[0]), got_buf, + &written)) { + assert(!"failed to get"); + } + assert(written == value_count); + + unsigned gen_buf[MAX_GEN_VALUES]; + + for (size_t v_i = 0; v_i < generated_value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + gen_buf[v_i] = v; + } + qsort(gen_buf, generated_value_count, sizeof(gen_buf[0]), cmp_u); + + /* Every generated value should appear in the buffer. + * There may be more in the buffer; ignore them. */ + size_t v_i = 0; + for (size_t gen_i = 0; gen_i < generated_value_count; gen_i++) { + int found = 0; + const unsigned gv = gen_buf[gen_i]; + assert(value_count <= got_buf_ceil); + /* got_buf should be sorted, so we can pick up where we left off */ + while (v_i < value_count) { + if (gv == got_buf[v_i]) { + /* Intentionally don't increment v_i on match, + * because gen_buf can repeat values. */ + found = 1; + break; + } + v_i++; + } + if (!found) { + fprintf(stderr, "NOT FOUND: state %d -- value: %u\n", + id, gv); + return EXIT_FAILURE; + } + } + } + + free(got_buf); + idmap_free(m); + return EXIT_SUCCESS; +} From 7c6644f53b94c3be7246a67eaa985b03dfb0d526 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 10 Jul 2023 14:02:31 -0400 Subject: [PATCH 047/373] Remove theft test harness for deleted ADT (ipriq). --- theft/Makefile | 1 - theft/fuzz_adt_ipriq.c | 197 ----------------------------------------- 2 files changed, 198 deletions(-) delete mode 100644 theft/fuzz_adt_ipriq.c diff --git a/theft/Makefile b/theft/Makefile index 0d38d8cfc..921c482a9 100644 --- a/theft/Makefile +++ b/theft/Makefile @@ -6,7 +6,6 @@ SRC += theft/util.c SRC += theft/wrap.c SRC += theft/fuzz_adt_edge_set.c -SRC += theft/fuzz_adt_ipriq.c SRC += theft/fuzz_adt_priq.c SRC += theft/fuzz_capture_string_set.c SRC += theft/fuzz_literals.c diff --git a/theft/fuzz_adt_ipriq.c b/theft/fuzz_adt_ipriq.c deleted file mode 100644 index 1847ef6ce..000000000 --- a/theft/fuzz_adt_ipriq.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2021 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include "type_info_adt_ipriq.h" - -#include -#include - -struct model { - size_t used; - size_t entries[]; -}; - -static enum ipriq_cmp_res -cmp_size_t(size_t a, size_t b, void *opaque) -{ - (void)opaque; - return a < b ? IPRIQ_CMP_LT : - a > b ? IPRIQ_CMP_GT : IPRIQ_CMP_EQ; -} - -static int exec_add(size_t x, struct model *m, struct ipriq *pq) -{ - if (!ipriq_add(pq, x)) { - return 0; - } - - m->entries[m->used] = x; - m->used++; - return 1; -} - -static int find_min_pos(const struct model *m, size_t *pos) -{ - size_t i; - if (m->used == 0) { - return 0; - } - - size_t res, min; - res = 0; - min = m->entries[0]; - - for (i = 1; i < m->used; i++) { - if (m->entries[i] < min) { - res = i; - min = m->entries[i]; - } - } - *pos = res; - return 1; -} - -static int exec_peek(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_peek(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (peek)"); - } - - return res == m->entries[pos]; -} - -static int exec_pop(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_pop(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (pop)"); - } - - if (res != m->entries[pos]) { - return 0; - } - - assert(m->used > 0); - if (pos < m->used - 1) { - m->entries[pos] = m->entries[m->used - 1]; - } - m->used--; - return 1; -} - -static enum theft_trial_res -compare_against_model(const struct ipriq_scenario *scen) -{ - enum theft_trial_res res = THEFT_TRIAL_FAIL; - size_t i; - - struct model *m = malloc(sizeof(*m) - + scen->count * sizeof(m->entries[0])); - if (m == NULL) { - return THEFT_TRIAL_ERROR; - } - m->used = 0; - - struct ipriq *pq = ipriq_new(NULL, cmp_size_t, NULL); - if (pq == NULL) { - return THEFT_TRIAL_ERROR; - } - - for (i = 0; i < scen->count; i++) { - const struct ipriq_op *op = &scen->ops[i]; - - switch (op->t) { - case IPRIQ_OP_ADD: - if (!exec_add(op->u.add.x, m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_PEEK: - if (!exec_peek(m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_POP: - if (!exec_pop(m, pq)) { - goto cleanup; - } - break; - - default: - assert(false); break; - } - } - - res = THEFT_TRIAL_PASS; - -cleanup: - free(m); - - return res; -} - -static enum theft_trial_res -prop_ipriq_model(struct theft *t, void *arg1) -{ - const struct ipriq_scenario *scen = arg1; - (void)t; - return compare_against_model(scen); -} - -static bool -test_ipriq(theft_seed seed, uintptr_t limit) -{ - enum theft_run_res res; - - struct ipriq_hook_env env = { - .tag = 'I', - .limit = limit, - }; - - struct theft_run_config config = { - .name = __func__, - .prop1 = prop_ipriq_model, - .type_info = { &type_info_adt_ipriq }, - .trials = 1000, - .hooks = { - .trial_pre = theft_hook_first_fail_halt, - .env = &env, - }, - .fork = { - .enable = true, - }, - - .seed = seed, - }; - - (void)limit; - - res = theft_run(&config); - printf("%s: %s\n", __func__, theft_run_res_str(res)); - - return res == THEFT_RUN_PASS; -} - -void -register_test_adt_ipriq(void) -{ - reg_test1("adt_ipriq", test_ipriq, 10000); -} From c646868316ac8281ab38c33e2f9dae5cb7a5f7d8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 2 Jun 2023 11:41:28 -0400 Subject: [PATCH 048/373] Add pcre-anchor test for anchoring edge case. --- tests/pcre-anchor/in81.re | 1 + tests/pcre-anchor/out81.fsm | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/pcre-anchor/in81.re create mode 100644 tests/pcre-anchor/out81.fsm diff --git a/tests/pcre-anchor/in81.re b/tests/pcre-anchor/in81.re new file mode 100644 index 000000000..8b5fad7c3 --- /dev/null +++ b/tests/pcre-anchor/in81.re @@ -0,0 +1 @@ +($x)* \ No newline at end of file diff --git a/tests/pcre-anchor/out81.fsm b/tests/pcre-anchor/out81.fsm new file mode 100644 index 000000000..2cdc2f023 --- /dev/null +++ b/tests/pcre-anchor/out81.fsm @@ -0,0 +1,5 @@ +0 -> 0 ?; +0 -> 1 "\n"; + +start: 0; +end: 0, 1; \ No newline at end of file From 0789d614a22e2302132fb80f1fde4f6a056f793c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 31 May 2023 17:14:35 -0400 Subject: [PATCH 049/373] fuzz/run_fuzzer: Run single seed file when given as argument. --- fuzz/run_fuzzer | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fuzz/run_fuzzer b/fuzz/run_fuzzer index be8ba1d95..429ffa961 100755 --- a/fuzz/run_fuzzer +++ b/fuzz/run_fuzzer @@ -4,6 +4,8 @@ BUILD=../build FUZZER=${BUILD}/fuzz/fuzzer SEEDS=${BUILD}/fuzz/fuzzer_seeds +ARG=$1 + SECONDS=${SECONDS:-60} WORKERS=${WORKERS:-4} SEEDS=${SEEDS:-seeds} @@ -25,5 +27,9 @@ if [ ! -d "${SEEDS}" ]; then mkdir -p "${SEEDS}" fi -echo "\n==== ${FUZZER}" -${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +if [ -z "${ARG}" ]; then + echo "\n==== ${FUZZER}" + exec ${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +else + exec ${FUZZER} ${ARG} +fi \ No newline at end of file From 1ca3726322abed8c02f7f16bffe8b32683189605 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 12 Sep 2023 12:14:20 +0100 Subject: [PATCH 050/373] Don't purge the seed cache for PRs syncing clones. --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 429ec2523..27fdf78f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,8 +409,15 @@ jobs: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - # note we do the fuzzing unconditionally; each run adds to the corpus + # note we do the fuzzing unconditionally; each run adds to the corpus. + # + # We only run fuzzing for PRs in the base repo, this prevents attempting + # to purge the seed cache from a PR syncing a forked repo, which fails + # due to a permissions error (I'm unsure why, I think PRs from clones can't + # purge a cache in CI presumably for security/DoS reasons). PRs from clones + # still run fuzzing, just from empty, and do not save their seeds. - name: Restore seeds (mode ${{ matrix.mode }}) + if: github.repository == 'katef/libfsm' uses: actions/cache/restore@v3 id: cache-seeds with: From 239927c3696e6953e1a48840a01451addac32c2b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 2 Oct 2023 09:43:22 -0400 Subject: [PATCH 051/373] ci.yml: Update fuzzer modes for CI. - 'd' (default) no longer exists, it's now 'r' - add 's', 'i', 'M' --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27fdf78f7..3315fc025 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -374,7 +374,7 @@ jobs: cc: [ clang ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op - mode: [ m, p, d ] + mode: [ r, s, m, i, M, p ] exclude: - os: macos cc: gcc # it's clang anyway From 928a696db5d47ce950ba9c56f366e0fe81da5639 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 19 Oct 2023 13:34:02 -0400 Subject: [PATCH 052/373] Add error code for unsupported PCRE cases (RE_EUNSUPPPCRE), reject one. There's extra error codes in #440 for regexes that aren't UNSATISFIABLE per se, but depend on particular corner cases in PCRE that probably aren't worth supporting in an automata-based implementation. Add a test case for one, tests/pcre/in48.re: ^a|$[^x]b* This is a tricky one to handle properly; according to PCRE it should match either "a" OR "\n", but nothing else. The newline match is because $ is a non-input-consuming check that evaluation is either at the end of input, or at a newline immediately before the end. In this case `$[^x]b*` matches exactly one newline; it's equivalent to "$\n". This probably isn't worth supporting, but we can detect cases where a potential newline match appears after a $ and reject them as an unsupported PCRE behavior. --- include/re/re.h | 2 + src/libre/ast.h | 1 + src/libre/ast_analysis.c | 103 +++++++++++++++++++++++++++++++++++++-- src/libre/ast_analysis.h | 4 +- src/libre/print/tree.c | 1 + src/libre/re.c | 10 +++- src/libre/strerror.c | 2 + tests/pcre/in48.re | 1 + tests/pcre/out48.err | 1 + 9 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 tests/pcre/in48.re create mode 100644 tests/pcre/out48.err diff --git a/include/re/re.h b/include/re/re.h index deab6caed..f876f635a 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -46,6 +46,8 @@ enum re_errno { RE_EERRNO = 1 | RE_MISC, RE_EBADDIALECT = 2 | RE_MISC, RE_EBADGROUP = 3 | RE_MISC, + RE_EUNSUPCAPTUR = 4 | RE_MISC, + RE_EUNSUPPPCRE = 5 | RE_MISC, RE_ENEGRANGE = 0 | RE_MARK | RE_GROUP, RE_ENEGCOUNT = 1 | RE_MARK | RE_GROUP, diff --git a/src/libre/ast.h b/src/libre/ast.h index 233744847..24031887e 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -98,6 +98,7 @@ enum ast_flags { AST_FLAG_ANCHORED_START = 1 << 6, AST_FLAG_ANCHORED_END = 1 << 7, AST_FLAG_END_NL = 1 << 8, + AST_FLAG_MATCHES_1NEWLINE= 1 << 9, AST_FLAG_NONE = 0x00 }; diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 3298f62c2..e3e85526a 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -552,6 +552,46 @@ set_flags_subtree(struct ast_expr *n, enum ast_flags flags) } } +static int +can_consume_single_newline(struct ast_expr *n) +{ + if (!can_consume_input(n)) { return 0; } + + if (n->flags & AST_FLAG_MATCHES_1NEWLINE) { return 1; } + + switch (n->type) { + case AST_EXPR_LITERAL: + return n->u.literal.c == '\n'; + + case AST_EXPR_CODEPOINT: + return n->u.codepoint.u == (uint32_t)'\n'; + + case AST_EXPR_RANGE: + if ((n->u.range.from.type == AST_ENDPOINT_LITERAL) && + (n->u.range.to.type == AST_ENDPOINT_LITERAL)) { + return n->u.range.from.u.literal.c <= '\n' + && n->u.range.to.u.literal.c >= '\n'; + } else if ((n->u.range.from.type == AST_ENDPOINT_CODEPOINT) && + (n->u.range.to.type == AST_ENDPOINT_CODEPOINT)) { + return n->u.range.from.u.codepoint.u <= '\n' + && n->u.range.to.u.codepoint.u >= '\n'; + } else if (n->u.range.from.type == AST_ENDPOINT_NAMED) { + /* TODO: unreachable? */ + break; + } + break; + + case AST_EXPR_SUBTRACT: + return can_consume_single_newline(n->u.subtract.a) + && !can_consume_single_newline(n->u.subtract.b); + + default: + break; + } + + return 0; +} + struct anchoring_env { enum re_flags re_flags; @@ -562,6 +602,7 @@ struct anchoring_env { /* Corresponding flag for end anchors while sweeping backward. */ int followed_by_consuming; + int followed_by_consuming_newline; int before_start_anchor; }; @@ -644,6 +685,9 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_LITERAL: case AST_EXPR_CODEPOINT: case AST_EXPR_RANGE: + if (can_consume_single_newline(n)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } break; /* handled outside switch/case */ case AST_EXPR_CONCAT: { @@ -810,6 +854,13 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } } + for (i = 0; i < n->u.concat.count; i++) { + struct ast_expr *child = n->u.concat.n[i]; + if (can_consume_single_newline(child)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + } + break; } @@ -846,6 +897,9 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } else if (res == AST_ANALYSIS_OK) { all_set_past_always_consuming &= child_env.past_always_consuming; any_sat = 1; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE + || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + continue; } else { return res; } @@ -858,6 +912,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) all_end_anchored = 0; } } + + if (child->flags & AST_FLAG_MATCHES_1NEWLINE) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } } if (!env->past_always_consuming && all_set_past_always_consuming) { @@ -925,6 +983,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) return res; } + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_END && n->u.repeat.min > 0) { /* FIXME: if repeating something that is always * anchored at the end, repeat.max could be @@ -964,6 +1026,11 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } while(0) PROPAGATE_CHILD_FLAGS("GROUP", n, n->u.group.e); + + if (n->u.group.e->flags & AST_FLAG_MATCHES_1NEWLINE) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + break; case AST_EXPR_SUBTRACT: @@ -991,6 +1058,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } return res; } + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + break; default: @@ -1048,11 +1119,18 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) assert(n->flags & AST_FLAG_ANCHORED_END); if (env->followed_by_consuming) { - LOG(3 - LOG_ANCHORING, - "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", - __func__); - set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_UNSATISFIABLE; + if (env->followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: END anchor & followed_by_consuming, returning UNSUPPORTED_PCRE\n", + __func__); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } else { + LOG(3 - LOG_ANCHORING, + "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_UNSATISFIABLE; + } } break; @@ -1113,6 +1191,8 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) set_flags(n, AST_FLAG_UNSATISFIABLE); } } else if (res != AST_ANALYSIS_OK) { + LOG(3 - LOG_ANCHORING, + "%s: CONCAT: got res of %d, bubbling up\n", __func__, res); return res; } @@ -1128,6 +1208,15 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming = 1; } + if (!env->followed_by_consuming_newline && + (child_env.followed_by_consuming_newline + || child->flags & AST_FLAG_MATCHES_1NEWLINE)) { + LOG(3 - LOG_ANCHORING, + "%s: setting followed_by_consuming_newline due to child %p's analysis\n", + __func__, (void *)child); + env->followed_by_consuming_newline = 1; + } + if (!env->before_start_anchor && child_env.before_start_anchor && !is_nullable(child)) { LOG(3 - LOG_ANCHORING, @@ -1169,6 +1258,10 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) all_set_followed_by_consuming &= child_env.followed_by_consuming; all_set_before_start_anchor &= child_env.before_start_anchor; any_sat = 1; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE + || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED_*, bubbling up\n", __func__); + return res; } else { return res; } diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index f9ec8ebbb..5390cce57 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -30,7 +30,9 @@ enum ast_analysis_res { AST_ANALYSIS_UNSATISFIABLE, AST_ANALYSIS_ERROR_NULL = -1, - AST_ANALYSIS_ERROR_MEMORY = -2 + AST_ANALYSIS_ERROR_MEMORY = -2, + AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE = -3, + AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE = -4 }; enum ast_analysis_res diff --git a/src/libre/print/tree.c b/src/libre/print/tree.c index 5d2f78691..ccf50cb23 100644 --- a/src/libre/print/tree.c +++ b/src/libre/print/tree.c @@ -62,6 +62,7 @@ fprintf_flags(FILE *f, enum ast_flags flags) PR_FLAG(END_NL, "N"); PR_FLAG(CAN_CONSUME, "c"); PR_FLAG(ALWAYS_CONSUMES, "C"); + PR_FLAG(MATCHES_1NEWLINE, "n"); #undef PR_FLAG diff --git a/src/libre/re.c b/src/libre/re.c index 6c423dc36..5e4604aa3 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -125,7 +125,15 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, if (res < 0) { ast_free(ast); - if (err != NULL) { err->e = RE_EERRNO; } + if (err != NULL) { + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + err->e = RE_EUNSUPPPCRE; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE) { + err->e = RE_EUNSUPCAPTUR; + } else if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + } + } return NULL; } diff --git a/src/libre/strerror.c b/src/libre/strerror.c index 009d61df2..d66e750a4 100644 --- a/src/libre/strerror.c +++ b/src/libre/strerror.c @@ -20,6 +20,8 @@ re_strerror(enum re_errno e) case RE_EERRNO: return strerror(errno); case RE_EBADDIALECT: return "Bad dialect"; case RE_EBADGROUP: return "Bad group"; + case RE_EUNSUPCAPTUR: return "Cannot support captures in this case"; + case RE_EUNSUPPPCRE: return "Unsupported PCRE edge case"; case RE_ENEGRANGE: return "Negative group range"; case RE_ENEGCOUNT: return "Negative count range"; diff --git a/tests/pcre/in48.re b/tests/pcre/in48.re new file mode 100644 index 000000000..15490c598 --- /dev/null +++ b/tests/pcre/in48.re @@ -0,0 +1 @@ +^a|$[^x]b* \ No newline at end of file diff --git a/tests/pcre/out48.err b/tests/pcre/out48.err new file mode 100644 index 000000000..b03d96d7e --- /dev/null +++ b/tests/pcre/out48.err @@ -0,0 +1 @@ +tests/pcre/in48.re: Unsupported PCRE edge case From de237ff5d20436eb2c70543a78fbb33e1982c80a Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Mon, 18 Sep 2023 10:57:31 +0500 Subject: [PATCH 053/373] Update to Unicode 15.1 --- share/ucd/CaseFolding.txt | 9 ++++++--- share/ucd/Makefile | 2 +- share/ucd/Scripts.txt | 14 ++++++++------ share/ucd/UnicodeData.txt | 7 +++++++ src/libre/class/utf8_Common.c | 4 ++-- src/libre/class/utf8_Han.c | 1 + src/libre/class/utf8_L.c | 2 ++ src/libre/class/utf8_Lo.c | 2 ++ src/libre/class/utf8_S.c | 3 ++- src/libre/class/utf8_So.c | 3 ++- src/libre/class/utf8_assigned.c | 7 ++++--- 11 files changed, 37 insertions(+), 17 deletions(-) diff --git a/share/ucd/CaseFolding.txt b/share/ucd/CaseFolding.txt index 65aa0fcd6..69c5c64b4 100644 --- a/share/ucd/CaseFolding.txt +++ b/share/ucd/CaseFolding.txt @@ -1,6 +1,6 @@ -# CaseFolding-15.0.0.txt -# Date: 2022-02-02, 23:35:35 GMT -# © 2022 Unicode®, Inc. +# CaseFolding-15.1.0.txt +# Date: 2023-05-12, 21:53:10 GMT +# © 2023 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see https://www.unicode.org/terms_of_use.html # @@ -929,6 +929,7 @@ 1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI 1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA 1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD3; S; 0390; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA 1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI 1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI 1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY @@ -937,6 +938,7 @@ 1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA 1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA 1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE3; S; 03B0; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA 1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI 1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI 1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI @@ -1328,6 +1330,7 @@ FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB05; S; FB06; # LATIN SMALL LIGATURE LONG S T FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH diff --git a/share/ucd/Makefile b/share/ucd/Makefile index 8af79f1b4..c5b9781d0 100644 --- a/share/ucd/Makefile +++ b/share/ucd/Makefile @@ -1,4 +1,4 @@ -UCD_URL ?= https://www.unicode.org/Public/15.0.0/ucd/ +UCD_URL ?= https://www.unicode.org/Public/15.1.0/ucd/ WGET ?= wget diff --git a/share/ucd/Scripts.txt b/share/ucd/Scripts.txt index 2b138bffb..0b3f717cb 100644 --- a/share/ucd/Scripts.txt +++ b/share/ucd/Scripts.txt @@ -1,6 +1,6 @@ -# Scripts-15.0.0.txt -# Date: 2022-04-26, 23:15:02 GMT -# © 2022 Unicode®, Inc. +# Scripts-15.1.0.txt +# Date: 2023-07-28, 16:01:07 GMT +# © 2023 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see https://www.unicode.org/terms_of_use.html # @@ -357,7 +357,7 @@ 2E5B ; Common # Ps BOTTOM HALF LEFT PARENTHESIS 2E5C ; Common # Pe BOTTOM HALF RIGHT PARENTHESIS 2E5D ; Common # Pd OBLIQUE HYPHEN -2FF0..2FFB ; Common # So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +2FF0..2FFF ; Common # So [16] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION 3000 ; Common # Zs IDEOGRAPHIC SPACE 3001..3003 ; Common # Po [3] IDEOGRAPHIC COMMA..DITTO MARK 3004 ; Common # So JAPANESE INDUSTRIAL STANDARD SYMBOL @@ -399,6 +399,7 @@ 3192..3195 ; Common # No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK 3196..319F ; Common # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK 31C0..31E3 ; Common # So [36] CJK STROKE T..CJK STROKE Q +31EF ; Common # So IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION 3220..3229 ; Common # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN 322A..3247 ; Common # So [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO 3248..324F ; Common # No [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE @@ -629,7 +630,7 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR E0001 ; Common # Cf LANGUAGE TAG E0020..E007F ; Common # Cf [96] TAG SPACE..CANCEL TAG -# Total code points: 8301 +# Total code points: 8306 # ================================================ @@ -1593,11 +1594,12 @@ FA70..FAD9 ; Han # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILI 2B740..2B81D ; Han # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D 2B820..2CEA1 ; Han # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 2CEB0..2EBE0 ; Han # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBF0..2EE5D ; Han # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D 2F800..2FA1D ; Han # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D 30000..3134A ; Han # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A 31350..323AF ; Han # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF -# Total code points: 98408 +# Total code points: 99030 # ================================================ diff --git a/share/ucd/UnicodeData.txt b/share/ucd/UnicodeData.txt index ea963a716..bdcc41850 100644 --- a/share/ucd/UnicodeData.txt +++ b/share/ucd/UnicodeData.txt @@ -11231,6 +11231,10 @@ 2FF9;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM UPPER RIGHT;So;0;ON;;;;;N;;;;; 2FFA;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER LEFT;So;0;ON;;;;;N;;;;; 2FFB;IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID;So;0;ON;;;;;N;;;;; +2FFC;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM RIGHT;So;0;ON;;;;;N;;;;; +2FFD;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER RIGHT;So;0;ON;;;;;N;;;;; +2FFE;IDEOGRAPHIC DESCRIPTION CHARACTER HORIZONTAL REFLECTION;So;0;ON;;;;;N;;;;; +2FFF;IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION;So;0;ON;;;;;N;;;;; 3000;IDEOGRAPHIC SPACE;Zs;0;WS; 0020;;;;N;;;;; 3001;IDEOGRAPHIC COMMA;Po;0;ON;;;;;N;;;;; 3002;IDEOGRAPHIC FULL STOP;Po;0;ON;;;;;N;IDEOGRAPHIC PERIOD;;;; @@ -11705,6 +11709,7 @@ 31E1;CJK STROKE HZZZG;So;0;ON;;;;;N;;;;; 31E2;CJK STROKE PG;So;0;ON;;;;;N;;;;; 31E3;CJK STROKE Q;So;0;ON;;;;;N;;;;; +31EF;IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION;So;0;ON;;;;;N;;;;; 31F0;KATAKANA LETTER SMALL KU;Lo;0;L;;;;;N;;;;; 31F1;KATAKANA LETTER SMALL SI;Lo;0;L;;;;;N;;;;; 31F2;KATAKANA LETTER SMALL SU;Lo;0;L;;;;;N;;;;; @@ -34035,6 +34040,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2CEA1;;Lo;0;L;;;;;N;;;;; 2CEB0;;Lo;0;L;;;;;N;;;;; 2EBE0;;Lo;0;L;;;;;N;;;;; +2EBF0;;Lo;0;L;;;;;N;;;;; +2EE5D;;Lo;0;L;;;;;N;;;;; 2F800;CJK COMPATIBILITY IDEOGRAPH-2F800;Lo;0;L;4E3D;;;;N;;;;; 2F801;CJK COMPATIBILITY IDEOGRAPH-2F801;Lo;0;L;4E38;;;;N;;;;; 2F802;CJK COMPATIBILITY IDEOGRAPH-2F802;Lo;0;L;4E41;;;;N;;;;; diff --git a/src/libre/class/utf8_Common.c b/src/libre/class/utf8_Common.c index 23e3cbf4b..a6ad8b58a 100644 --- a/src/libre/class/utf8_Common.c +++ b/src/libre/class/utf8_Common.c @@ -57,8 +57,7 @@ static const struct range ranges[] = { { 0x2B76UL, 0x2B95UL }, { 0x2B97UL, 0x2BFFUL }, { 0x2E00UL, 0x2E5DUL }, - { 0x2FF0UL, 0x2FFBUL }, - { 0x3000UL, 0x3004UL }, + { 0x2FF0UL, 0x3004UL }, { 0x3006UL, 0x3006UL }, { 0x3008UL, 0x3020UL }, { 0x3030UL, 0x3037UL }, @@ -68,6 +67,7 @@ static const struct range ranges[] = { { 0x30FBUL, 0x30FCUL }, { 0x3190UL, 0x319FUL }, { 0x31C0UL, 0x31E3UL }, + { 0x31EFUL, 0x31EFUL }, { 0x3220UL, 0x325FUL }, { 0x327FUL, 0x32CFUL }, { 0x32FFUL, 0x32FFUL }, diff --git a/src/libre/class/utf8_Han.c b/src/libre/class/utf8_Han.c index b22f0ed22..ce3bbce61 100644 --- a/src/libre/class/utf8_Han.c +++ b/src/libre/class/utf8_Han.c @@ -35,6 +35,7 @@ static const struct range ranges[] = { { 0x2B740UL, 0x2B81DUL }, { 0x2B820UL, 0x2CEA1UL }, { 0x2CEB0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x3134AUL }, { 0x31350UL, 0x323AFUL }, diff --git a/src/libre/class/utf8_L.c b/src/libre/class/utf8_L.c index 97eaa3ffe..9ecd7ba14 100644 --- a/src/libre/class/utf8_L.c +++ b/src/libre/class/utf8_L.c @@ -669,6 +669,8 @@ static const struct range ranges[] = { { 0x2CEA1UL, 0x2CEA1UL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EBF0UL }, + { 0x2EE5DUL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, diff --git a/src/libre/class/utf8_Lo.c b/src/libre/class/utf8_Lo.c index 2188ecd6f..c361d6892 100644 --- a/src/libre/class/utf8_Lo.c +++ b/src/libre/class/utf8_Lo.c @@ -520,6 +520,8 @@ static const struct range ranges[] = { { 0x2CEA1UL, 0x2CEA1UL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EBF0UL }, + { 0x2EE5DUL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, diff --git a/src/libre/class/utf8_S.c b/src/libre/class/utf8_S.c index 5a114bea5..1ad790595 100644 --- a/src/libre/class/utf8_S.c +++ b/src/libre/class/utf8_S.c @@ -110,7 +110,7 @@ static const struct range ranges[] = { { 0x2E80UL, 0x2E99UL }, { 0x2E9BUL, 0x2EF3UL }, { 0x2F00UL, 0x2FD5UL }, - { 0x2FF0UL, 0x2FFBUL }, + { 0x2FF0UL, 0x2FFFUL }, { 0x3004UL, 0x3004UL }, { 0x3012UL, 0x3013UL }, { 0x3020UL, 0x3020UL }, @@ -120,6 +120,7 @@ static const struct range ranges[] = { { 0x3190UL, 0x3191UL }, { 0x3196UL, 0x319FUL }, { 0x31C0UL, 0x31E3UL }, + { 0x31EFUL, 0x31EFUL }, { 0x3200UL, 0x321EUL }, { 0x322AUL, 0x3247UL }, { 0x3250UL, 0x3250UL }, diff --git a/src/libre/class/utf8_So.c b/src/libre/class/utf8_So.c index 2ddd4cd3e..3bb2b2f82 100644 --- a/src/libre/class/utf8_So.c +++ b/src/libre/class/utf8_So.c @@ -89,7 +89,7 @@ static const struct range ranges[] = { { 0x2E80UL, 0x2E99UL }, { 0x2E9BUL, 0x2EF3UL }, { 0x2F00UL, 0x2FD5UL }, - { 0x2FF0UL, 0x2FFBUL }, + { 0x2FF0UL, 0x2FFFUL }, { 0x3004UL, 0x3004UL }, { 0x3012UL, 0x3013UL }, { 0x3020UL, 0x3020UL }, @@ -98,6 +98,7 @@ static const struct range ranges[] = { { 0x3190UL, 0x3191UL }, { 0x3196UL, 0x319FUL }, { 0x31C0UL, 0x31E3UL }, + { 0x31EFUL, 0x31EFUL }, { 0x3200UL, 0x321EUL }, { 0x322AUL, 0x3247UL }, { 0x3250UL, 0x3250UL }, diff --git a/src/libre/class/utf8_assigned.c b/src/libre/class/utf8_assigned.c index bb764f63f..d29b53bf2 100644 --- a/src/libre/class/utf8_assigned.c +++ b/src/libre/class/utf8_assigned.c @@ -272,14 +272,13 @@ static const struct range ranges[] = { { 0x2E80UL, 0x2E99UL }, { 0x2E9BUL, 0x2EF3UL }, { 0x2F00UL, 0x2FD5UL }, - { 0x2FF0UL, 0x2FFBUL }, - { 0x3000UL, 0x303FUL }, + { 0x2FF0UL, 0x303FUL }, { 0x3041UL, 0x3096UL }, { 0x3099UL, 0x30FFUL }, { 0x3105UL, 0x312FUL }, { 0x3131UL, 0x318EUL }, { 0x3190UL, 0x31E3UL }, - { 0x31F0UL, 0x321EUL }, + { 0x31EFUL, 0x321EUL }, { 0x3220UL, 0x3400UL }, { 0x4DBFUL, 0x4E00UL }, { 0x9FFFUL, 0xA48CUL }, @@ -716,6 +715,8 @@ static const struct range ranges[] = { { 0x2CEA1UL, 0x2CEA1UL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EBF0UL }, + { 0x2EE5DUL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, From 450d4f490231b90754cfe729ea010c3767cece35 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 30 Oct 2023 15:06:50 +0000 Subject: [PATCH 054/373] Reject nodes with AST flags set for re_is_literal() One concession here is that I'm allowing `(?n)` because we don't currently indicate this in the AST. I'm also explicitly permitting `RE_END_NL`, because by the API contract this is the caller's responsibility. --- src/libre/ast.c | 22 ++++++++++++++++++++-- tests/re_literal/in13.re | 1 + tests/re_literal/in14.re | 1 + tests/re_literal/in15.re | 1 + tests/re_literal/in16.re | 1 + tests/re_literal/in17.re | 1 + tests/re_literal/in18.re | 1 + tests/re_literal/in19.re | 1 + tests/re_literal/in20.re | 1 + tests/re_literal/in21.re | 1 + tests/re_literal/in22.re | 1 + tests/re_literal/in23.re | 1 + tests/re_literal/in24.re | 1 + tests/re_literal/out13.txt | 1 + tests/re_literal/out14.txt | 1 + tests/re_literal/out15.txt | 2 ++ tests/re_literal/out16.txt | 1 + tests/re_literal/out17.txt | 1 + tests/re_literal/out18.txt | 1 + tests/re_literal/out19.txt | 2 ++ tests/re_literal/out20.txt | 1 + tests/re_literal/out21.txt | 2 ++ tests/re_literal/out22.txt | 1 + tests/re_literal/out23.txt | 1 + tests/re_literal/out24.txt | 2 ++ 25 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/re_literal/in13.re create mode 100644 tests/re_literal/in14.re create mode 100644 tests/re_literal/in15.re create mode 100644 tests/re_literal/in16.re create mode 100644 tests/re_literal/in17.re create mode 100644 tests/re_literal/in18.re create mode 100644 tests/re_literal/in19.re create mode 100644 tests/re_literal/in20.re create mode 100644 tests/re_literal/in21.re create mode 100644 tests/re_literal/in22.re create mode 100644 tests/re_literal/in23.re create mode 100644 tests/re_literal/in24.re create mode 100644 tests/re_literal/out13.txt create mode 100644 tests/re_literal/out14.txt create mode 100644 tests/re_literal/out15.txt create mode 100644 tests/re_literal/out16.txt create mode 100644 tests/re_literal/out17.txt create mode 100644 tests/re_literal/out18.txt create mode 100644 tests/re_literal/out19.txt create mode 100644 tests/re_literal/out20.txt create mode 100644 tests/re_literal/out21.txt create mode 100644 tests/re_literal/out22.txt create mode 100644 tests/re_literal/out23.txt create mode 100644 tests/re_literal/out24.txt diff --git a/src/libre/ast.c b/src/libre/ast.c index 2e1d21817..82c52a790 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -997,13 +997,31 @@ ast_expr_is_literal(const struct ast_expr *e, *anchor_end = 1; } +done: + for (i = 0; i < count; i++) { if (nodes[i] == NULL || nodes[i]->type != AST_EXPR_LITERAL) { return 0; } - } -done: + /* mask out AST flags that we don't care about; these may be present or not */ + int ast_flags = nodes[i]->flags + & ~(AST_FLAG_FIRST | AST_FLAG_LAST | AST_FLAG_ALWAYS_CONSUMES | AST_FLAG_CAN_CONSUME); + + /* we reject anything else */ + if (ast_flags != 0) { + return 0; + } + + /* mask out re flags that we don't handle here yet (the caller is responsible for these) */ + int re_flags = nodes[i]->re_flags + & ~(RE_END_NL); + + /* and we don't permit any other re_flags */ + if (re_flags != 0) { + return 0; + } + } *n = count + is_end_nl; diff --git a/tests/re_literal/in13.re b/tests/re_literal/in13.re new file mode 100644 index 000000000..fa9f7f5ff --- /dev/null +++ b/tests/re_literal/in13.re @@ -0,0 +1 @@ +^(?i)a$ \ No newline at end of file diff --git a/tests/re_literal/in14.re b/tests/re_literal/in14.re new file mode 100644 index 000000000..2500582f0 --- /dev/null +++ b/tests/re_literal/in14.re @@ -0,0 +1 @@ +(?:x(?i)a) \ No newline at end of file diff --git a/tests/re_literal/in15.re b/tests/re_literal/in15.re new file mode 100644 index 000000000..27e49d98b --- /dev/null +++ b/tests/re_literal/in15.re @@ -0,0 +1 @@ +(?i) \ No newline at end of file diff --git a/tests/re_literal/in16.re b/tests/re_literal/in16.re new file mode 100644 index 000000000..33e13db9d --- /dev/null +++ b/tests/re_literal/in16.re @@ -0,0 +1 @@ +(?i)xy \ No newline at end of file diff --git a/tests/re_literal/in17.re b/tests/re_literal/in17.re new file mode 100644 index 000000000..92e7f56a7 --- /dev/null +++ b/tests/re_literal/in17.re @@ -0,0 +1 @@ +a(?i)xy \ No newline at end of file diff --git a/tests/re_literal/in18.re b/tests/re_literal/in18.re new file mode 100644 index 000000000..9fe02bd25 --- /dev/null +++ b/tests/re_literal/in18.re @@ -0,0 +1 @@ +a(?:x(?i)xy) \ No newline at end of file diff --git a/tests/re_literal/in19.re b/tests/re_literal/in19.re new file mode 100644 index 000000000..297e030e4 --- /dev/null +++ b/tests/re_literal/in19.re @@ -0,0 +1 @@ +a(?:x(?-i)xy) \ No newline at end of file diff --git a/tests/re_literal/in20.re b/tests/re_literal/in20.re new file mode 100644 index 000000000..800e172fd --- /dev/null +++ b/tests/re_literal/in20.re @@ -0,0 +1 @@ +(?i)(?:(?-i)a)(?:x(?-i)xy) \ No newline at end of file diff --git a/tests/re_literal/in21.re b/tests/re_literal/in21.re new file mode 100644 index 000000000..4b3072ef2 --- /dev/null +++ b/tests/re_literal/in21.re @@ -0,0 +1 @@ +(?i)(?:(?-i)a)(?:(?-i)x(?-i)xy) \ No newline at end of file diff --git a/tests/re_literal/in22.re b/tests/re_literal/in22.re new file mode 100644 index 000000000..40352311b --- /dev/null +++ b/tests/re_literal/in22.re @@ -0,0 +1 @@ +(?s)a \ No newline at end of file diff --git a/tests/re_literal/in23.re b/tests/re_literal/in23.re new file mode 100644 index 000000000..f53b1b835 --- /dev/null +++ b/tests/re_literal/in23.re @@ -0,0 +1 @@ +(?x)a \ No newline at end of file diff --git a/tests/re_literal/in24.re b/tests/re_literal/in24.re new file mode 100644 index 000000000..bda0b6e86 --- /dev/null +++ b/tests/re_literal/in24.re @@ -0,0 +1 @@ +(?n)a \ No newline at end of file diff --git a/tests/re_literal/out13.txt b/tests/re_literal/out13.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out13.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out14.txt b/tests/re_literal/out14.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out14.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out15.txt b/tests/re_literal/out15.txt new file mode 100644 index 000000000..82977dfb2 --- /dev/null +++ b/tests/re_literal/out15.txt @@ -0,0 +1,2 @@ +anchors: +literal: diff --git a/tests/re_literal/out16.txt b/tests/re_literal/out16.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out16.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out17.txt b/tests/re_literal/out17.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out17.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out18.txt b/tests/re_literal/out18.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out18.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out19.txt b/tests/re_literal/out19.txt new file mode 100644 index 000000000..f761b32c2 --- /dev/null +++ b/tests/re_literal/out19.txt @@ -0,0 +1,2 @@ +anchors: +literal: axxy diff --git a/tests/re_literal/out20.txt b/tests/re_literal/out20.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out20.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out21.txt b/tests/re_literal/out21.txt new file mode 100644 index 000000000..f761b32c2 --- /dev/null +++ b/tests/re_literal/out21.txt @@ -0,0 +1,2 @@ +anchors: +literal: axxy diff --git a/tests/re_literal/out22.txt b/tests/re_literal/out22.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out22.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out23.txt b/tests/re_literal/out23.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out23.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out24.txt b/tests/re_literal/out24.txt new file mode 100644 index 000000000..e9278cb7d --- /dev/null +++ b/tests/re_literal/out24.txt @@ -0,0 +1,2 @@ +anchors: +literal: a From 9653595a2e6a9d35023514e9680140cac767e276 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 27 Nov 2023 16:58:41 -0500 Subject: [PATCH 055/373] tests/pcre-anchor: Add regression tests 82-86. These are anchoring edge cases found by fuzzing. They have to do with incomplete pruning of repeated nodes that are unsatisfiable due to anchoring, nesting of anchors inside groups and/or ALT subtrees, etc. The fixes will go in their own commit. --- tests/pcre-anchor/in82.re | 1 + tests/pcre-anchor/in83.re | 1 + tests/pcre-anchor/in84.re | 1 + tests/pcre-anchor/in85.re | 1 + tests/pcre-anchor/in86.re | 1 + tests/pcre-anchor/out82.fsm | 5 +++++ tests/pcre-anchor/out83.fsm | 5 +++++ tests/pcre-anchor/out84.fsm | 5 +++++ tests/pcre-anchor/out85.fsm | 4 ++++ tests/pcre-anchor/out86.fsm | 5 +++++ 10 files changed, 29 insertions(+) create mode 100644 tests/pcre-anchor/in82.re create mode 100644 tests/pcre-anchor/in83.re create mode 100644 tests/pcre-anchor/in84.re create mode 100644 tests/pcre-anchor/in85.re create mode 100644 tests/pcre-anchor/in86.re create mode 100644 tests/pcre-anchor/out82.fsm create mode 100644 tests/pcre-anchor/out83.fsm create mode 100644 tests/pcre-anchor/out84.fsm create mode 100644 tests/pcre-anchor/out85.fsm create mode 100644 tests/pcre-anchor/out86.fsm diff --git a/tests/pcre-anchor/in82.re b/tests/pcre-anchor/in82.re new file mode 100644 index 000000000..60c5f181f --- /dev/null +++ b/tests/pcre-anchor/in82.re @@ -0,0 +1 @@ +(^)+a \ No newline at end of file diff --git a/tests/pcre-anchor/in83.re b/tests/pcre-anchor/in83.re new file mode 100644 index 000000000..ef122c481 --- /dev/null +++ b/tests/pcre-anchor/in83.re @@ -0,0 +1 @@ +(?:a|^)*^x \ No newline at end of file diff --git a/tests/pcre-anchor/in84.re b/tests/pcre-anchor/in84.re new file mode 100644 index 000000000..3aac549f7 --- /dev/null +++ b/tests/pcre-anchor/in84.re @@ -0,0 +1 @@ +(?:(?:$|^)a) \ No newline at end of file diff --git a/tests/pcre-anchor/in85.re b/tests/pcre-anchor/in85.re new file mode 100644 index 000000000..d72dc5d4b --- /dev/null +++ b/tests/pcre-anchor/in85.re @@ -0,0 +1 @@ +$(?:a|^)b? \ No newline at end of file diff --git a/tests/pcre-anchor/in86.re b/tests/pcre-anchor/in86.re new file mode 100644 index 000000000..b288d3dc3 --- /dev/null +++ b/tests/pcre-anchor/in86.re @@ -0,0 +1 @@ +(?:(?:^a)*|b^)^c \ No newline at end of file diff --git a/tests/pcre-anchor/out82.fsm b/tests/pcre-anchor/out82.fsm new file mode 100644 index 000000000..84503d994 --- /dev/null +++ b/tests/pcre-anchor/out82.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out83.fsm b/tests/pcre-anchor/out83.fsm new file mode 100644 index 000000000..28643e39e --- /dev/null +++ b/tests/pcre-anchor/out83.fsm @@ -0,0 +1,5 @@ +0 -> 1 "x"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out84.fsm b/tests/pcre-anchor/out84.fsm new file mode 100644 index 000000000..84503d994 --- /dev/null +++ b/tests/pcre-anchor/out84.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out85.fsm b/tests/pcre-anchor/out85.fsm new file mode 100644 index 000000000..9b7e7aba6 --- /dev/null +++ b/tests/pcre-anchor/out85.fsm @@ -0,0 +1,4 @@ +0 -> 1 "\n"; + +start: 0; +end: 0, 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out86.fsm b/tests/pcre-anchor/out86.fsm new file mode 100644 index 000000000..4bcad67dc --- /dev/null +++ b/tests/pcre-anchor/out86.fsm @@ -0,0 +1,5 @@ +0 -> 1 "c"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file From 341b38039606e9f00fc7e888d625df5302adf033 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 24 Oct 2023 17:11:21 -0400 Subject: [PATCH 056/373] tests/pcre-anchor/in87.re: bubble up anchor flags to group node. --- src/libre/ast_analysis.c | 9 +++++++++ tests/pcre-anchor/in87.re | 1 + tests/pcre-anchor/out87.fsm | 6 ++++++ 3 files changed, 16 insertions(+) create mode 100644 tests/pcre-anchor/in87.re create mode 100644 tests/pcre-anchor/out87.fsm diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 3298f62c2..7f7a50c6f 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -36,6 +36,7 @@ /* Mask for end-anchor flags */ #define END_ANCHOR_FLAG_MASK (AST_FLAG_ANCHORED_END | AST_FLAG_END_NL) +#define ANCHOR_FLAG_MASK (END_ANCHOR_FLAG_MASK | AST_FLAG_ANCHORED_START) static int is_nullable(const struct ast_expr *n) @@ -1231,6 +1232,14 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) } return res; } + + if (n->u.group.e->flags & ANCHOR_FLAG_MASK) { + LOG(3 - LOG_ANCHORING, + "%s: bubbling up anchoring flags from %p to %p\n", + __func__, (void *)n->u.group.e, (void *)n); + set_flags(n, n->u.group.e->flags & ANCHOR_FLAG_MASK); + } + break; case AST_EXPR_SUBTRACT: diff --git a/tests/pcre-anchor/in87.re b/tests/pcre-anchor/in87.re new file mode 100644 index 000000000..a8006a0fe --- /dev/null +++ b/tests/pcre-anchor/in87.re @@ -0,0 +1 @@ +(($|^)a)b \ No newline at end of file diff --git a/tests/pcre-anchor/out87.fsm b/tests/pcre-anchor/out87.fsm new file mode 100644 index 000000000..6f6aa823b --- /dev/null +++ b/tests/pcre-anchor/out87.fsm @@ -0,0 +1,6 @@ +0 -> 1 "a"; +1 -> 2 "b"; +2 -> 2 ?; + +start: 0; +end: 2; \ No newline at end of file From 1bb18fcf401789b09e19e4e3b648c601706faa1e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 27 Oct 2023 11:49:06 -0400 Subject: [PATCH 057/373] More pcre and pcre-anchor test cases from fuzzing. --- tests/pcre-anchor/in88.re | 1 + tests/pcre-anchor/in89.re | 1 + tests/pcre-anchor/in90.re | 1 + tests/pcre-anchor/in91.re | 1 + tests/pcre-anchor/out88.fsm | 4 ++++ tests/pcre-anchor/out89.fsm | 4 ++++ tests/pcre-anchor/out90.fsm | 4 ++++ tests/pcre-anchor/out91.fsm | 4 ++++ tests/pcre/in49.re | 1 + tests/pcre/out49.err | 1 + 10 files changed, 22 insertions(+) create mode 100644 tests/pcre-anchor/in88.re create mode 100644 tests/pcre-anchor/in89.re create mode 100644 tests/pcre-anchor/in90.re create mode 100644 tests/pcre-anchor/in91.re create mode 100644 tests/pcre-anchor/out88.fsm create mode 100644 tests/pcre-anchor/out89.fsm create mode 100644 tests/pcre-anchor/out90.fsm create mode 100644 tests/pcre-anchor/out91.fsm create mode 100644 tests/pcre/in49.re create mode 100644 tests/pcre/out49.err diff --git a/tests/pcre-anchor/in88.re b/tests/pcre-anchor/in88.re new file mode 100644 index 000000000..3d2764532 --- /dev/null +++ b/tests/pcre-anchor/in88.re @@ -0,0 +1 @@ +(a$b)*|.^ \ No newline at end of file diff --git a/tests/pcre-anchor/in89.re b/tests/pcre-anchor/in89.re new file mode 100644 index 000000000..bf51de7e9 --- /dev/null +++ b/tests/pcre-anchor/in89.re @@ -0,0 +1 @@ +$.| \ No newline at end of file diff --git a/tests/pcre-anchor/in90.re b/tests/pcre-anchor/in90.re new file mode 100644 index 000000000..519e2e283 --- /dev/null +++ b/tests/pcre-anchor/in90.re @@ -0,0 +1 @@ +.^| \ No newline at end of file diff --git a/tests/pcre-anchor/in91.re b/tests/pcre-anchor/in91.re new file mode 100644 index 000000000..3ce75c157 --- /dev/null +++ b/tests/pcre-anchor/in91.re @@ -0,0 +1 @@ +$.^| \ No newline at end of file diff --git a/tests/pcre-anchor/out88.fsm b/tests/pcre-anchor/out88.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out88.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out89.fsm b/tests/pcre-anchor/out89.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out89.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out90.fsm b/tests/pcre-anchor/out90.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out90.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out91.fsm b/tests/pcre-anchor/out91.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out91.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre/in49.re b/tests/pcre/in49.re new file mode 100644 index 000000000..07a2d0949 --- /dev/null +++ b/tests/pcre/in49.re @@ -0,0 +1 @@ +(?:(?:|^a)+|^)^b \ No newline at end of file diff --git a/tests/pcre/out49.err b/tests/pcre/out49.err new file mode 100644 index 000000000..d22e4cd6d --- /dev/null +++ b/tests/pcre/out49.err @@ -0,0 +1 @@ +tests/pcre/in49.re: Unsupported PCRE edge case From 120f9c53ac96ea5eb5512b26fd9b729e2c578390 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 7 Nov 2023 15:57:05 -0500 Subject: [PATCH 058/373] new pcre-anchor tests --- tests/pcre-anchor/in92.re | 1 + tests/pcre-anchor/in93.re | 1 + tests/pcre-anchor/out92.fsm | 19 +++++++++++++++++++ tests/pcre-anchor/out93.fsm | 18 ++++++++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 tests/pcre-anchor/in92.re create mode 100644 tests/pcre-anchor/in93.re create mode 100644 tests/pcre-anchor/out92.fsm create mode 100644 tests/pcre-anchor/out93.fsm diff --git a/tests/pcre-anchor/in92.re b/tests/pcre-anchor/in92.re new file mode 100644 index 000000000..e32c3a5f5 --- /dev/null +++ b/tests/pcre-anchor/in92.re @@ -0,0 +1 @@ +(a|$)(b|^) \ No newline at end of file diff --git a/tests/pcre-anchor/in93.re b/tests/pcre-anchor/in93.re new file mode 100644 index 000000000..228f2e8e0 --- /dev/null +++ b/tests/pcre-anchor/in93.re @@ -0,0 +1 @@ +(a|^)(b|$) \ No newline at end of file diff --git a/tests/pcre-anchor/out92.fsm b/tests/pcre-anchor/out92.fsm new file mode 100644 index 000000000..e2aafdeb6 --- /dev/null +++ b/tests/pcre-anchor/out92.fsm @@ -0,0 +1,19 @@ +# This is a tricky one -- with '(a|$)(b|^)' the ^ must not be linked to +# the global start or have an unanchored end loop after, because by the +# time execution gets there it has already either matched some input +# ('a') or anchored to the end -- the only matching inputs should be "", +# "\n", and "ab", but "a^", "$b", or anything else +# that does not contain "ab" must not match. + +0 -> 1; +0 -> 4 "\n"; + +1 -> 1 ?; +1 -> 2 "a"; + +2 -> 3 "b"; + +3 -> 3 ?; + +start: 0; +end: 0, 3, 4; diff --git a/tests/pcre-anchor/out93.fsm b/tests/pcre-anchor/out93.fsm new file mode 100644 index 000000000..aed23cab9 --- /dev/null +++ b/tests/pcre-anchor/out93.fsm @@ -0,0 +1,18 @@ +# /(a|^)(b|$)/ => "", "\n", "a" "b "ab" + +0 -> 1; +0 -> 3; + +1 -> 1 ?; +1 -> 2 "a"; + +2 -> 4 "b"; +2 -> 5 "\n"; + +3 -> 4 "b"; +3 -> 5 "\n"; + +4 -> 4 ?; + +start: 0; +end: 2, 3, 4, 5; From 20cb71a8eb573d54de10cccdf979b5e36eeb3e96 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 27 Nov 2023 17:01:27 -0500 Subject: [PATCH 059/373] pcre-anchor/in94.re: One more test from fuzzing. --- tests/pcre-anchor/in94.re | 1 + tests/pcre-anchor/out94.fsm | 525 ++++++++++++++++++++++++++++++++++++ 2 files changed, 526 insertions(+) create mode 100644 tests/pcre-anchor/in94.re create mode 100644 tests/pcre-anchor/out94.fsm diff --git a/tests/pcre-anchor/in94.re b/tests/pcre-anchor/in94.re new file mode 100644 index 000000000..349808e15 --- /dev/null +++ b/tests/pcre-anchor/in94.re @@ -0,0 +1 @@ +.$|b. \ No newline at end of file diff --git a/tests/pcre-anchor/out94.fsm b/tests/pcre-anchor/out94.fsm new file mode 100644 index 000000000..53d73459c --- /dev/null +++ b/tests/pcre-anchor/out94.fsm @@ -0,0 +1,525 @@ +0 -> 1; +0 -> 4; +2 -> 3; +4 -> 5; +4 -> 4 ?; +5 -> 9 "\x00"; +5 -> 9 "\x01"; +5 -> 9 "\x02"; +5 -> 9 "\x03"; +5 -> 9 "\x04"; +5 -> 9 "\x05"; +5 -> 9 "\x06"; +5 -> 9 "\x07"; +5 -> 9 "\x08"; +5 -> 9 "\t"; +5 -> 9 "\v"; +5 -> 9 "\f"; +5 -> 9 "\r"; +5 -> 9 "\x0e"; +5 -> 9 "\x0f"; +5 -> 9 "\x10"; +5 -> 9 "\x11"; +5 -> 9 "\x12"; +5 -> 9 "\x13"; +5 -> 9 "\x14"; +5 -> 9 "\x15"; +5 -> 9 "\x16"; +5 -> 9 "\x17"; +5 -> 9 "\x18"; +5 -> 9 "\x19"; +5 -> 9 "\x1a"; +5 -> 9 "\x1b"; +5 -> 9 "\x1c"; +5 -> 9 "\x1d"; +5 -> 9 "\x1e"; +5 -> 9 "\x1f"; +5 -> 9 " "; +5 -> 9 "!"; +5 -> 9 "\""; +5 -> 9 "#"; +5 -> 9 "$"; +5 -> 9 "%"; +5 -> 9 "&"; +5 -> 9 "'"; +5 -> 9 "("; +5 -> 9 ")"; +5 -> 9 "*"; +5 -> 9 "+"; +5 -> 9 ","; +5 -> 9 "-"; +5 -> 9 "."; +5 -> 9 "/"; +5 -> 9 "0"; +5 -> 9 "1"; +5 -> 9 "2"; +5 -> 9 "3"; +5 -> 9 "4"; +5 -> 9 "5"; +5 -> 9 "6"; +5 -> 9 "7"; +5 -> 9 "8"; +5 -> 9 "9"; +5 -> 9 ":"; +5 -> 9 ";"; +5 -> 9 "<"; +5 -> 9 "="; +5 -> 9 ">"; +5 -> 9 "?"; +5 -> 9 "@"; +5 -> 9 "A"; +5 -> 9 "B"; +5 -> 9 "C"; +5 -> 9 "D"; +5 -> 9 "E"; +5 -> 9 "F"; +5 -> 9 "G"; +5 -> 9 "H"; +5 -> 9 "I"; +5 -> 9 "J"; +5 -> 9 "K"; +5 -> 9 "L"; +5 -> 9 "M"; +5 -> 9 "N"; +5 -> 9 "O"; +5 -> 9 "P"; +5 -> 9 "Q"; +5 -> 9 "R"; +5 -> 9 "S"; +5 -> 9 "T"; +5 -> 9 "U"; +5 -> 9 "V"; +5 -> 9 "W"; +5 -> 9 "X"; +5 -> 9 "Y"; +5 -> 9 "Z"; +5 -> 9 "["; +5 -> 9 "\\"; +5 -> 9 "]"; +5 -> 9 "^"; +5 -> 9 "_"; +5 -> 9 "`"; +5 -> 9 "a"; +5 -> 9 "b"; +5 -> 10 "b"; +5 -> 9 "c"; +5 -> 9 "d"; +5 -> 9 "e"; +5 -> 9 "f"; +5 -> 9 "g"; +5 -> 9 "h"; +5 -> 9 "i"; +5 -> 9 "j"; +5 -> 9 "k"; +5 -> 9 "l"; +5 -> 9 "m"; +5 -> 9 "n"; +5 -> 9 "o"; +5 -> 9 "p"; +5 -> 9 "q"; +5 -> 9 "r"; +5 -> 9 "s"; +5 -> 9 "t"; +5 -> 9 "u"; +5 -> 9 "v"; +5 -> 9 "w"; +5 -> 9 "x"; +5 -> 9 "y"; +5 -> 9 "z"; +5 -> 9 "{"; +5 -> 9 "|"; +5 -> 9 "}"; +5 -> 9 "~"; +5 -> 9 "\x7f"; +5 -> 9 "\x80"; +5 -> 9 "\x81"; +5 -> 9 "\x82"; +5 -> 9 "\x83"; +5 -> 9 "\x84"; +5 -> 9 "\x85"; +5 -> 9 "\x86"; +5 -> 9 "\x87"; +5 -> 9 "\x88"; +5 -> 9 "\x89"; +5 -> 9 "\x8a"; +5 -> 9 "\x8b"; +5 -> 9 "\x8c"; +5 -> 9 "\x8d"; +5 -> 9 "\x8e"; +5 -> 9 "\x8f"; +5 -> 9 "\x90"; +5 -> 9 "\x91"; +5 -> 9 "\x92"; +5 -> 9 "\x93"; +5 -> 9 "\x94"; +5 -> 9 "\x95"; +5 -> 9 "\x96"; +5 -> 9 "\x97"; +5 -> 9 "\x98"; +5 -> 9 "\x99"; +5 -> 9 "\x9a"; +5 -> 9 "\x9b"; +5 -> 9 "\x9c"; +5 -> 9 "\x9d"; +5 -> 9 "\x9e"; +5 -> 9 "\x9f"; +5 -> 9 "\xa0"; +5 -> 9 "\xa1"; +5 -> 9 "\xa2"; +5 -> 9 "\xa3"; +5 -> 9 "\xa4"; +5 -> 9 "\xa5"; +5 -> 9 "\xa6"; +5 -> 9 "\xa7"; +5 -> 9 "\xa8"; +5 -> 9 "\xa9"; +5 -> 9 "\xaa"; +5 -> 9 "\xab"; +5 -> 9 "\xac"; +5 -> 9 "\xad"; +5 -> 9 "\xae"; +5 -> 9 "\xaf"; +5 -> 9 "\xb0"; +5 -> 9 "\xb1"; +5 -> 9 "\xb2"; +5 -> 9 "\xb3"; +5 -> 9 "\xb4"; +5 -> 9 "\xb5"; +5 -> 9 "\xb6"; +5 -> 9 "\xb7"; +5 -> 9 "\xb8"; +5 -> 9 "\xb9"; +5 -> 9 "\xba"; +5 -> 9 "\xbb"; +5 -> 9 "\xbc"; +5 -> 9 "\xbd"; +5 -> 9 "\xbe"; +5 -> 9 "\xbf"; +5 -> 9 "\xc0"; +5 -> 9 "\xc1"; +5 -> 9 "\xc2"; +5 -> 9 "\xc3"; +5 -> 9 "\xc4"; +5 -> 9 "\xc5"; +5 -> 9 "\xc6"; +5 -> 9 "\xc7"; +5 -> 9 "\xc8"; +5 -> 9 "\xc9"; +5 -> 9 "\xca"; +5 -> 9 "\xcb"; +5 -> 9 "\xcc"; +5 -> 9 "\xcd"; +5 -> 9 "\xce"; +5 -> 9 "\xcf"; +5 -> 9 "\xd0"; +5 -> 9 "\xd1"; +5 -> 9 "\xd2"; +5 -> 9 "\xd3"; +5 -> 9 "\xd4"; +5 -> 9 "\xd5"; +5 -> 9 "\xd6"; +5 -> 9 "\xd7"; +5 -> 9 "\xd8"; +5 -> 9 "\xd9"; +5 -> 9 "\xda"; +5 -> 9 "\xdb"; +5 -> 9 "\xdc"; +5 -> 9 "\xdd"; +5 -> 9 "\xde"; +5 -> 9 "\xdf"; +5 -> 9 "\xe0"; +5 -> 9 "\xe1"; +5 -> 9 "\xe2"; +5 -> 9 "\xe3"; +5 -> 9 "\xe4"; +5 -> 9 "\xe5"; +5 -> 9 "\xe6"; +5 -> 9 "\xe7"; +5 -> 9 "\xe8"; +5 -> 9 "\xe9"; +5 -> 9 "\xea"; +5 -> 9 "\xeb"; +5 -> 9 "\xec"; +5 -> 9 "\xed"; +5 -> 9 "\xee"; +5 -> 9 "\xef"; +5 -> 9 "\xf0"; +5 -> 9 "\xf1"; +5 -> 9 "\xf2"; +5 -> 9 "\xf3"; +5 -> 9 "\xf4"; +5 -> 9 "\xf5"; +5 -> 9 "\xf6"; +5 -> 9 "\xf7"; +5 -> 9 "\xf8"; +5 -> 9 "\xf9"; +5 -> 9 "\xfa"; +5 -> 9 "\xfb"; +5 -> 9 "\xfc"; +5 -> 9 "\xfd"; +5 -> 9 "\xfe"; +5 -> 9 "\xff"; +6 -> 3; +6 -> 6 ?; +7 -> 6; +8 -> 3; +8 -> 3 "\n"; +9 -> 8; +10 -> 7 "\x00"; +10 -> 7 "\x01"; +10 -> 7 "\x02"; +10 -> 7 "\x03"; +10 -> 7 "\x04"; +10 -> 7 "\x05"; +10 -> 7 "\x06"; +10 -> 7 "\x07"; +10 -> 7 "\x08"; +10 -> 7 "\t"; +10 -> 7 "\v"; +10 -> 7 "\f"; +10 -> 7 "\r"; +10 -> 7 "\x0e"; +10 -> 7 "\x0f"; +10 -> 7 "\x10"; +10 -> 7 "\x11"; +10 -> 7 "\x12"; +10 -> 7 "\x13"; +10 -> 7 "\x14"; +10 -> 7 "\x15"; +10 -> 7 "\x16"; +10 -> 7 "\x17"; +10 -> 7 "\x18"; +10 -> 7 "\x19"; +10 -> 7 "\x1a"; +10 -> 7 "\x1b"; +10 -> 7 "\x1c"; +10 -> 7 "\x1d"; +10 -> 7 "\x1e"; +10 -> 7 "\x1f"; +10 -> 7 " "; +10 -> 7 "!"; +10 -> 7 "\""; +10 -> 7 "#"; +10 -> 7 "$"; +10 -> 7 "%"; +10 -> 7 "&"; +10 -> 7 "'"; +10 -> 7 "("; +10 -> 7 ")"; +10 -> 7 "*"; +10 -> 7 "+"; +10 -> 7 ","; +10 -> 7 "-"; +10 -> 7 "."; +10 -> 7 "/"; +10 -> 7 "0"; +10 -> 7 "1"; +10 -> 7 "2"; +10 -> 7 "3"; +10 -> 7 "4"; +10 -> 7 "5"; +10 -> 7 "6"; +10 -> 7 "7"; +10 -> 7 "8"; +10 -> 7 "9"; +10 -> 7 ":"; +10 -> 7 ";"; +10 -> 7 "<"; +10 -> 7 "="; +10 -> 7 ">"; +10 -> 7 "?"; +10 -> 7 "@"; +10 -> 7 "A"; +10 -> 7 "B"; +10 -> 7 "C"; +10 -> 7 "D"; +10 -> 7 "E"; +10 -> 7 "F"; +10 -> 7 "G"; +10 -> 7 "H"; +10 -> 7 "I"; +10 -> 7 "J"; +10 -> 7 "K"; +10 -> 7 "L"; +10 -> 7 "M"; +10 -> 7 "N"; +10 -> 7 "O"; +10 -> 7 "P"; +10 -> 7 "Q"; +10 -> 7 "R"; +10 -> 7 "S"; +10 -> 7 "T"; +10 -> 7 "U"; +10 -> 7 "V"; +10 -> 7 "W"; +10 -> 7 "X"; +10 -> 7 "Y"; +10 -> 7 "Z"; +10 -> 7 "["; +10 -> 7 "\\"; +10 -> 7 "]"; +10 -> 7 "^"; +10 -> 7 "_"; +10 -> 7 "`"; +10 -> 7 "a"; +10 -> 7 "b"; +10 -> 7 "c"; +10 -> 7 "d"; +10 -> 7 "e"; +10 -> 7 "f"; +10 -> 7 "g"; +10 -> 7 "h"; +10 -> 7 "i"; +10 -> 7 "j"; +10 -> 7 "k"; +10 -> 7 "l"; +10 -> 7 "m"; +10 -> 7 "n"; +10 -> 7 "o"; +10 -> 7 "p"; +10 -> 7 "q"; +10 -> 7 "r"; +10 -> 7 "s"; +10 -> 7 "t"; +10 -> 7 "u"; +10 -> 7 "v"; +10 -> 7 "w"; +10 -> 7 "x"; +10 -> 7 "y"; +10 -> 7 "z"; +10 -> 7 "{"; +10 -> 7 "|"; +10 -> 7 "}"; +10 -> 7 "~"; +10 -> 7 "\x7f"; +10 -> 7 "\x80"; +10 -> 7 "\x81"; +10 -> 7 "\x82"; +10 -> 7 "\x83"; +10 -> 7 "\x84"; +10 -> 7 "\x85"; +10 -> 7 "\x86"; +10 -> 7 "\x87"; +10 -> 7 "\x88"; +10 -> 7 "\x89"; +10 -> 7 "\x8a"; +10 -> 7 "\x8b"; +10 -> 7 "\x8c"; +10 -> 7 "\x8d"; +10 -> 7 "\x8e"; +10 -> 7 "\x8f"; +10 -> 7 "\x90"; +10 -> 7 "\x91"; +10 -> 7 "\x92"; +10 -> 7 "\x93"; +10 -> 7 "\x94"; +10 -> 7 "\x95"; +10 -> 7 "\x96"; +10 -> 7 "\x97"; +10 -> 7 "\x98"; +10 -> 7 "\x99"; +10 -> 7 "\x9a"; +10 -> 7 "\x9b"; +10 -> 7 "\x9c"; +10 -> 7 "\x9d"; +10 -> 7 "\x9e"; +10 -> 7 "\x9f"; +10 -> 7 "\xa0"; +10 -> 7 "\xa1"; +10 -> 7 "\xa2"; +10 -> 7 "\xa3"; +10 -> 7 "\xa4"; +10 -> 7 "\xa5"; +10 -> 7 "\xa6"; +10 -> 7 "\xa7"; +10 -> 7 "\xa8"; +10 -> 7 "\xa9"; +10 -> 7 "\xaa"; +10 -> 7 "\xab"; +10 -> 7 "\xac"; +10 -> 7 "\xad"; +10 -> 7 "\xae"; +10 -> 7 "\xaf"; +10 -> 7 "\xb0"; +10 -> 7 "\xb1"; +10 -> 7 "\xb2"; +10 -> 7 "\xb3"; +10 -> 7 "\xb4"; +10 -> 7 "\xb5"; +10 -> 7 "\xb6"; +10 -> 7 "\xb7"; +10 -> 7 "\xb8"; +10 -> 7 "\xb9"; +10 -> 7 "\xba"; +10 -> 7 "\xbb"; +10 -> 7 "\xbc"; +10 -> 7 "\xbd"; +10 -> 7 "\xbe"; +10 -> 7 "\xbf"; +10 -> 7 "\xc0"; +10 -> 7 "\xc1"; +10 -> 7 "\xc2"; +10 -> 7 "\xc3"; +10 -> 7 "\xc4"; +10 -> 7 "\xc5"; +10 -> 7 "\xc6"; +10 -> 7 "\xc7"; +10 -> 7 "\xc8"; +10 -> 7 "\xc9"; +10 -> 7 "\xca"; +10 -> 7 "\xcb"; +10 -> 7 "\xcc"; +10 -> 7 "\xcd"; +10 -> 7 "\xce"; +10 -> 7 "\xcf"; +10 -> 7 "\xd0"; +10 -> 7 "\xd1"; +10 -> 7 "\xd2"; +10 -> 7 "\xd3"; +10 -> 7 "\xd4"; +10 -> 7 "\xd5"; +10 -> 7 "\xd6"; +10 -> 7 "\xd7"; +10 -> 7 "\xd8"; +10 -> 7 "\xd9"; +10 -> 7 "\xda"; +10 -> 7 "\xdb"; +10 -> 7 "\xdc"; +10 -> 7 "\xdd"; +10 -> 7 "\xde"; +10 -> 7 "\xdf"; +10 -> 7 "\xe0"; +10 -> 7 "\xe1"; +10 -> 7 "\xe2"; +10 -> 7 "\xe3"; +10 -> 7 "\xe4"; +10 -> 7 "\xe5"; +10 -> 7 "\xe6"; +10 -> 7 "\xe7"; +10 -> 7 "\xe8"; +10 -> 7 "\xe9"; +10 -> 7 "\xea"; +10 -> 7 "\xeb"; +10 -> 7 "\xec"; +10 -> 7 "\xed"; +10 -> 7 "\xee"; +10 -> 7 "\xef"; +10 -> 7 "\xf0"; +10 -> 7 "\xf1"; +10 -> 7 "\xf2"; +10 -> 7 "\xf3"; +10 -> 7 "\xf4"; +10 -> 7 "\xf5"; +10 -> 7 "\xf6"; +10 -> 7 "\xf7"; +10 -> 7 "\xf8"; +10 -> 7 "\xf9"; +10 -> 7 "\xfa"; +10 -> 7 "\xfb"; +10 -> 7 "\xfc"; +10 -> 7 "\xfd"; +10 -> 7 "\xfe"; +10 -> 7 "\xff"; + +start: 0; +end: 3; From b8b4baf987645ce1f9180212c82d5d44caf70699 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 27 Nov 2023 17:02:44 -0500 Subject: [PATCH 060/373] pcre-anchor/in95-98.re: More anchoring tests from fuzzing. --- tests/pcre-anchor/in95.re | 1 + tests/pcre-anchor/in96.re | 1 + tests/pcre-anchor/in97.re | 1 + tests/pcre-anchor/in98.re | 1 + tests/pcre-anchor/out95.fsm | 771 ++++++++++++++++++++++++++++++++++++ tests/pcre-anchor/out96.fsm | 771 ++++++++++++++++++++++++++++++++++++ tests/pcre-anchor/out97.fsm | 5 + tests/pcre-anchor/out98.fsm | 771 ++++++++++++++++++++++++++++++++++++ 8 files changed, 2322 insertions(+) create mode 100644 tests/pcre-anchor/in95.re create mode 100644 tests/pcre-anchor/in96.re create mode 100644 tests/pcre-anchor/in97.re create mode 100644 tests/pcre-anchor/in98.re create mode 100644 tests/pcre-anchor/out95.fsm create mode 100644 tests/pcre-anchor/out96.fsm create mode 100644 tests/pcre-anchor/out97.fsm create mode 100644 tests/pcre-anchor/out98.fsm diff --git a/tests/pcre-anchor/in95.re b/tests/pcre-anchor/in95.re new file mode 100644 index 000000000..0b01ad029 --- /dev/null +++ b/tests/pcre-anchor/in95.re @@ -0,0 +1 @@ +a$|^b$(|c)^$ \ No newline at end of file diff --git a/tests/pcre-anchor/in96.re b/tests/pcre-anchor/in96.re new file mode 100644 index 000000000..4968820fe --- /dev/null +++ b/tests/pcre-anchor/in96.re @@ -0,0 +1 @@ +a$|$b(|c)^$ \ No newline at end of file diff --git a/tests/pcre-anchor/in97.re b/tests/pcre-anchor/in97.re new file mode 100644 index 000000000..bff8c08e1 --- /dev/null +++ b/tests/pcre-anchor/in97.re @@ -0,0 +1 @@ +^a$|^$b$c^d$-$(|e)^$ \ No newline at end of file diff --git a/tests/pcre-anchor/in98.re b/tests/pcre-anchor/in98.re new file mode 100644 index 000000000..c711c61cb --- /dev/null +++ b/tests/pcre-anchor/in98.re @@ -0,0 +1 @@ +a$|^b(^|$c) \ No newline at end of file diff --git a/tests/pcre-anchor/out95.fsm b/tests/pcre-anchor/out95.fsm new file mode 100644 index 000000000..ee435ee3c --- /dev/null +++ b/tests/pcre-anchor/out95.fsm @@ -0,0 +1,771 @@ +0 -> 0 "\x00"; +0 -> 0 "\x01"; +0 -> 0 "\x02"; +0 -> 0 "\x03"; +0 -> 0 "\x04"; +0 -> 0 "\x05"; +0 -> 0 "\x06"; +0 -> 0 "\x07"; +0 -> 0 "\x08"; +0 -> 0 "\t"; +0 -> 0 "\n"; +0 -> 0 "\v"; +0 -> 0 "\f"; +0 -> 0 "\r"; +0 -> 0 "\x0e"; +0 -> 0 "\x0f"; +0 -> 0 "\x10"; +0 -> 0 "\x11"; +0 -> 0 "\x12"; +0 -> 0 "\x13"; +0 -> 0 "\x14"; +0 -> 0 "\x15"; +0 -> 0 "\x16"; +0 -> 0 "\x17"; +0 -> 0 "\x18"; +0 -> 0 "\x19"; +0 -> 0 "\x1a"; +0 -> 0 "\x1b"; +0 -> 0 "\x1c"; +0 -> 0 "\x1d"; +0 -> 0 "\x1e"; +0 -> 0 "\x1f"; +0 -> 0 " "; +0 -> 0 "!"; +0 -> 0 "\""; +0 -> 0 "#"; +0 -> 0 "$"; +0 -> 0 "%"; +0 -> 0 "&"; +0 -> 0 "'"; +0 -> 0 "("; +0 -> 0 ")"; +0 -> 0 "*"; +0 -> 0 "+"; +0 -> 0 ","; +0 -> 0 "-"; +0 -> 0 "."; +0 -> 0 "/"; +0 -> 0 "0"; +0 -> 0 "1"; +0 -> 0 "2"; +0 -> 0 "3"; +0 -> 0 "4"; +0 -> 0 "5"; +0 -> 0 "6"; +0 -> 0 "7"; +0 -> 0 "8"; +0 -> 0 "9"; +0 -> 0 ":"; +0 -> 0 ";"; +0 -> 0 "<"; +0 -> 0 "="; +0 -> 0 ">"; +0 -> 0 "?"; +0 -> 0 "@"; +0 -> 0 "A"; +0 -> 0 "B"; +0 -> 0 "C"; +0 -> 0 "D"; +0 -> 0 "E"; +0 -> 0 "F"; +0 -> 0 "G"; +0 -> 0 "H"; +0 -> 0 "I"; +0 -> 0 "J"; +0 -> 0 "K"; +0 -> 0 "L"; +0 -> 0 "M"; +0 -> 0 "N"; +0 -> 0 "O"; +0 -> 0 "P"; +0 -> 0 "Q"; +0 -> 0 "R"; +0 -> 0 "S"; +0 -> 0 "T"; +0 -> 0 "U"; +0 -> 0 "V"; +0 -> 0 "W"; +0 -> 0 "X"; +0 -> 0 "Y"; +0 -> 0 "Z"; +0 -> 0 "["; +0 -> 0 "\\"; +0 -> 0 "]"; +0 -> 0 "^"; +0 -> 0 "_"; +0 -> 0 "`"; +0 -> 1 "a"; +0 -> 0 "b"; +0 -> 0 "c"; +0 -> 0 "d"; +0 -> 0 "e"; +0 -> 0 "f"; +0 -> 0 "g"; +0 -> 0 "h"; +0 -> 0 "i"; +0 -> 0 "j"; +0 -> 0 "k"; +0 -> 0 "l"; +0 -> 0 "m"; +0 -> 0 "n"; +0 -> 0 "o"; +0 -> 0 "p"; +0 -> 0 "q"; +0 -> 0 "r"; +0 -> 0 "s"; +0 -> 0 "t"; +0 -> 0 "u"; +0 -> 0 "v"; +0 -> 0 "w"; +0 -> 0 "x"; +0 -> 0 "y"; +0 -> 0 "z"; +0 -> 0 "{"; +0 -> 0 "|"; +0 -> 0 "}"; +0 -> 0 "~"; +0 -> 0 "\x7f"; +0 -> 0 "\x80"; +0 -> 0 "\x81"; +0 -> 0 "\x82"; +0 -> 0 "\x83"; +0 -> 0 "\x84"; +0 -> 0 "\x85"; +0 -> 0 "\x86"; +0 -> 0 "\x87"; +0 -> 0 "\x88"; +0 -> 0 "\x89"; +0 -> 0 "\x8a"; +0 -> 0 "\x8b"; +0 -> 0 "\x8c"; +0 -> 0 "\x8d"; +0 -> 0 "\x8e"; +0 -> 0 "\x8f"; +0 -> 0 "\x90"; +0 -> 0 "\x91"; +0 -> 0 "\x92"; +0 -> 0 "\x93"; +0 -> 0 "\x94"; +0 -> 0 "\x95"; +0 -> 0 "\x96"; +0 -> 0 "\x97"; +0 -> 0 "\x98"; +0 -> 0 "\x99"; +0 -> 0 "\x9a"; +0 -> 0 "\x9b"; +0 -> 0 "\x9c"; +0 -> 0 "\x9d"; +0 -> 0 "\x9e"; +0 -> 0 "\x9f"; +0 -> 0 "\xa0"; +0 -> 0 "\xa1"; +0 -> 0 "\xa2"; +0 -> 0 "\xa3"; +0 -> 0 "\xa4"; +0 -> 0 "\xa5"; +0 -> 0 "\xa6"; +0 -> 0 "\xa7"; +0 -> 0 "\xa8"; +0 -> 0 "\xa9"; +0 -> 0 "\xaa"; +0 -> 0 "\xab"; +0 -> 0 "\xac"; +0 -> 0 "\xad"; +0 -> 0 "\xae"; +0 -> 0 "\xaf"; +0 -> 0 "\xb0"; +0 -> 0 "\xb1"; +0 -> 0 "\xb2"; +0 -> 0 "\xb3"; +0 -> 0 "\xb4"; +0 -> 0 "\xb5"; +0 -> 0 "\xb6"; +0 -> 0 "\xb7"; +0 -> 0 "\xb8"; +0 -> 0 "\xb9"; +0 -> 0 "\xba"; +0 -> 0 "\xbb"; +0 -> 0 "\xbc"; +0 -> 0 "\xbd"; +0 -> 0 "\xbe"; +0 -> 0 "\xbf"; +0 -> 0 "\xc0"; +0 -> 0 "\xc1"; +0 -> 0 "\xc2"; +0 -> 0 "\xc3"; +0 -> 0 "\xc4"; +0 -> 0 "\xc5"; +0 -> 0 "\xc6"; +0 -> 0 "\xc7"; +0 -> 0 "\xc8"; +0 -> 0 "\xc9"; +0 -> 0 "\xca"; +0 -> 0 "\xcb"; +0 -> 0 "\xcc"; +0 -> 0 "\xcd"; +0 -> 0 "\xce"; +0 -> 0 "\xcf"; +0 -> 0 "\xd0"; +0 -> 0 "\xd1"; +0 -> 0 "\xd2"; +0 -> 0 "\xd3"; +0 -> 0 "\xd4"; +0 -> 0 "\xd5"; +0 -> 0 "\xd6"; +0 -> 0 "\xd7"; +0 -> 0 "\xd8"; +0 -> 0 "\xd9"; +0 -> 0 "\xda"; +0 -> 0 "\xdb"; +0 -> 0 "\xdc"; +0 -> 0 "\xdd"; +0 -> 0 "\xde"; +0 -> 0 "\xdf"; +0 -> 0 "\xe0"; +0 -> 0 "\xe1"; +0 -> 0 "\xe2"; +0 -> 0 "\xe3"; +0 -> 0 "\xe4"; +0 -> 0 "\xe5"; +0 -> 0 "\xe6"; +0 -> 0 "\xe7"; +0 -> 0 "\xe8"; +0 -> 0 "\xe9"; +0 -> 0 "\xea"; +0 -> 0 "\xeb"; +0 -> 0 "\xec"; +0 -> 0 "\xed"; +0 -> 0 "\xee"; +0 -> 0 "\xef"; +0 -> 0 "\xf0"; +0 -> 0 "\xf1"; +0 -> 0 "\xf2"; +0 -> 0 "\xf3"; +0 -> 0 "\xf4"; +0 -> 0 "\xf5"; +0 -> 0 "\xf6"; +0 -> 0 "\xf7"; +0 -> 0 "\xf8"; +0 -> 0 "\xf9"; +0 -> 0 "\xfa"; +0 -> 0 "\xfb"; +0 -> 0 "\xfc"; +0 -> 0 "\xfd"; +0 -> 0 "\xfe"; +0 -> 0 "\xff"; +1 -> 0 "\x00"; +1 -> 0 "\x01"; +1 -> 0 "\x02"; +1 -> 0 "\x03"; +1 -> 0 "\x04"; +1 -> 0 "\x05"; +1 -> 0 "\x06"; +1 -> 0 "\x07"; +1 -> 0 "\x08"; +1 -> 0 "\t"; +1 -> 2 "\n"; +1 -> 0 "\v"; +1 -> 0 "\f"; +1 -> 0 "\r"; +1 -> 0 "\x0e"; +1 -> 0 "\x0f"; +1 -> 0 "\x10"; +1 -> 0 "\x11"; +1 -> 0 "\x12"; +1 -> 0 "\x13"; +1 -> 0 "\x14"; +1 -> 0 "\x15"; +1 -> 0 "\x16"; +1 -> 0 "\x17"; +1 -> 0 "\x18"; +1 -> 0 "\x19"; +1 -> 0 "\x1a"; +1 -> 0 "\x1b"; +1 -> 0 "\x1c"; +1 -> 0 "\x1d"; +1 -> 0 "\x1e"; +1 -> 0 "\x1f"; +1 -> 0 " "; +1 -> 0 "!"; +1 -> 0 "\""; +1 -> 0 "#"; +1 -> 0 "$"; +1 -> 0 "%"; +1 -> 0 "&"; +1 -> 0 "'"; +1 -> 0 "("; +1 -> 0 ")"; +1 -> 0 "*"; +1 -> 0 "+"; +1 -> 0 ","; +1 -> 0 "-"; +1 -> 0 "."; +1 -> 0 "/"; +1 -> 0 "0"; +1 -> 0 "1"; +1 -> 0 "2"; +1 -> 0 "3"; +1 -> 0 "4"; +1 -> 0 "5"; +1 -> 0 "6"; +1 -> 0 "7"; +1 -> 0 "8"; +1 -> 0 "9"; +1 -> 0 ":"; +1 -> 0 ";"; +1 -> 0 "<"; +1 -> 0 "="; +1 -> 0 ">"; +1 -> 0 "?"; +1 -> 0 "@"; +1 -> 0 "A"; +1 -> 0 "B"; +1 -> 0 "C"; +1 -> 0 "D"; +1 -> 0 "E"; +1 -> 0 "F"; +1 -> 0 "G"; +1 -> 0 "H"; +1 -> 0 "I"; +1 -> 0 "J"; +1 -> 0 "K"; +1 -> 0 "L"; +1 -> 0 "M"; +1 -> 0 "N"; +1 -> 0 "O"; +1 -> 0 "P"; +1 -> 0 "Q"; +1 -> 0 "R"; +1 -> 0 "S"; +1 -> 0 "T"; +1 -> 0 "U"; +1 -> 0 "V"; +1 -> 0 "W"; +1 -> 0 "X"; +1 -> 0 "Y"; +1 -> 0 "Z"; +1 -> 0 "["; +1 -> 0 "\\"; +1 -> 0 "]"; +1 -> 0 "^"; +1 -> 0 "_"; +1 -> 0 "`"; +1 -> 1 "a"; +1 -> 0 "b"; +1 -> 0 "c"; +1 -> 0 "d"; +1 -> 0 "e"; +1 -> 0 "f"; +1 -> 0 "g"; +1 -> 0 "h"; +1 -> 0 "i"; +1 -> 0 "j"; +1 -> 0 "k"; +1 -> 0 "l"; +1 -> 0 "m"; +1 -> 0 "n"; +1 -> 0 "o"; +1 -> 0 "p"; +1 -> 0 "q"; +1 -> 0 "r"; +1 -> 0 "s"; +1 -> 0 "t"; +1 -> 0 "u"; +1 -> 0 "v"; +1 -> 0 "w"; +1 -> 0 "x"; +1 -> 0 "y"; +1 -> 0 "z"; +1 -> 0 "{"; +1 -> 0 "|"; +1 -> 0 "}"; +1 -> 0 "~"; +1 -> 0 "\x7f"; +1 -> 0 "\x80"; +1 -> 0 "\x81"; +1 -> 0 "\x82"; +1 -> 0 "\x83"; +1 -> 0 "\x84"; +1 -> 0 "\x85"; +1 -> 0 "\x86"; +1 -> 0 "\x87"; +1 -> 0 "\x88"; +1 -> 0 "\x89"; +1 -> 0 "\x8a"; +1 -> 0 "\x8b"; +1 -> 0 "\x8c"; +1 -> 0 "\x8d"; +1 -> 0 "\x8e"; +1 -> 0 "\x8f"; +1 -> 0 "\x90"; +1 -> 0 "\x91"; +1 -> 0 "\x92"; +1 -> 0 "\x93"; +1 -> 0 "\x94"; +1 -> 0 "\x95"; +1 -> 0 "\x96"; +1 -> 0 "\x97"; +1 -> 0 "\x98"; +1 -> 0 "\x99"; +1 -> 0 "\x9a"; +1 -> 0 "\x9b"; +1 -> 0 "\x9c"; +1 -> 0 "\x9d"; +1 -> 0 "\x9e"; +1 -> 0 "\x9f"; +1 -> 0 "\xa0"; +1 -> 0 "\xa1"; +1 -> 0 "\xa2"; +1 -> 0 "\xa3"; +1 -> 0 "\xa4"; +1 -> 0 "\xa5"; +1 -> 0 "\xa6"; +1 -> 0 "\xa7"; +1 -> 0 "\xa8"; +1 -> 0 "\xa9"; +1 -> 0 "\xaa"; +1 -> 0 "\xab"; +1 -> 0 "\xac"; +1 -> 0 "\xad"; +1 -> 0 "\xae"; +1 -> 0 "\xaf"; +1 -> 0 "\xb0"; +1 -> 0 "\xb1"; +1 -> 0 "\xb2"; +1 -> 0 "\xb3"; +1 -> 0 "\xb4"; +1 -> 0 "\xb5"; +1 -> 0 "\xb6"; +1 -> 0 "\xb7"; +1 -> 0 "\xb8"; +1 -> 0 "\xb9"; +1 -> 0 "\xba"; +1 -> 0 "\xbb"; +1 -> 0 "\xbc"; +1 -> 0 "\xbd"; +1 -> 0 "\xbe"; +1 -> 0 "\xbf"; +1 -> 0 "\xc0"; +1 -> 0 "\xc1"; +1 -> 0 "\xc2"; +1 -> 0 "\xc3"; +1 -> 0 "\xc4"; +1 -> 0 "\xc5"; +1 -> 0 "\xc6"; +1 -> 0 "\xc7"; +1 -> 0 "\xc8"; +1 -> 0 "\xc9"; +1 -> 0 "\xca"; +1 -> 0 "\xcb"; +1 -> 0 "\xcc"; +1 -> 0 "\xcd"; +1 -> 0 "\xce"; +1 -> 0 "\xcf"; +1 -> 0 "\xd0"; +1 -> 0 "\xd1"; +1 -> 0 "\xd2"; +1 -> 0 "\xd3"; +1 -> 0 "\xd4"; +1 -> 0 "\xd5"; +1 -> 0 "\xd6"; +1 -> 0 "\xd7"; +1 -> 0 "\xd8"; +1 -> 0 "\xd9"; +1 -> 0 "\xda"; +1 -> 0 "\xdb"; +1 -> 0 "\xdc"; +1 -> 0 "\xdd"; +1 -> 0 "\xde"; +1 -> 0 "\xdf"; +1 -> 0 "\xe0"; +1 -> 0 "\xe1"; +1 -> 0 "\xe2"; +1 -> 0 "\xe3"; +1 -> 0 "\xe4"; +1 -> 0 "\xe5"; +1 -> 0 "\xe6"; +1 -> 0 "\xe7"; +1 -> 0 "\xe8"; +1 -> 0 "\xe9"; +1 -> 0 "\xea"; +1 -> 0 "\xeb"; +1 -> 0 "\xec"; +1 -> 0 "\xed"; +1 -> 0 "\xee"; +1 -> 0 "\xef"; +1 -> 0 "\xf0"; +1 -> 0 "\xf1"; +1 -> 0 "\xf2"; +1 -> 0 "\xf3"; +1 -> 0 "\xf4"; +1 -> 0 "\xf5"; +1 -> 0 "\xf6"; +1 -> 0 "\xf7"; +1 -> 0 "\xf8"; +1 -> 0 "\xf9"; +1 -> 0 "\xfa"; +1 -> 0 "\xfb"; +1 -> 0 "\xfc"; +1 -> 0 "\xfd"; +1 -> 0 "\xfe"; +1 -> 0 "\xff"; +2 -> 0 "\x00"; +2 -> 0 "\x01"; +2 -> 0 "\x02"; +2 -> 0 "\x03"; +2 -> 0 "\x04"; +2 -> 0 "\x05"; +2 -> 0 "\x06"; +2 -> 0 "\x07"; +2 -> 0 "\x08"; +2 -> 0 "\t"; +2 -> 0 "\n"; +2 -> 0 "\v"; +2 -> 0 "\f"; +2 -> 0 "\r"; +2 -> 0 "\x0e"; +2 -> 0 "\x0f"; +2 -> 0 "\x10"; +2 -> 0 "\x11"; +2 -> 0 "\x12"; +2 -> 0 "\x13"; +2 -> 0 "\x14"; +2 -> 0 "\x15"; +2 -> 0 "\x16"; +2 -> 0 "\x17"; +2 -> 0 "\x18"; +2 -> 0 "\x19"; +2 -> 0 "\x1a"; +2 -> 0 "\x1b"; +2 -> 0 "\x1c"; +2 -> 0 "\x1d"; +2 -> 0 "\x1e"; +2 -> 0 "\x1f"; +2 -> 0 " "; +2 -> 0 "!"; +2 -> 0 "\""; +2 -> 0 "#"; +2 -> 0 "$"; +2 -> 0 "%"; +2 -> 0 "&"; +2 -> 0 "'"; +2 -> 0 "("; +2 -> 0 ")"; +2 -> 0 "*"; +2 -> 0 "+"; +2 -> 0 ","; +2 -> 0 "-"; +2 -> 0 "."; +2 -> 0 "/"; +2 -> 0 "0"; +2 -> 0 "1"; +2 -> 0 "2"; +2 -> 0 "3"; +2 -> 0 "4"; +2 -> 0 "5"; +2 -> 0 "6"; +2 -> 0 "7"; +2 -> 0 "8"; +2 -> 0 "9"; +2 -> 0 ":"; +2 -> 0 ";"; +2 -> 0 "<"; +2 -> 0 "="; +2 -> 0 ">"; +2 -> 0 "?"; +2 -> 0 "@"; +2 -> 0 "A"; +2 -> 0 "B"; +2 -> 0 "C"; +2 -> 0 "D"; +2 -> 0 "E"; +2 -> 0 "F"; +2 -> 0 "G"; +2 -> 0 "H"; +2 -> 0 "I"; +2 -> 0 "J"; +2 -> 0 "K"; +2 -> 0 "L"; +2 -> 0 "M"; +2 -> 0 "N"; +2 -> 0 "O"; +2 -> 0 "P"; +2 -> 0 "Q"; +2 -> 0 "R"; +2 -> 0 "S"; +2 -> 0 "T"; +2 -> 0 "U"; +2 -> 0 "V"; +2 -> 0 "W"; +2 -> 0 "X"; +2 -> 0 "Y"; +2 -> 0 "Z"; +2 -> 0 "["; +2 -> 0 "\\"; +2 -> 0 "]"; +2 -> 0 "^"; +2 -> 0 "_"; +2 -> 0 "`"; +2 -> 1 "a"; +2 -> 0 "b"; +2 -> 0 "c"; +2 -> 0 "d"; +2 -> 0 "e"; +2 -> 0 "f"; +2 -> 0 "g"; +2 -> 0 "h"; +2 -> 0 "i"; +2 -> 0 "j"; +2 -> 0 "k"; +2 -> 0 "l"; +2 -> 0 "m"; +2 -> 0 "n"; +2 -> 0 "o"; +2 -> 0 "p"; +2 -> 0 "q"; +2 -> 0 "r"; +2 -> 0 "s"; +2 -> 0 "t"; +2 -> 0 "u"; +2 -> 0 "v"; +2 -> 0 "w"; +2 -> 0 "x"; +2 -> 0 "y"; +2 -> 0 "z"; +2 -> 0 "{"; +2 -> 0 "|"; +2 -> 0 "}"; +2 -> 0 "~"; +2 -> 0 "\x7f"; +2 -> 0 "\x80"; +2 -> 0 "\x81"; +2 -> 0 "\x82"; +2 -> 0 "\x83"; +2 -> 0 "\x84"; +2 -> 0 "\x85"; +2 -> 0 "\x86"; +2 -> 0 "\x87"; +2 -> 0 "\x88"; +2 -> 0 "\x89"; +2 -> 0 "\x8a"; +2 -> 0 "\x8b"; +2 -> 0 "\x8c"; +2 -> 0 "\x8d"; +2 -> 0 "\x8e"; +2 -> 0 "\x8f"; +2 -> 0 "\x90"; +2 -> 0 "\x91"; +2 -> 0 "\x92"; +2 -> 0 "\x93"; +2 -> 0 "\x94"; +2 -> 0 "\x95"; +2 -> 0 "\x96"; +2 -> 0 "\x97"; +2 -> 0 "\x98"; +2 -> 0 "\x99"; +2 -> 0 "\x9a"; +2 -> 0 "\x9b"; +2 -> 0 "\x9c"; +2 -> 0 "\x9d"; +2 -> 0 "\x9e"; +2 -> 0 "\x9f"; +2 -> 0 "\xa0"; +2 -> 0 "\xa1"; +2 -> 0 "\xa2"; +2 -> 0 "\xa3"; +2 -> 0 "\xa4"; +2 -> 0 "\xa5"; +2 -> 0 "\xa6"; +2 -> 0 "\xa7"; +2 -> 0 "\xa8"; +2 -> 0 "\xa9"; +2 -> 0 "\xaa"; +2 -> 0 "\xab"; +2 -> 0 "\xac"; +2 -> 0 "\xad"; +2 -> 0 "\xae"; +2 -> 0 "\xaf"; +2 -> 0 "\xb0"; +2 -> 0 "\xb1"; +2 -> 0 "\xb2"; +2 -> 0 "\xb3"; +2 -> 0 "\xb4"; +2 -> 0 "\xb5"; +2 -> 0 "\xb6"; +2 -> 0 "\xb7"; +2 -> 0 "\xb8"; +2 -> 0 "\xb9"; +2 -> 0 "\xba"; +2 -> 0 "\xbb"; +2 -> 0 "\xbc"; +2 -> 0 "\xbd"; +2 -> 0 "\xbe"; +2 -> 0 "\xbf"; +2 -> 0 "\xc0"; +2 -> 0 "\xc1"; +2 -> 0 "\xc2"; +2 -> 0 "\xc3"; +2 -> 0 "\xc4"; +2 -> 0 "\xc5"; +2 -> 0 "\xc6"; +2 -> 0 "\xc7"; +2 -> 0 "\xc8"; +2 -> 0 "\xc9"; +2 -> 0 "\xca"; +2 -> 0 "\xcb"; +2 -> 0 "\xcc"; +2 -> 0 "\xcd"; +2 -> 0 "\xce"; +2 -> 0 "\xcf"; +2 -> 0 "\xd0"; +2 -> 0 "\xd1"; +2 -> 0 "\xd2"; +2 -> 0 "\xd3"; +2 -> 0 "\xd4"; +2 -> 0 "\xd5"; +2 -> 0 "\xd6"; +2 -> 0 "\xd7"; +2 -> 0 "\xd8"; +2 -> 0 "\xd9"; +2 -> 0 "\xda"; +2 -> 0 "\xdb"; +2 -> 0 "\xdc"; +2 -> 0 "\xdd"; +2 -> 0 "\xde"; +2 -> 0 "\xdf"; +2 -> 0 "\xe0"; +2 -> 0 "\xe1"; +2 -> 0 "\xe2"; +2 -> 0 "\xe3"; +2 -> 0 "\xe4"; +2 -> 0 "\xe5"; +2 -> 0 "\xe6"; +2 -> 0 "\xe7"; +2 -> 0 "\xe8"; +2 -> 0 "\xe9"; +2 -> 0 "\xea"; +2 -> 0 "\xeb"; +2 -> 0 "\xec"; +2 -> 0 "\xed"; +2 -> 0 "\xee"; +2 -> 0 "\xef"; +2 -> 0 "\xf0"; +2 -> 0 "\xf1"; +2 -> 0 "\xf2"; +2 -> 0 "\xf3"; +2 -> 0 "\xf4"; +2 -> 0 "\xf5"; +2 -> 0 "\xf6"; +2 -> 0 "\xf7"; +2 -> 0 "\xf8"; +2 -> 0 "\xf9"; +2 -> 0 "\xfa"; +2 -> 0 "\xfb"; +2 -> 0 "\xfc"; +2 -> 0 "\xfd"; +2 -> 0 "\xfe"; +2 -> 0 "\xff"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out96.fsm b/tests/pcre-anchor/out96.fsm new file mode 100644 index 000000000..ee435ee3c --- /dev/null +++ b/tests/pcre-anchor/out96.fsm @@ -0,0 +1,771 @@ +0 -> 0 "\x00"; +0 -> 0 "\x01"; +0 -> 0 "\x02"; +0 -> 0 "\x03"; +0 -> 0 "\x04"; +0 -> 0 "\x05"; +0 -> 0 "\x06"; +0 -> 0 "\x07"; +0 -> 0 "\x08"; +0 -> 0 "\t"; +0 -> 0 "\n"; +0 -> 0 "\v"; +0 -> 0 "\f"; +0 -> 0 "\r"; +0 -> 0 "\x0e"; +0 -> 0 "\x0f"; +0 -> 0 "\x10"; +0 -> 0 "\x11"; +0 -> 0 "\x12"; +0 -> 0 "\x13"; +0 -> 0 "\x14"; +0 -> 0 "\x15"; +0 -> 0 "\x16"; +0 -> 0 "\x17"; +0 -> 0 "\x18"; +0 -> 0 "\x19"; +0 -> 0 "\x1a"; +0 -> 0 "\x1b"; +0 -> 0 "\x1c"; +0 -> 0 "\x1d"; +0 -> 0 "\x1e"; +0 -> 0 "\x1f"; +0 -> 0 " "; +0 -> 0 "!"; +0 -> 0 "\""; +0 -> 0 "#"; +0 -> 0 "$"; +0 -> 0 "%"; +0 -> 0 "&"; +0 -> 0 "'"; +0 -> 0 "("; +0 -> 0 ")"; +0 -> 0 "*"; +0 -> 0 "+"; +0 -> 0 ","; +0 -> 0 "-"; +0 -> 0 "."; +0 -> 0 "/"; +0 -> 0 "0"; +0 -> 0 "1"; +0 -> 0 "2"; +0 -> 0 "3"; +0 -> 0 "4"; +0 -> 0 "5"; +0 -> 0 "6"; +0 -> 0 "7"; +0 -> 0 "8"; +0 -> 0 "9"; +0 -> 0 ":"; +0 -> 0 ";"; +0 -> 0 "<"; +0 -> 0 "="; +0 -> 0 ">"; +0 -> 0 "?"; +0 -> 0 "@"; +0 -> 0 "A"; +0 -> 0 "B"; +0 -> 0 "C"; +0 -> 0 "D"; +0 -> 0 "E"; +0 -> 0 "F"; +0 -> 0 "G"; +0 -> 0 "H"; +0 -> 0 "I"; +0 -> 0 "J"; +0 -> 0 "K"; +0 -> 0 "L"; +0 -> 0 "M"; +0 -> 0 "N"; +0 -> 0 "O"; +0 -> 0 "P"; +0 -> 0 "Q"; +0 -> 0 "R"; +0 -> 0 "S"; +0 -> 0 "T"; +0 -> 0 "U"; +0 -> 0 "V"; +0 -> 0 "W"; +0 -> 0 "X"; +0 -> 0 "Y"; +0 -> 0 "Z"; +0 -> 0 "["; +0 -> 0 "\\"; +0 -> 0 "]"; +0 -> 0 "^"; +0 -> 0 "_"; +0 -> 0 "`"; +0 -> 1 "a"; +0 -> 0 "b"; +0 -> 0 "c"; +0 -> 0 "d"; +0 -> 0 "e"; +0 -> 0 "f"; +0 -> 0 "g"; +0 -> 0 "h"; +0 -> 0 "i"; +0 -> 0 "j"; +0 -> 0 "k"; +0 -> 0 "l"; +0 -> 0 "m"; +0 -> 0 "n"; +0 -> 0 "o"; +0 -> 0 "p"; +0 -> 0 "q"; +0 -> 0 "r"; +0 -> 0 "s"; +0 -> 0 "t"; +0 -> 0 "u"; +0 -> 0 "v"; +0 -> 0 "w"; +0 -> 0 "x"; +0 -> 0 "y"; +0 -> 0 "z"; +0 -> 0 "{"; +0 -> 0 "|"; +0 -> 0 "}"; +0 -> 0 "~"; +0 -> 0 "\x7f"; +0 -> 0 "\x80"; +0 -> 0 "\x81"; +0 -> 0 "\x82"; +0 -> 0 "\x83"; +0 -> 0 "\x84"; +0 -> 0 "\x85"; +0 -> 0 "\x86"; +0 -> 0 "\x87"; +0 -> 0 "\x88"; +0 -> 0 "\x89"; +0 -> 0 "\x8a"; +0 -> 0 "\x8b"; +0 -> 0 "\x8c"; +0 -> 0 "\x8d"; +0 -> 0 "\x8e"; +0 -> 0 "\x8f"; +0 -> 0 "\x90"; +0 -> 0 "\x91"; +0 -> 0 "\x92"; +0 -> 0 "\x93"; +0 -> 0 "\x94"; +0 -> 0 "\x95"; +0 -> 0 "\x96"; +0 -> 0 "\x97"; +0 -> 0 "\x98"; +0 -> 0 "\x99"; +0 -> 0 "\x9a"; +0 -> 0 "\x9b"; +0 -> 0 "\x9c"; +0 -> 0 "\x9d"; +0 -> 0 "\x9e"; +0 -> 0 "\x9f"; +0 -> 0 "\xa0"; +0 -> 0 "\xa1"; +0 -> 0 "\xa2"; +0 -> 0 "\xa3"; +0 -> 0 "\xa4"; +0 -> 0 "\xa5"; +0 -> 0 "\xa6"; +0 -> 0 "\xa7"; +0 -> 0 "\xa8"; +0 -> 0 "\xa9"; +0 -> 0 "\xaa"; +0 -> 0 "\xab"; +0 -> 0 "\xac"; +0 -> 0 "\xad"; +0 -> 0 "\xae"; +0 -> 0 "\xaf"; +0 -> 0 "\xb0"; +0 -> 0 "\xb1"; +0 -> 0 "\xb2"; +0 -> 0 "\xb3"; +0 -> 0 "\xb4"; +0 -> 0 "\xb5"; +0 -> 0 "\xb6"; +0 -> 0 "\xb7"; +0 -> 0 "\xb8"; +0 -> 0 "\xb9"; +0 -> 0 "\xba"; +0 -> 0 "\xbb"; +0 -> 0 "\xbc"; +0 -> 0 "\xbd"; +0 -> 0 "\xbe"; +0 -> 0 "\xbf"; +0 -> 0 "\xc0"; +0 -> 0 "\xc1"; +0 -> 0 "\xc2"; +0 -> 0 "\xc3"; +0 -> 0 "\xc4"; +0 -> 0 "\xc5"; +0 -> 0 "\xc6"; +0 -> 0 "\xc7"; +0 -> 0 "\xc8"; +0 -> 0 "\xc9"; +0 -> 0 "\xca"; +0 -> 0 "\xcb"; +0 -> 0 "\xcc"; +0 -> 0 "\xcd"; +0 -> 0 "\xce"; +0 -> 0 "\xcf"; +0 -> 0 "\xd0"; +0 -> 0 "\xd1"; +0 -> 0 "\xd2"; +0 -> 0 "\xd3"; +0 -> 0 "\xd4"; +0 -> 0 "\xd5"; +0 -> 0 "\xd6"; +0 -> 0 "\xd7"; +0 -> 0 "\xd8"; +0 -> 0 "\xd9"; +0 -> 0 "\xda"; +0 -> 0 "\xdb"; +0 -> 0 "\xdc"; +0 -> 0 "\xdd"; +0 -> 0 "\xde"; +0 -> 0 "\xdf"; +0 -> 0 "\xe0"; +0 -> 0 "\xe1"; +0 -> 0 "\xe2"; +0 -> 0 "\xe3"; +0 -> 0 "\xe4"; +0 -> 0 "\xe5"; +0 -> 0 "\xe6"; +0 -> 0 "\xe7"; +0 -> 0 "\xe8"; +0 -> 0 "\xe9"; +0 -> 0 "\xea"; +0 -> 0 "\xeb"; +0 -> 0 "\xec"; +0 -> 0 "\xed"; +0 -> 0 "\xee"; +0 -> 0 "\xef"; +0 -> 0 "\xf0"; +0 -> 0 "\xf1"; +0 -> 0 "\xf2"; +0 -> 0 "\xf3"; +0 -> 0 "\xf4"; +0 -> 0 "\xf5"; +0 -> 0 "\xf6"; +0 -> 0 "\xf7"; +0 -> 0 "\xf8"; +0 -> 0 "\xf9"; +0 -> 0 "\xfa"; +0 -> 0 "\xfb"; +0 -> 0 "\xfc"; +0 -> 0 "\xfd"; +0 -> 0 "\xfe"; +0 -> 0 "\xff"; +1 -> 0 "\x00"; +1 -> 0 "\x01"; +1 -> 0 "\x02"; +1 -> 0 "\x03"; +1 -> 0 "\x04"; +1 -> 0 "\x05"; +1 -> 0 "\x06"; +1 -> 0 "\x07"; +1 -> 0 "\x08"; +1 -> 0 "\t"; +1 -> 2 "\n"; +1 -> 0 "\v"; +1 -> 0 "\f"; +1 -> 0 "\r"; +1 -> 0 "\x0e"; +1 -> 0 "\x0f"; +1 -> 0 "\x10"; +1 -> 0 "\x11"; +1 -> 0 "\x12"; +1 -> 0 "\x13"; +1 -> 0 "\x14"; +1 -> 0 "\x15"; +1 -> 0 "\x16"; +1 -> 0 "\x17"; +1 -> 0 "\x18"; +1 -> 0 "\x19"; +1 -> 0 "\x1a"; +1 -> 0 "\x1b"; +1 -> 0 "\x1c"; +1 -> 0 "\x1d"; +1 -> 0 "\x1e"; +1 -> 0 "\x1f"; +1 -> 0 " "; +1 -> 0 "!"; +1 -> 0 "\""; +1 -> 0 "#"; +1 -> 0 "$"; +1 -> 0 "%"; +1 -> 0 "&"; +1 -> 0 "'"; +1 -> 0 "("; +1 -> 0 ")"; +1 -> 0 "*"; +1 -> 0 "+"; +1 -> 0 ","; +1 -> 0 "-"; +1 -> 0 "."; +1 -> 0 "/"; +1 -> 0 "0"; +1 -> 0 "1"; +1 -> 0 "2"; +1 -> 0 "3"; +1 -> 0 "4"; +1 -> 0 "5"; +1 -> 0 "6"; +1 -> 0 "7"; +1 -> 0 "8"; +1 -> 0 "9"; +1 -> 0 ":"; +1 -> 0 ";"; +1 -> 0 "<"; +1 -> 0 "="; +1 -> 0 ">"; +1 -> 0 "?"; +1 -> 0 "@"; +1 -> 0 "A"; +1 -> 0 "B"; +1 -> 0 "C"; +1 -> 0 "D"; +1 -> 0 "E"; +1 -> 0 "F"; +1 -> 0 "G"; +1 -> 0 "H"; +1 -> 0 "I"; +1 -> 0 "J"; +1 -> 0 "K"; +1 -> 0 "L"; +1 -> 0 "M"; +1 -> 0 "N"; +1 -> 0 "O"; +1 -> 0 "P"; +1 -> 0 "Q"; +1 -> 0 "R"; +1 -> 0 "S"; +1 -> 0 "T"; +1 -> 0 "U"; +1 -> 0 "V"; +1 -> 0 "W"; +1 -> 0 "X"; +1 -> 0 "Y"; +1 -> 0 "Z"; +1 -> 0 "["; +1 -> 0 "\\"; +1 -> 0 "]"; +1 -> 0 "^"; +1 -> 0 "_"; +1 -> 0 "`"; +1 -> 1 "a"; +1 -> 0 "b"; +1 -> 0 "c"; +1 -> 0 "d"; +1 -> 0 "e"; +1 -> 0 "f"; +1 -> 0 "g"; +1 -> 0 "h"; +1 -> 0 "i"; +1 -> 0 "j"; +1 -> 0 "k"; +1 -> 0 "l"; +1 -> 0 "m"; +1 -> 0 "n"; +1 -> 0 "o"; +1 -> 0 "p"; +1 -> 0 "q"; +1 -> 0 "r"; +1 -> 0 "s"; +1 -> 0 "t"; +1 -> 0 "u"; +1 -> 0 "v"; +1 -> 0 "w"; +1 -> 0 "x"; +1 -> 0 "y"; +1 -> 0 "z"; +1 -> 0 "{"; +1 -> 0 "|"; +1 -> 0 "}"; +1 -> 0 "~"; +1 -> 0 "\x7f"; +1 -> 0 "\x80"; +1 -> 0 "\x81"; +1 -> 0 "\x82"; +1 -> 0 "\x83"; +1 -> 0 "\x84"; +1 -> 0 "\x85"; +1 -> 0 "\x86"; +1 -> 0 "\x87"; +1 -> 0 "\x88"; +1 -> 0 "\x89"; +1 -> 0 "\x8a"; +1 -> 0 "\x8b"; +1 -> 0 "\x8c"; +1 -> 0 "\x8d"; +1 -> 0 "\x8e"; +1 -> 0 "\x8f"; +1 -> 0 "\x90"; +1 -> 0 "\x91"; +1 -> 0 "\x92"; +1 -> 0 "\x93"; +1 -> 0 "\x94"; +1 -> 0 "\x95"; +1 -> 0 "\x96"; +1 -> 0 "\x97"; +1 -> 0 "\x98"; +1 -> 0 "\x99"; +1 -> 0 "\x9a"; +1 -> 0 "\x9b"; +1 -> 0 "\x9c"; +1 -> 0 "\x9d"; +1 -> 0 "\x9e"; +1 -> 0 "\x9f"; +1 -> 0 "\xa0"; +1 -> 0 "\xa1"; +1 -> 0 "\xa2"; +1 -> 0 "\xa3"; +1 -> 0 "\xa4"; +1 -> 0 "\xa5"; +1 -> 0 "\xa6"; +1 -> 0 "\xa7"; +1 -> 0 "\xa8"; +1 -> 0 "\xa9"; +1 -> 0 "\xaa"; +1 -> 0 "\xab"; +1 -> 0 "\xac"; +1 -> 0 "\xad"; +1 -> 0 "\xae"; +1 -> 0 "\xaf"; +1 -> 0 "\xb0"; +1 -> 0 "\xb1"; +1 -> 0 "\xb2"; +1 -> 0 "\xb3"; +1 -> 0 "\xb4"; +1 -> 0 "\xb5"; +1 -> 0 "\xb6"; +1 -> 0 "\xb7"; +1 -> 0 "\xb8"; +1 -> 0 "\xb9"; +1 -> 0 "\xba"; +1 -> 0 "\xbb"; +1 -> 0 "\xbc"; +1 -> 0 "\xbd"; +1 -> 0 "\xbe"; +1 -> 0 "\xbf"; +1 -> 0 "\xc0"; +1 -> 0 "\xc1"; +1 -> 0 "\xc2"; +1 -> 0 "\xc3"; +1 -> 0 "\xc4"; +1 -> 0 "\xc5"; +1 -> 0 "\xc6"; +1 -> 0 "\xc7"; +1 -> 0 "\xc8"; +1 -> 0 "\xc9"; +1 -> 0 "\xca"; +1 -> 0 "\xcb"; +1 -> 0 "\xcc"; +1 -> 0 "\xcd"; +1 -> 0 "\xce"; +1 -> 0 "\xcf"; +1 -> 0 "\xd0"; +1 -> 0 "\xd1"; +1 -> 0 "\xd2"; +1 -> 0 "\xd3"; +1 -> 0 "\xd4"; +1 -> 0 "\xd5"; +1 -> 0 "\xd6"; +1 -> 0 "\xd7"; +1 -> 0 "\xd8"; +1 -> 0 "\xd9"; +1 -> 0 "\xda"; +1 -> 0 "\xdb"; +1 -> 0 "\xdc"; +1 -> 0 "\xdd"; +1 -> 0 "\xde"; +1 -> 0 "\xdf"; +1 -> 0 "\xe0"; +1 -> 0 "\xe1"; +1 -> 0 "\xe2"; +1 -> 0 "\xe3"; +1 -> 0 "\xe4"; +1 -> 0 "\xe5"; +1 -> 0 "\xe6"; +1 -> 0 "\xe7"; +1 -> 0 "\xe8"; +1 -> 0 "\xe9"; +1 -> 0 "\xea"; +1 -> 0 "\xeb"; +1 -> 0 "\xec"; +1 -> 0 "\xed"; +1 -> 0 "\xee"; +1 -> 0 "\xef"; +1 -> 0 "\xf0"; +1 -> 0 "\xf1"; +1 -> 0 "\xf2"; +1 -> 0 "\xf3"; +1 -> 0 "\xf4"; +1 -> 0 "\xf5"; +1 -> 0 "\xf6"; +1 -> 0 "\xf7"; +1 -> 0 "\xf8"; +1 -> 0 "\xf9"; +1 -> 0 "\xfa"; +1 -> 0 "\xfb"; +1 -> 0 "\xfc"; +1 -> 0 "\xfd"; +1 -> 0 "\xfe"; +1 -> 0 "\xff"; +2 -> 0 "\x00"; +2 -> 0 "\x01"; +2 -> 0 "\x02"; +2 -> 0 "\x03"; +2 -> 0 "\x04"; +2 -> 0 "\x05"; +2 -> 0 "\x06"; +2 -> 0 "\x07"; +2 -> 0 "\x08"; +2 -> 0 "\t"; +2 -> 0 "\n"; +2 -> 0 "\v"; +2 -> 0 "\f"; +2 -> 0 "\r"; +2 -> 0 "\x0e"; +2 -> 0 "\x0f"; +2 -> 0 "\x10"; +2 -> 0 "\x11"; +2 -> 0 "\x12"; +2 -> 0 "\x13"; +2 -> 0 "\x14"; +2 -> 0 "\x15"; +2 -> 0 "\x16"; +2 -> 0 "\x17"; +2 -> 0 "\x18"; +2 -> 0 "\x19"; +2 -> 0 "\x1a"; +2 -> 0 "\x1b"; +2 -> 0 "\x1c"; +2 -> 0 "\x1d"; +2 -> 0 "\x1e"; +2 -> 0 "\x1f"; +2 -> 0 " "; +2 -> 0 "!"; +2 -> 0 "\""; +2 -> 0 "#"; +2 -> 0 "$"; +2 -> 0 "%"; +2 -> 0 "&"; +2 -> 0 "'"; +2 -> 0 "("; +2 -> 0 ")"; +2 -> 0 "*"; +2 -> 0 "+"; +2 -> 0 ","; +2 -> 0 "-"; +2 -> 0 "."; +2 -> 0 "/"; +2 -> 0 "0"; +2 -> 0 "1"; +2 -> 0 "2"; +2 -> 0 "3"; +2 -> 0 "4"; +2 -> 0 "5"; +2 -> 0 "6"; +2 -> 0 "7"; +2 -> 0 "8"; +2 -> 0 "9"; +2 -> 0 ":"; +2 -> 0 ";"; +2 -> 0 "<"; +2 -> 0 "="; +2 -> 0 ">"; +2 -> 0 "?"; +2 -> 0 "@"; +2 -> 0 "A"; +2 -> 0 "B"; +2 -> 0 "C"; +2 -> 0 "D"; +2 -> 0 "E"; +2 -> 0 "F"; +2 -> 0 "G"; +2 -> 0 "H"; +2 -> 0 "I"; +2 -> 0 "J"; +2 -> 0 "K"; +2 -> 0 "L"; +2 -> 0 "M"; +2 -> 0 "N"; +2 -> 0 "O"; +2 -> 0 "P"; +2 -> 0 "Q"; +2 -> 0 "R"; +2 -> 0 "S"; +2 -> 0 "T"; +2 -> 0 "U"; +2 -> 0 "V"; +2 -> 0 "W"; +2 -> 0 "X"; +2 -> 0 "Y"; +2 -> 0 "Z"; +2 -> 0 "["; +2 -> 0 "\\"; +2 -> 0 "]"; +2 -> 0 "^"; +2 -> 0 "_"; +2 -> 0 "`"; +2 -> 1 "a"; +2 -> 0 "b"; +2 -> 0 "c"; +2 -> 0 "d"; +2 -> 0 "e"; +2 -> 0 "f"; +2 -> 0 "g"; +2 -> 0 "h"; +2 -> 0 "i"; +2 -> 0 "j"; +2 -> 0 "k"; +2 -> 0 "l"; +2 -> 0 "m"; +2 -> 0 "n"; +2 -> 0 "o"; +2 -> 0 "p"; +2 -> 0 "q"; +2 -> 0 "r"; +2 -> 0 "s"; +2 -> 0 "t"; +2 -> 0 "u"; +2 -> 0 "v"; +2 -> 0 "w"; +2 -> 0 "x"; +2 -> 0 "y"; +2 -> 0 "z"; +2 -> 0 "{"; +2 -> 0 "|"; +2 -> 0 "}"; +2 -> 0 "~"; +2 -> 0 "\x7f"; +2 -> 0 "\x80"; +2 -> 0 "\x81"; +2 -> 0 "\x82"; +2 -> 0 "\x83"; +2 -> 0 "\x84"; +2 -> 0 "\x85"; +2 -> 0 "\x86"; +2 -> 0 "\x87"; +2 -> 0 "\x88"; +2 -> 0 "\x89"; +2 -> 0 "\x8a"; +2 -> 0 "\x8b"; +2 -> 0 "\x8c"; +2 -> 0 "\x8d"; +2 -> 0 "\x8e"; +2 -> 0 "\x8f"; +2 -> 0 "\x90"; +2 -> 0 "\x91"; +2 -> 0 "\x92"; +2 -> 0 "\x93"; +2 -> 0 "\x94"; +2 -> 0 "\x95"; +2 -> 0 "\x96"; +2 -> 0 "\x97"; +2 -> 0 "\x98"; +2 -> 0 "\x99"; +2 -> 0 "\x9a"; +2 -> 0 "\x9b"; +2 -> 0 "\x9c"; +2 -> 0 "\x9d"; +2 -> 0 "\x9e"; +2 -> 0 "\x9f"; +2 -> 0 "\xa0"; +2 -> 0 "\xa1"; +2 -> 0 "\xa2"; +2 -> 0 "\xa3"; +2 -> 0 "\xa4"; +2 -> 0 "\xa5"; +2 -> 0 "\xa6"; +2 -> 0 "\xa7"; +2 -> 0 "\xa8"; +2 -> 0 "\xa9"; +2 -> 0 "\xaa"; +2 -> 0 "\xab"; +2 -> 0 "\xac"; +2 -> 0 "\xad"; +2 -> 0 "\xae"; +2 -> 0 "\xaf"; +2 -> 0 "\xb0"; +2 -> 0 "\xb1"; +2 -> 0 "\xb2"; +2 -> 0 "\xb3"; +2 -> 0 "\xb4"; +2 -> 0 "\xb5"; +2 -> 0 "\xb6"; +2 -> 0 "\xb7"; +2 -> 0 "\xb8"; +2 -> 0 "\xb9"; +2 -> 0 "\xba"; +2 -> 0 "\xbb"; +2 -> 0 "\xbc"; +2 -> 0 "\xbd"; +2 -> 0 "\xbe"; +2 -> 0 "\xbf"; +2 -> 0 "\xc0"; +2 -> 0 "\xc1"; +2 -> 0 "\xc2"; +2 -> 0 "\xc3"; +2 -> 0 "\xc4"; +2 -> 0 "\xc5"; +2 -> 0 "\xc6"; +2 -> 0 "\xc7"; +2 -> 0 "\xc8"; +2 -> 0 "\xc9"; +2 -> 0 "\xca"; +2 -> 0 "\xcb"; +2 -> 0 "\xcc"; +2 -> 0 "\xcd"; +2 -> 0 "\xce"; +2 -> 0 "\xcf"; +2 -> 0 "\xd0"; +2 -> 0 "\xd1"; +2 -> 0 "\xd2"; +2 -> 0 "\xd3"; +2 -> 0 "\xd4"; +2 -> 0 "\xd5"; +2 -> 0 "\xd6"; +2 -> 0 "\xd7"; +2 -> 0 "\xd8"; +2 -> 0 "\xd9"; +2 -> 0 "\xda"; +2 -> 0 "\xdb"; +2 -> 0 "\xdc"; +2 -> 0 "\xdd"; +2 -> 0 "\xde"; +2 -> 0 "\xdf"; +2 -> 0 "\xe0"; +2 -> 0 "\xe1"; +2 -> 0 "\xe2"; +2 -> 0 "\xe3"; +2 -> 0 "\xe4"; +2 -> 0 "\xe5"; +2 -> 0 "\xe6"; +2 -> 0 "\xe7"; +2 -> 0 "\xe8"; +2 -> 0 "\xe9"; +2 -> 0 "\xea"; +2 -> 0 "\xeb"; +2 -> 0 "\xec"; +2 -> 0 "\xed"; +2 -> 0 "\xee"; +2 -> 0 "\xef"; +2 -> 0 "\xf0"; +2 -> 0 "\xf1"; +2 -> 0 "\xf2"; +2 -> 0 "\xf3"; +2 -> 0 "\xf4"; +2 -> 0 "\xf5"; +2 -> 0 "\xf6"; +2 -> 0 "\xf7"; +2 -> 0 "\xf8"; +2 -> 0 "\xf9"; +2 -> 0 "\xfa"; +2 -> 0 "\xfb"; +2 -> 0 "\xfc"; +2 -> 0 "\xfd"; +2 -> 0 "\xfe"; +2 -> 0 "\xff"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out97.fsm b/tests/pcre-anchor/out97.fsm new file mode 100644 index 000000000..1e6bf8fca --- /dev/null +++ b/tests/pcre-anchor/out97.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; +1 -> 2 "\n"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out98.fsm b/tests/pcre-anchor/out98.fsm new file mode 100644 index 000000000..ee435ee3c --- /dev/null +++ b/tests/pcre-anchor/out98.fsm @@ -0,0 +1,771 @@ +0 -> 0 "\x00"; +0 -> 0 "\x01"; +0 -> 0 "\x02"; +0 -> 0 "\x03"; +0 -> 0 "\x04"; +0 -> 0 "\x05"; +0 -> 0 "\x06"; +0 -> 0 "\x07"; +0 -> 0 "\x08"; +0 -> 0 "\t"; +0 -> 0 "\n"; +0 -> 0 "\v"; +0 -> 0 "\f"; +0 -> 0 "\r"; +0 -> 0 "\x0e"; +0 -> 0 "\x0f"; +0 -> 0 "\x10"; +0 -> 0 "\x11"; +0 -> 0 "\x12"; +0 -> 0 "\x13"; +0 -> 0 "\x14"; +0 -> 0 "\x15"; +0 -> 0 "\x16"; +0 -> 0 "\x17"; +0 -> 0 "\x18"; +0 -> 0 "\x19"; +0 -> 0 "\x1a"; +0 -> 0 "\x1b"; +0 -> 0 "\x1c"; +0 -> 0 "\x1d"; +0 -> 0 "\x1e"; +0 -> 0 "\x1f"; +0 -> 0 " "; +0 -> 0 "!"; +0 -> 0 "\""; +0 -> 0 "#"; +0 -> 0 "$"; +0 -> 0 "%"; +0 -> 0 "&"; +0 -> 0 "'"; +0 -> 0 "("; +0 -> 0 ")"; +0 -> 0 "*"; +0 -> 0 "+"; +0 -> 0 ","; +0 -> 0 "-"; +0 -> 0 "."; +0 -> 0 "/"; +0 -> 0 "0"; +0 -> 0 "1"; +0 -> 0 "2"; +0 -> 0 "3"; +0 -> 0 "4"; +0 -> 0 "5"; +0 -> 0 "6"; +0 -> 0 "7"; +0 -> 0 "8"; +0 -> 0 "9"; +0 -> 0 ":"; +0 -> 0 ";"; +0 -> 0 "<"; +0 -> 0 "="; +0 -> 0 ">"; +0 -> 0 "?"; +0 -> 0 "@"; +0 -> 0 "A"; +0 -> 0 "B"; +0 -> 0 "C"; +0 -> 0 "D"; +0 -> 0 "E"; +0 -> 0 "F"; +0 -> 0 "G"; +0 -> 0 "H"; +0 -> 0 "I"; +0 -> 0 "J"; +0 -> 0 "K"; +0 -> 0 "L"; +0 -> 0 "M"; +0 -> 0 "N"; +0 -> 0 "O"; +0 -> 0 "P"; +0 -> 0 "Q"; +0 -> 0 "R"; +0 -> 0 "S"; +0 -> 0 "T"; +0 -> 0 "U"; +0 -> 0 "V"; +0 -> 0 "W"; +0 -> 0 "X"; +0 -> 0 "Y"; +0 -> 0 "Z"; +0 -> 0 "["; +0 -> 0 "\\"; +0 -> 0 "]"; +0 -> 0 "^"; +0 -> 0 "_"; +0 -> 0 "`"; +0 -> 1 "a"; +0 -> 0 "b"; +0 -> 0 "c"; +0 -> 0 "d"; +0 -> 0 "e"; +0 -> 0 "f"; +0 -> 0 "g"; +0 -> 0 "h"; +0 -> 0 "i"; +0 -> 0 "j"; +0 -> 0 "k"; +0 -> 0 "l"; +0 -> 0 "m"; +0 -> 0 "n"; +0 -> 0 "o"; +0 -> 0 "p"; +0 -> 0 "q"; +0 -> 0 "r"; +0 -> 0 "s"; +0 -> 0 "t"; +0 -> 0 "u"; +0 -> 0 "v"; +0 -> 0 "w"; +0 -> 0 "x"; +0 -> 0 "y"; +0 -> 0 "z"; +0 -> 0 "{"; +0 -> 0 "|"; +0 -> 0 "}"; +0 -> 0 "~"; +0 -> 0 "\x7f"; +0 -> 0 "\x80"; +0 -> 0 "\x81"; +0 -> 0 "\x82"; +0 -> 0 "\x83"; +0 -> 0 "\x84"; +0 -> 0 "\x85"; +0 -> 0 "\x86"; +0 -> 0 "\x87"; +0 -> 0 "\x88"; +0 -> 0 "\x89"; +0 -> 0 "\x8a"; +0 -> 0 "\x8b"; +0 -> 0 "\x8c"; +0 -> 0 "\x8d"; +0 -> 0 "\x8e"; +0 -> 0 "\x8f"; +0 -> 0 "\x90"; +0 -> 0 "\x91"; +0 -> 0 "\x92"; +0 -> 0 "\x93"; +0 -> 0 "\x94"; +0 -> 0 "\x95"; +0 -> 0 "\x96"; +0 -> 0 "\x97"; +0 -> 0 "\x98"; +0 -> 0 "\x99"; +0 -> 0 "\x9a"; +0 -> 0 "\x9b"; +0 -> 0 "\x9c"; +0 -> 0 "\x9d"; +0 -> 0 "\x9e"; +0 -> 0 "\x9f"; +0 -> 0 "\xa0"; +0 -> 0 "\xa1"; +0 -> 0 "\xa2"; +0 -> 0 "\xa3"; +0 -> 0 "\xa4"; +0 -> 0 "\xa5"; +0 -> 0 "\xa6"; +0 -> 0 "\xa7"; +0 -> 0 "\xa8"; +0 -> 0 "\xa9"; +0 -> 0 "\xaa"; +0 -> 0 "\xab"; +0 -> 0 "\xac"; +0 -> 0 "\xad"; +0 -> 0 "\xae"; +0 -> 0 "\xaf"; +0 -> 0 "\xb0"; +0 -> 0 "\xb1"; +0 -> 0 "\xb2"; +0 -> 0 "\xb3"; +0 -> 0 "\xb4"; +0 -> 0 "\xb5"; +0 -> 0 "\xb6"; +0 -> 0 "\xb7"; +0 -> 0 "\xb8"; +0 -> 0 "\xb9"; +0 -> 0 "\xba"; +0 -> 0 "\xbb"; +0 -> 0 "\xbc"; +0 -> 0 "\xbd"; +0 -> 0 "\xbe"; +0 -> 0 "\xbf"; +0 -> 0 "\xc0"; +0 -> 0 "\xc1"; +0 -> 0 "\xc2"; +0 -> 0 "\xc3"; +0 -> 0 "\xc4"; +0 -> 0 "\xc5"; +0 -> 0 "\xc6"; +0 -> 0 "\xc7"; +0 -> 0 "\xc8"; +0 -> 0 "\xc9"; +0 -> 0 "\xca"; +0 -> 0 "\xcb"; +0 -> 0 "\xcc"; +0 -> 0 "\xcd"; +0 -> 0 "\xce"; +0 -> 0 "\xcf"; +0 -> 0 "\xd0"; +0 -> 0 "\xd1"; +0 -> 0 "\xd2"; +0 -> 0 "\xd3"; +0 -> 0 "\xd4"; +0 -> 0 "\xd5"; +0 -> 0 "\xd6"; +0 -> 0 "\xd7"; +0 -> 0 "\xd8"; +0 -> 0 "\xd9"; +0 -> 0 "\xda"; +0 -> 0 "\xdb"; +0 -> 0 "\xdc"; +0 -> 0 "\xdd"; +0 -> 0 "\xde"; +0 -> 0 "\xdf"; +0 -> 0 "\xe0"; +0 -> 0 "\xe1"; +0 -> 0 "\xe2"; +0 -> 0 "\xe3"; +0 -> 0 "\xe4"; +0 -> 0 "\xe5"; +0 -> 0 "\xe6"; +0 -> 0 "\xe7"; +0 -> 0 "\xe8"; +0 -> 0 "\xe9"; +0 -> 0 "\xea"; +0 -> 0 "\xeb"; +0 -> 0 "\xec"; +0 -> 0 "\xed"; +0 -> 0 "\xee"; +0 -> 0 "\xef"; +0 -> 0 "\xf0"; +0 -> 0 "\xf1"; +0 -> 0 "\xf2"; +0 -> 0 "\xf3"; +0 -> 0 "\xf4"; +0 -> 0 "\xf5"; +0 -> 0 "\xf6"; +0 -> 0 "\xf7"; +0 -> 0 "\xf8"; +0 -> 0 "\xf9"; +0 -> 0 "\xfa"; +0 -> 0 "\xfb"; +0 -> 0 "\xfc"; +0 -> 0 "\xfd"; +0 -> 0 "\xfe"; +0 -> 0 "\xff"; +1 -> 0 "\x00"; +1 -> 0 "\x01"; +1 -> 0 "\x02"; +1 -> 0 "\x03"; +1 -> 0 "\x04"; +1 -> 0 "\x05"; +1 -> 0 "\x06"; +1 -> 0 "\x07"; +1 -> 0 "\x08"; +1 -> 0 "\t"; +1 -> 2 "\n"; +1 -> 0 "\v"; +1 -> 0 "\f"; +1 -> 0 "\r"; +1 -> 0 "\x0e"; +1 -> 0 "\x0f"; +1 -> 0 "\x10"; +1 -> 0 "\x11"; +1 -> 0 "\x12"; +1 -> 0 "\x13"; +1 -> 0 "\x14"; +1 -> 0 "\x15"; +1 -> 0 "\x16"; +1 -> 0 "\x17"; +1 -> 0 "\x18"; +1 -> 0 "\x19"; +1 -> 0 "\x1a"; +1 -> 0 "\x1b"; +1 -> 0 "\x1c"; +1 -> 0 "\x1d"; +1 -> 0 "\x1e"; +1 -> 0 "\x1f"; +1 -> 0 " "; +1 -> 0 "!"; +1 -> 0 "\""; +1 -> 0 "#"; +1 -> 0 "$"; +1 -> 0 "%"; +1 -> 0 "&"; +1 -> 0 "'"; +1 -> 0 "("; +1 -> 0 ")"; +1 -> 0 "*"; +1 -> 0 "+"; +1 -> 0 ","; +1 -> 0 "-"; +1 -> 0 "."; +1 -> 0 "/"; +1 -> 0 "0"; +1 -> 0 "1"; +1 -> 0 "2"; +1 -> 0 "3"; +1 -> 0 "4"; +1 -> 0 "5"; +1 -> 0 "6"; +1 -> 0 "7"; +1 -> 0 "8"; +1 -> 0 "9"; +1 -> 0 ":"; +1 -> 0 ";"; +1 -> 0 "<"; +1 -> 0 "="; +1 -> 0 ">"; +1 -> 0 "?"; +1 -> 0 "@"; +1 -> 0 "A"; +1 -> 0 "B"; +1 -> 0 "C"; +1 -> 0 "D"; +1 -> 0 "E"; +1 -> 0 "F"; +1 -> 0 "G"; +1 -> 0 "H"; +1 -> 0 "I"; +1 -> 0 "J"; +1 -> 0 "K"; +1 -> 0 "L"; +1 -> 0 "M"; +1 -> 0 "N"; +1 -> 0 "O"; +1 -> 0 "P"; +1 -> 0 "Q"; +1 -> 0 "R"; +1 -> 0 "S"; +1 -> 0 "T"; +1 -> 0 "U"; +1 -> 0 "V"; +1 -> 0 "W"; +1 -> 0 "X"; +1 -> 0 "Y"; +1 -> 0 "Z"; +1 -> 0 "["; +1 -> 0 "\\"; +1 -> 0 "]"; +1 -> 0 "^"; +1 -> 0 "_"; +1 -> 0 "`"; +1 -> 1 "a"; +1 -> 0 "b"; +1 -> 0 "c"; +1 -> 0 "d"; +1 -> 0 "e"; +1 -> 0 "f"; +1 -> 0 "g"; +1 -> 0 "h"; +1 -> 0 "i"; +1 -> 0 "j"; +1 -> 0 "k"; +1 -> 0 "l"; +1 -> 0 "m"; +1 -> 0 "n"; +1 -> 0 "o"; +1 -> 0 "p"; +1 -> 0 "q"; +1 -> 0 "r"; +1 -> 0 "s"; +1 -> 0 "t"; +1 -> 0 "u"; +1 -> 0 "v"; +1 -> 0 "w"; +1 -> 0 "x"; +1 -> 0 "y"; +1 -> 0 "z"; +1 -> 0 "{"; +1 -> 0 "|"; +1 -> 0 "}"; +1 -> 0 "~"; +1 -> 0 "\x7f"; +1 -> 0 "\x80"; +1 -> 0 "\x81"; +1 -> 0 "\x82"; +1 -> 0 "\x83"; +1 -> 0 "\x84"; +1 -> 0 "\x85"; +1 -> 0 "\x86"; +1 -> 0 "\x87"; +1 -> 0 "\x88"; +1 -> 0 "\x89"; +1 -> 0 "\x8a"; +1 -> 0 "\x8b"; +1 -> 0 "\x8c"; +1 -> 0 "\x8d"; +1 -> 0 "\x8e"; +1 -> 0 "\x8f"; +1 -> 0 "\x90"; +1 -> 0 "\x91"; +1 -> 0 "\x92"; +1 -> 0 "\x93"; +1 -> 0 "\x94"; +1 -> 0 "\x95"; +1 -> 0 "\x96"; +1 -> 0 "\x97"; +1 -> 0 "\x98"; +1 -> 0 "\x99"; +1 -> 0 "\x9a"; +1 -> 0 "\x9b"; +1 -> 0 "\x9c"; +1 -> 0 "\x9d"; +1 -> 0 "\x9e"; +1 -> 0 "\x9f"; +1 -> 0 "\xa0"; +1 -> 0 "\xa1"; +1 -> 0 "\xa2"; +1 -> 0 "\xa3"; +1 -> 0 "\xa4"; +1 -> 0 "\xa5"; +1 -> 0 "\xa6"; +1 -> 0 "\xa7"; +1 -> 0 "\xa8"; +1 -> 0 "\xa9"; +1 -> 0 "\xaa"; +1 -> 0 "\xab"; +1 -> 0 "\xac"; +1 -> 0 "\xad"; +1 -> 0 "\xae"; +1 -> 0 "\xaf"; +1 -> 0 "\xb0"; +1 -> 0 "\xb1"; +1 -> 0 "\xb2"; +1 -> 0 "\xb3"; +1 -> 0 "\xb4"; +1 -> 0 "\xb5"; +1 -> 0 "\xb6"; +1 -> 0 "\xb7"; +1 -> 0 "\xb8"; +1 -> 0 "\xb9"; +1 -> 0 "\xba"; +1 -> 0 "\xbb"; +1 -> 0 "\xbc"; +1 -> 0 "\xbd"; +1 -> 0 "\xbe"; +1 -> 0 "\xbf"; +1 -> 0 "\xc0"; +1 -> 0 "\xc1"; +1 -> 0 "\xc2"; +1 -> 0 "\xc3"; +1 -> 0 "\xc4"; +1 -> 0 "\xc5"; +1 -> 0 "\xc6"; +1 -> 0 "\xc7"; +1 -> 0 "\xc8"; +1 -> 0 "\xc9"; +1 -> 0 "\xca"; +1 -> 0 "\xcb"; +1 -> 0 "\xcc"; +1 -> 0 "\xcd"; +1 -> 0 "\xce"; +1 -> 0 "\xcf"; +1 -> 0 "\xd0"; +1 -> 0 "\xd1"; +1 -> 0 "\xd2"; +1 -> 0 "\xd3"; +1 -> 0 "\xd4"; +1 -> 0 "\xd5"; +1 -> 0 "\xd6"; +1 -> 0 "\xd7"; +1 -> 0 "\xd8"; +1 -> 0 "\xd9"; +1 -> 0 "\xda"; +1 -> 0 "\xdb"; +1 -> 0 "\xdc"; +1 -> 0 "\xdd"; +1 -> 0 "\xde"; +1 -> 0 "\xdf"; +1 -> 0 "\xe0"; +1 -> 0 "\xe1"; +1 -> 0 "\xe2"; +1 -> 0 "\xe3"; +1 -> 0 "\xe4"; +1 -> 0 "\xe5"; +1 -> 0 "\xe6"; +1 -> 0 "\xe7"; +1 -> 0 "\xe8"; +1 -> 0 "\xe9"; +1 -> 0 "\xea"; +1 -> 0 "\xeb"; +1 -> 0 "\xec"; +1 -> 0 "\xed"; +1 -> 0 "\xee"; +1 -> 0 "\xef"; +1 -> 0 "\xf0"; +1 -> 0 "\xf1"; +1 -> 0 "\xf2"; +1 -> 0 "\xf3"; +1 -> 0 "\xf4"; +1 -> 0 "\xf5"; +1 -> 0 "\xf6"; +1 -> 0 "\xf7"; +1 -> 0 "\xf8"; +1 -> 0 "\xf9"; +1 -> 0 "\xfa"; +1 -> 0 "\xfb"; +1 -> 0 "\xfc"; +1 -> 0 "\xfd"; +1 -> 0 "\xfe"; +1 -> 0 "\xff"; +2 -> 0 "\x00"; +2 -> 0 "\x01"; +2 -> 0 "\x02"; +2 -> 0 "\x03"; +2 -> 0 "\x04"; +2 -> 0 "\x05"; +2 -> 0 "\x06"; +2 -> 0 "\x07"; +2 -> 0 "\x08"; +2 -> 0 "\t"; +2 -> 0 "\n"; +2 -> 0 "\v"; +2 -> 0 "\f"; +2 -> 0 "\r"; +2 -> 0 "\x0e"; +2 -> 0 "\x0f"; +2 -> 0 "\x10"; +2 -> 0 "\x11"; +2 -> 0 "\x12"; +2 -> 0 "\x13"; +2 -> 0 "\x14"; +2 -> 0 "\x15"; +2 -> 0 "\x16"; +2 -> 0 "\x17"; +2 -> 0 "\x18"; +2 -> 0 "\x19"; +2 -> 0 "\x1a"; +2 -> 0 "\x1b"; +2 -> 0 "\x1c"; +2 -> 0 "\x1d"; +2 -> 0 "\x1e"; +2 -> 0 "\x1f"; +2 -> 0 " "; +2 -> 0 "!"; +2 -> 0 "\""; +2 -> 0 "#"; +2 -> 0 "$"; +2 -> 0 "%"; +2 -> 0 "&"; +2 -> 0 "'"; +2 -> 0 "("; +2 -> 0 ")"; +2 -> 0 "*"; +2 -> 0 "+"; +2 -> 0 ","; +2 -> 0 "-"; +2 -> 0 "."; +2 -> 0 "/"; +2 -> 0 "0"; +2 -> 0 "1"; +2 -> 0 "2"; +2 -> 0 "3"; +2 -> 0 "4"; +2 -> 0 "5"; +2 -> 0 "6"; +2 -> 0 "7"; +2 -> 0 "8"; +2 -> 0 "9"; +2 -> 0 ":"; +2 -> 0 ";"; +2 -> 0 "<"; +2 -> 0 "="; +2 -> 0 ">"; +2 -> 0 "?"; +2 -> 0 "@"; +2 -> 0 "A"; +2 -> 0 "B"; +2 -> 0 "C"; +2 -> 0 "D"; +2 -> 0 "E"; +2 -> 0 "F"; +2 -> 0 "G"; +2 -> 0 "H"; +2 -> 0 "I"; +2 -> 0 "J"; +2 -> 0 "K"; +2 -> 0 "L"; +2 -> 0 "M"; +2 -> 0 "N"; +2 -> 0 "O"; +2 -> 0 "P"; +2 -> 0 "Q"; +2 -> 0 "R"; +2 -> 0 "S"; +2 -> 0 "T"; +2 -> 0 "U"; +2 -> 0 "V"; +2 -> 0 "W"; +2 -> 0 "X"; +2 -> 0 "Y"; +2 -> 0 "Z"; +2 -> 0 "["; +2 -> 0 "\\"; +2 -> 0 "]"; +2 -> 0 "^"; +2 -> 0 "_"; +2 -> 0 "`"; +2 -> 1 "a"; +2 -> 0 "b"; +2 -> 0 "c"; +2 -> 0 "d"; +2 -> 0 "e"; +2 -> 0 "f"; +2 -> 0 "g"; +2 -> 0 "h"; +2 -> 0 "i"; +2 -> 0 "j"; +2 -> 0 "k"; +2 -> 0 "l"; +2 -> 0 "m"; +2 -> 0 "n"; +2 -> 0 "o"; +2 -> 0 "p"; +2 -> 0 "q"; +2 -> 0 "r"; +2 -> 0 "s"; +2 -> 0 "t"; +2 -> 0 "u"; +2 -> 0 "v"; +2 -> 0 "w"; +2 -> 0 "x"; +2 -> 0 "y"; +2 -> 0 "z"; +2 -> 0 "{"; +2 -> 0 "|"; +2 -> 0 "}"; +2 -> 0 "~"; +2 -> 0 "\x7f"; +2 -> 0 "\x80"; +2 -> 0 "\x81"; +2 -> 0 "\x82"; +2 -> 0 "\x83"; +2 -> 0 "\x84"; +2 -> 0 "\x85"; +2 -> 0 "\x86"; +2 -> 0 "\x87"; +2 -> 0 "\x88"; +2 -> 0 "\x89"; +2 -> 0 "\x8a"; +2 -> 0 "\x8b"; +2 -> 0 "\x8c"; +2 -> 0 "\x8d"; +2 -> 0 "\x8e"; +2 -> 0 "\x8f"; +2 -> 0 "\x90"; +2 -> 0 "\x91"; +2 -> 0 "\x92"; +2 -> 0 "\x93"; +2 -> 0 "\x94"; +2 -> 0 "\x95"; +2 -> 0 "\x96"; +2 -> 0 "\x97"; +2 -> 0 "\x98"; +2 -> 0 "\x99"; +2 -> 0 "\x9a"; +2 -> 0 "\x9b"; +2 -> 0 "\x9c"; +2 -> 0 "\x9d"; +2 -> 0 "\x9e"; +2 -> 0 "\x9f"; +2 -> 0 "\xa0"; +2 -> 0 "\xa1"; +2 -> 0 "\xa2"; +2 -> 0 "\xa3"; +2 -> 0 "\xa4"; +2 -> 0 "\xa5"; +2 -> 0 "\xa6"; +2 -> 0 "\xa7"; +2 -> 0 "\xa8"; +2 -> 0 "\xa9"; +2 -> 0 "\xaa"; +2 -> 0 "\xab"; +2 -> 0 "\xac"; +2 -> 0 "\xad"; +2 -> 0 "\xae"; +2 -> 0 "\xaf"; +2 -> 0 "\xb0"; +2 -> 0 "\xb1"; +2 -> 0 "\xb2"; +2 -> 0 "\xb3"; +2 -> 0 "\xb4"; +2 -> 0 "\xb5"; +2 -> 0 "\xb6"; +2 -> 0 "\xb7"; +2 -> 0 "\xb8"; +2 -> 0 "\xb9"; +2 -> 0 "\xba"; +2 -> 0 "\xbb"; +2 -> 0 "\xbc"; +2 -> 0 "\xbd"; +2 -> 0 "\xbe"; +2 -> 0 "\xbf"; +2 -> 0 "\xc0"; +2 -> 0 "\xc1"; +2 -> 0 "\xc2"; +2 -> 0 "\xc3"; +2 -> 0 "\xc4"; +2 -> 0 "\xc5"; +2 -> 0 "\xc6"; +2 -> 0 "\xc7"; +2 -> 0 "\xc8"; +2 -> 0 "\xc9"; +2 -> 0 "\xca"; +2 -> 0 "\xcb"; +2 -> 0 "\xcc"; +2 -> 0 "\xcd"; +2 -> 0 "\xce"; +2 -> 0 "\xcf"; +2 -> 0 "\xd0"; +2 -> 0 "\xd1"; +2 -> 0 "\xd2"; +2 -> 0 "\xd3"; +2 -> 0 "\xd4"; +2 -> 0 "\xd5"; +2 -> 0 "\xd6"; +2 -> 0 "\xd7"; +2 -> 0 "\xd8"; +2 -> 0 "\xd9"; +2 -> 0 "\xda"; +2 -> 0 "\xdb"; +2 -> 0 "\xdc"; +2 -> 0 "\xdd"; +2 -> 0 "\xde"; +2 -> 0 "\xdf"; +2 -> 0 "\xe0"; +2 -> 0 "\xe1"; +2 -> 0 "\xe2"; +2 -> 0 "\xe3"; +2 -> 0 "\xe4"; +2 -> 0 "\xe5"; +2 -> 0 "\xe6"; +2 -> 0 "\xe7"; +2 -> 0 "\xe8"; +2 -> 0 "\xe9"; +2 -> 0 "\xea"; +2 -> 0 "\xeb"; +2 -> 0 "\xec"; +2 -> 0 "\xed"; +2 -> 0 "\xee"; +2 -> 0 "\xef"; +2 -> 0 "\xf0"; +2 -> 0 "\xf1"; +2 -> 0 "\xf2"; +2 -> 0 "\xf3"; +2 -> 0 "\xf4"; +2 -> 0 "\xf5"; +2 -> 0 "\xf6"; +2 -> 0 "\xf7"; +2 -> 0 "\xf8"; +2 -> 0 "\xf9"; +2 -> 0 "\xfa"; +2 -> 0 "\xfb"; +2 -> 0 "\xfc"; +2 -> 0 "\xfd"; +2 -> 0 "\xfe"; +2 -> 0 "\xff"; + +start: 0; +end: 1, 2; From 09e569d3537af0d35b792b0c64e27b28f011bc05 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 28 Nov 2023 10:31:33 -0500 Subject: [PATCH 061/373] Add error code for unsupported PCRE cases (RE_EUNSUPPPCRE), reject one. There's extra error codes in #440 for regexes that aren't UNSATISFIABLE per se, but depend on particular corner cases in PCRE that probably aren't worth supporting in an automata-based implementation. Add a test case for one, tests/pcre/in48.re: ^a|$[^x]b* This is a tricky one to handle properly; according to PCRE it should match either "a" OR "\n", but nothing else. The newline match is because $ is a non-input-consuming check that evaluation is either at the end of input, or at a newline immediately before the end. In this case `$[^x]b*` matches exactly one newline; it's equivalent to "$\n". This probably isn't worth supporting, but we can detect cases where a potential newline match appears after a $ and reject them as an unsupported PCRE behavior. --- include/re/re.h | 2 + src/libre/ast.h | 1 + src/libre/ast_analysis.c | 103 +++++++++++++++++++++++++++++++++++++-- src/libre/ast_analysis.h | 4 +- src/libre/print/tree.c | 1 + src/libre/re.c | 10 +++- src/libre/strerror.c | 2 + tests/pcre/in48.re | 1 + tests/pcre/out48.err | 1 + 9 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 tests/pcre/in48.re create mode 100644 tests/pcre/out48.err diff --git a/include/re/re.h b/include/re/re.h index deab6caed..f876f635a 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -46,6 +46,8 @@ enum re_errno { RE_EERRNO = 1 | RE_MISC, RE_EBADDIALECT = 2 | RE_MISC, RE_EBADGROUP = 3 | RE_MISC, + RE_EUNSUPCAPTUR = 4 | RE_MISC, + RE_EUNSUPPPCRE = 5 | RE_MISC, RE_ENEGRANGE = 0 | RE_MARK | RE_GROUP, RE_ENEGCOUNT = 1 | RE_MARK | RE_GROUP, diff --git a/src/libre/ast.h b/src/libre/ast.h index 233744847..24031887e 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -98,6 +98,7 @@ enum ast_flags { AST_FLAG_ANCHORED_START = 1 << 6, AST_FLAG_ANCHORED_END = 1 << 7, AST_FLAG_END_NL = 1 << 8, + AST_FLAG_MATCHES_1NEWLINE= 1 << 9, AST_FLAG_NONE = 0x00 }; diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 7f7a50c6f..c368c6bb3 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -553,6 +553,46 @@ set_flags_subtree(struct ast_expr *n, enum ast_flags flags) } } +static int +can_consume_single_newline(struct ast_expr *n) +{ + if (!can_consume_input(n)) { return 0; } + + if (n->flags & AST_FLAG_MATCHES_1NEWLINE) { return 1; } + + switch (n->type) { + case AST_EXPR_LITERAL: + return n->u.literal.c == '\n'; + + case AST_EXPR_CODEPOINT: + return n->u.codepoint.u == (uint32_t)'\n'; + + case AST_EXPR_RANGE: + if ((n->u.range.from.type == AST_ENDPOINT_LITERAL) && + (n->u.range.to.type == AST_ENDPOINT_LITERAL)) { + return n->u.range.from.u.literal.c <= '\n' + && n->u.range.to.u.literal.c >= '\n'; + } else if ((n->u.range.from.type == AST_ENDPOINT_CODEPOINT) && + (n->u.range.to.type == AST_ENDPOINT_CODEPOINT)) { + return n->u.range.from.u.codepoint.u <= '\n' + && n->u.range.to.u.codepoint.u >= '\n'; + } else if (n->u.range.from.type == AST_ENDPOINT_NAMED) { + /* TODO: unreachable? */ + break; + } + break; + + case AST_EXPR_SUBTRACT: + return can_consume_single_newline(n->u.subtract.a) + && !can_consume_single_newline(n->u.subtract.b); + + default: + break; + } + + return 0; +} + struct anchoring_env { enum re_flags re_flags; @@ -563,6 +603,7 @@ struct anchoring_env { /* Corresponding flag for end anchors while sweeping backward. */ int followed_by_consuming; + int followed_by_consuming_newline; int before_start_anchor; }; @@ -645,6 +686,9 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_LITERAL: case AST_EXPR_CODEPOINT: case AST_EXPR_RANGE: + if (can_consume_single_newline(n)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } break; /* handled outside switch/case */ case AST_EXPR_CONCAT: { @@ -811,6 +855,13 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } } + for (i = 0; i < n->u.concat.count; i++) { + struct ast_expr *child = n->u.concat.n[i]; + if (can_consume_single_newline(child)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + } + break; } @@ -847,6 +898,9 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } else if (res == AST_ANALYSIS_OK) { all_set_past_always_consuming &= child_env.past_always_consuming; any_sat = 1; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE + || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + continue; } else { return res; } @@ -859,6 +913,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) all_end_anchored = 0; } } + + if (child->flags & AST_FLAG_MATCHES_1NEWLINE) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } } if (!env->past_always_consuming && all_set_past_always_consuming) { @@ -926,6 +984,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) return res; } + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_END && n->u.repeat.min > 0) { /* FIXME: if repeating something that is always * anchored at the end, repeat.max could be @@ -965,6 +1027,11 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } while(0) PROPAGATE_CHILD_FLAGS("GROUP", n, n->u.group.e); + + if (n->u.group.e->flags & AST_FLAG_MATCHES_1NEWLINE) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + break; case AST_EXPR_SUBTRACT: @@ -992,6 +1059,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } return res; } + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + break; default: @@ -1049,11 +1120,18 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) assert(n->flags & AST_FLAG_ANCHORED_END); if (env->followed_by_consuming) { - LOG(3 - LOG_ANCHORING, - "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", - __func__); - set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_UNSATISFIABLE; + if (env->followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: END anchor & followed_by_consuming, returning UNSUPPORTED_PCRE\n", + __func__); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } else { + LOG(3 - LOG_ANCHORING, + "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_UNSATISFIABLE; + } } break; @@ -1114,6 +1192,8 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) set_flags(n, AST_FLAG_UNSATISFIABLE); } } else if (res != AST_ANALYSIS_OK) { + LOG(3 - LOG_ANCHORING, + "%s: CONCAT: got res of %d, bubbling up\n", __func__, res); return res; } @@ -1129,6 +1209,15 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming = 1; } + if (!env->followed_by_consuming_newline && + (child_env.followed_by_consuming_newline + || child->flags & AST_FLAG_MATCHES_1NEWLINE)) { + LOG(3 - LOG_ANCHORING, + "%s: setting followed_by_consuming_newline due to child %p's analysis\n", + __func__, (void *)child); + env->followed_by_consuming_newline = 1; + } + if (!env->before_start_anchor && child_env.before_start_anchor && !is_nullable(child)) { LOG(3 - LOG_ANCHORING, @@ -1170,6 +1259,10 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) all_set_followed_by_consuming &= child_env.followed_by_consuming; all_set_before_start_anchor &= child_env.before_start_anchor; any_sat = 1; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE + || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED_*, bubbling up\n", __func__); + return res; } else { return res; } diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index f9ec8ebbb..5390cce57 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -30,7 +30,9 @@ enum ast_analysis_res { AST_ANALYSIS_UNSATISFIABLE, AST_ANALYSIS_ERROR_NULL = -1, - AST_ANALYSIS_ERROR_MEMORY = -2 + AST_ANALYSIS_ERROR_MEMORY = -2, + AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE = -3, + AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE = -4 }; enum ast_analysis_res diff --git a/src/libre/print/tree.c b/src/libre/print/tree.c index 5d2f78691..ccf50cb23 100644 --- a/src/libre/print/tree.c +++ b/src/libre/print/tree.c @@ -62,6 +62,7 @@ fprintf_flags(FILE *f, enum ast_flags flags) PR_FLAG(END_NL, "N"); PR_FLAG(CAN_CONSUME, "c"); PR_FLAG(ALWAYS_CONSUMES, "C"); + PR_FLAG(MATCHES_1NEWLINE, "n"); #undef PR_FLAG diff --git a/src/libre/re.c b/src/libre/re.c index 6c423dc36..5e4604aa3 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -125,7 +125,15 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, if (res < 0) { ast_free(ast); - if (err != NULL) { err->e = RE_EERRNO; } + if (err != NULL) { + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + err->e = RE_EUNSUPPPCRE; + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE) { + err->e = RE_EUNSUPCAPTUR; + } else if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + } + } return NULL; } diff --git a/src/libre/strerror.c b/src/libre/strerror.c index 009d61df2..d66e750a4 100644 --- a/src/libre/strerror.c +++ b/src/libre/strerror.c @@ -20,6 +20,8 @@ re_strerror(enum re_errno e) case RE_EERRNO: return strerror(errno); case RE_EBADDIALECT: return "Bad dialect"; case RE_EBADGROUP: return "Bad group"; + case RE_EUNSUPCAPTUR: return "Cannot support captures in this case"; + case RE_EUNSUPPPCRE: return "Unsupported PCRE edge case"; case RE_ENEGRANGE: return "Negative group range"; case RE_ENEGCOUNT: return "Negative count range"; diff --git a/tests/pcre/in48.re b/tests/pcre/in48.re new file mode 100644 index 000000000..15490c598 --- /dev/null +++ b/tests/pcre/in48.re @@ -0,0 +1 @@ +^a|$[^x]b* \ No newline at end of file diff --git a/tests/pcre/out48.err b/tests/pcre/out48.err new file mode 100644 index 000000000..b03d96d7e --- /dev/null +++ b/tests/pcre/out48.err @@ -0,0 +1 @@ +tests/pcre/in48.re: Unsupported PCRE edge case From 04be2af3ecc3fe17ab36600bb1ac25800c94f38d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 29 Nov 2023 13:07:32 -0500 Subject: [PATCH 062/373] Analysis/linkage updates to fix new pcre-anchor tests. --- src/libre/ast.h | 8 + src/libre/ast_analysis.c | 899 +++++++++++++++++++++++++++++++++++++-- src/libre/ast_compile.c | 49 ++- src/libre/print/tree.c | 2 + 4 files changed, 924 insertions(+), 34 deletions(-) diff --git a/src/libre/ast.h b/src/libre/ast.h index 24031887e..7583af6a9 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -100,6 +100,10 @@ enum ast_flags { AST_FLAG_END_NL = 1 << 8, AST_FLAG_MATCHES_1NEWLINE= 1 << 9, + /* FIXME: These need a description above */ + AST_FLAG_CONSTRAINED_AT_START = 1 << 10, + AST_FLAG_CONSTRAINED_AT_END = 1 << 11, + AST_FLAG_NONE = 0x00 }; @@ -160,6 +164,8 @@ struct ast_expr { size_t count; /* used */ size_t alloc; /* allocated */ struct ast_expr **n; + int contains_empty_groups; + int nullable_alt_inside_plus_repeat; } alt; struct { @@ -174,11 +180,13 @@ struct ast_expr { struct ast_expr *e; unsigned min; unsigned max; + int contains_empty_groups; } repeat; struct { struct ast_expr *e; unsigned id; + int repeated; /* set during analysis */ } group; struct { diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index c368c6bb3..829db0715 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -27,6 +27,9 @@ #define LOG_CONCAT_FLAGS (0 + LOG_ANALYSIS) #define LOG_UNANCHORED_FLAGS (0 + LOG_ANALYSIS) +#define LOG_REPETITION_CASES (0 + LOG_ANALYSIS) +#define LOG_PINCER_ANCHORS (0 + LOG_ANALYSIS) + #define LOG(LEVEL, ...) \ do { \ if ((LEVEL) <= LOG_ANALYSIS) { \ @@ -74,6 +77,15 @@ set_flags(struct ast_expr *n, enum ast_flags flags) n->flags |= flags; } +static int +is_unsatisfiable_or_skipped(const struct ast_expr *n) +{ + return (n->flags & AST_FLAG_UNSATISFIABLE) + || (n->type == AST_EXPR_REPEAT + && n->u.repeat.min == 0 + && n->u.repeat.max == 0); +} + static enum ast_analysis_res analysis_iter(struct ast_expr *n) { @@ -593,6 +605,41 @@ can_consume_single_newline(struct ast_expr *n) return 0; } +/* Does the subtree match a literal '\n'? */ +static int +matches_newline(const struct ast_expr *n) +{ + switch (n->type) { + case AST_EXPR_LITERAL: + return n->u.literal.c == '\n'; + + case AST_EXPR_SUBTRACT: + return matches_newline(n->u.subtract.a) + && !matches_newline(n->u.subtract.b); + + case AST_EXPR_RANGE: + { + const struct ast_endpoint *f = &n->u.range.from; + const struct ast_endpoint *t = &n->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL + || t->type != AST_ENDPOINT_LITERAL) { + /* not implemented */ + LOG(1 - LOG_ANCHORING, "%s: not implemented\n", __func__); + return 0; + } + + const int res = f->u.literal.c <= '\n' && t->u.literal.c >= '\n'; + LOG(1 - LOG_ANCHORING, "%s: RANGE res %d\n", __func__, res); + return res; + } + + default: + break; + } + + return 0; +} + struct anchoring_env { enum re_flags re_flags; @@ -603,9 +650,32 @@ struct anchoring_env { /* Corresponding flag for end anchors while sweeping backward. */ int followed_by_consuming; + + /* Special case for detecting '$[^a]', which matches "\n" with + * a capture group 0 of (0,1) in PCRE. */ int followed_by_consuming_newline; + /* Flag for tracking whether we're in a part of the subtree that + * is always before a start anchor. This influences satisfiability + * and edge cases like '()*^'. */ int before_start_anchor; + + /* Flag used to detect and reject the awkward case in '$[^a]', + * where (according to PCRE) the character class after the '$' + * should match the literal newline, but nothing else, and only + * once. Because $ is actually a zero-width assertion that + * execution is either at the end of input or a trailing + * newline, it has the rather surprising result that '$[^a]' + * will not match "x" but *will* match "x\n" (because it has a $ + * before a trailing newline, and because the newline matches + * the non-skippable [^a]). We just return an unsupported + * error for this case. */ + enum newline_after_end_anchor_state { + NAEAS_NONE, + NAEAS_WOULD_MATCH_ONCE, + } newline_after_end_anchor_state; + + int after_end_anchor; }; /* Tree walker that analyzes the AST, marks which nodes and subtrees are @@ -670,8 +740,14 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_ANCHOR_END: set_flags(n, AST_FLAG_ANCHORED_END); + env->after_end_anchor = 1; + LOG(3 - LOG_ANCHORING, + "%s: END anchor\n", __func__); if (n->u.anchor.is_end_nl && !(env->re_flags & RE_ANCHORED)) { set_flags(n, AST_FLAG_END_NL); + if (env->newline_after_end_anchor_state == NAEAS_NONE) { + env->newline_after_end_anchor_state = NAEAS_WOULD_MATCH_ONCE; + } } break; @@ -684,6 +760,19 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) * These are the types that actually consume input. */ case AST_EXPR_LITERAL: + if (n->u.literal.c == '\n') { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + + if (env->newline_after_end_anchor_state == NAEAS_WOULD_MATCH_ONCE) { + LOG(3 - LOG_ANCHORING, + "%s: LITERAL: rejecting non-optional newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } + } + break; + case AST_EXPR_CODEPOINT: case AST_EXPR_RANGE: if (can_consume_single_newline(n)) { @@ -748,6 +837,8 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) env->past_always_consuming = 1; } + env->newline_after_end_anchor_state = child_env.newline_after_end_anchor_state; + env->after_end_anchor = child_env.after_end_anchor; } /* flow ANCHORED_START and ANCHORED_END flags upward */ @@ -896,8 +987,14 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) __func__, i, (void *)n->u.alt.n[i]); assert(child->flags & AST_FLAG_UNSATISFIABLE); } else if (res == AST_ANALYSIS_OK) { - all_set_past_always_consuming &= child_env.past_always_consuming; - any_sat = 1; + if (child->type == AST_EXPR_REPEAT + && /*child->u.repeat.min == 0 &&*/ child->u.repeat.max == 0) { + LOG(3 - LOG_ANCHORING, + "%s: ignoring pruned REPEAT node\n", __func__); + } else { + all_set_past_always_consuming &= child_env.past_always_consuming; + any_sat = 1; + } } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { continue; @@ -905,6 +1002,14 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) return res; } + if (env->after_end_anchor + && always_consumes_input(child)) { + LOG(3 - LOG_ANCHORING, + "%s: ALT child %zd is after_end_anchor and always_consumes_input => UNSATISFIABLE\n", + __func__, i); + set_flags(child, AST_FLAG_UNSATISFIABLE); + } + if (!(child->flags & AST_FLAG_UNSATISFIABLE)) { /* ignore unsat nodes */ if (!(child->flags & AST_FLAG_ANCHORED_START)) { all_start_anchored = 0; @@ -950,8 +1055,18 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) break; } - case AST_EXPR_REPEAT: - res = analysis_iter_anchoring(env, n->u.repeat.e); + case AST_EXPR_REPEAT:; + const int orig_after_end_anchor = env->after_end_anchor; + + if (n->u.repeat.min == 0) { + /* Call the child with a copy of the anchoring env, + * since any subtree repeated zero or more times can + * always be ignored. */ + struct anchoring_env child_env = *env; + res = analysis_iter_anchoring(&child_env, n->u.repeat.e); + } else { + res = analysis_iter_anchoring(env, n->u.repeat.e); + } /* * This logic corresponds to the equivalent case for tombstone nodes @@ -988,16 +1103,29 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) set_flags(n, AST_FLAG_MATCHES_1NEWLINE); } + if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_START && n->u.repeat.min > 0) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: repeating ANCHORED_START subtree >0 times -> ANCHORED_START\n", __func__); + set_flags(n, AST_FLAG_ANCHORED_START); + n->u.repeat.max = 1; + } + if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_END && n->u.repeat.min > 0) { - /* FIXME: if repeating something that is always - * anchored at the end, repeat.max could be - * capped at 1, but I have not yet found any - * inputs where that change is necessary to - * produce a correct result. */ LOG(3 - LOG_ANCHORING, "%s: REPEAT: repeating ANCHORED_END subtree >0 times -> ANCHORED_END\n", __func__); set_flags(n, n->u.repeat.e->flags & END_ANCHOR_FLAG_MASK); + n->u.repeat.max = 1; } + + /* If there's a {0,_} repeat group after an end anchor that always consumes + * input, set its max count to 0. This should be based on the after_end_anchor + * flag value *before* analyzing the repeated subtree. */ + if (orig_after_end_anchor + && always_consumes_input(n) + && n->u.repeat.min == 0) { + n->u.repeat.max = 0; + } + break; case AST_EXPR_GROUP: @@ -1042,6 +1170,15 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.subtract.a->flags & AST_FLAG_ANCHORED_END) { set_flags(n, n->u.subtract.a->flags & END_ANCHOR_FLAG_MASK); } + + if (env->newline_after_end_anchor_state == NAEAS_WOULD_MATCH_ONCE) { + LOG(3 - LOG_ANCHORING, + "%s: SUBTRACT: rejecting non-optional newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } + if (res != AST_ANALYSIS_OK) { if (res == AST_ANALYSIS_UNSATISFIABLE) { set_flags(n, AST_FLAG_UNSATISFIABLE); @@ -1155,6 +1292,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_CONCAT: { size_t i; + int start_anchored = 0; for (i = n->u.concat.count; i > 0; i--) { struct ast_expr *child = n->u.concat.n[i - 1]; assert(child->type != AST_EXPR_TOMBSTONE); @@ -1197,6 +1335,12 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) return res; } + if (child->flags & AST_FLAG_ANCHORED_START) { + LOG(3 - LOG_ANCHORING, + "%s: CONCAT: child %zd is ANCHORED_START\n", __func__, i); + start_anchored = 1; + } + /* If we were previously not past any nodes that always * consume input, but the child's analysis set that flag, * then copy the setting if the child cannot be skipped. */ @@ -1218,6 +1362,13 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming_newline = 1; } + if (start_anchored) { + LOG(3 - LOG_ANCHORING, + "%s: earliest unskipped child had ANCHORED_START, bubbling up\n", + __func__); + set_flags(n, AST_FLAG_ANCHORED_START); + } + if (!env->before_start_anchor && child_env.before_start_anchor && !is_nullable(child)) { LOG(3 - LOG_ANCHORING, @@ -1233,6 +1384,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_ALT: { int any_sat = 0; int all_set_followed_by_consuming = 1; + int any_set_followed_by_consuming_newline = 0; int all_set_before_start_anchor = 1; assert(n->u.alt.count > 0); @@ -1256,9 +1408,16 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) __func__, i, (void *)n->u.alt.n[i]); assert(child->flags & AST_FLAG_UNSATISFIABLE); } else if (res == AST_ANALYSIS_OK) { - all_set_followed_by_consuming &= child_env.followed_by_consuming; - all_set_before_start_anchor &= child_env.before_start_anchor; - any_sat = 1; + if (0 && child->type == AST_EXPR_REPEAT + && child->u.repeat.max == 0) { + LOG(3 - LOG_ANCHORING, + "%s: ignoring pruned REPEAT node\n", __func__); + } else { + all_set_followed_by_consuming &= child_env.followed_by_consuming; + all_set_before_start_anchor &= child_env.before_start_anchor; + any_set_followed_by_consuming_newline |= child_env.followed_by_consuming_newline; + any_sat = 1; + } } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED_*, bubbling up\n", __func__); @@ -1275,11 +1434,19 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming = 1; } + if (!env->followed_by_consuming_newline && any_set_followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: ALT: any_set_followed_by_consuming_newline -> setting env->followed_by_consuming_newline for feature PCRE rejection\n", + __func__); + env->followed_by_consuming_newline = 1; + } + if (!env->before_start_anchor && all_set_before_start_anchor) { LOG(3 - LOG_ANCHORING, - "%s: ALT: all_set_before_start_anchor -> setting env->before_start_anchor\n", + "%s: ALT: all_set_before_start_anchor -> setting env->before_start_anchor and ANCHORED_START\n", __func__); env->before_start_anchor = 1; + set_flags(n, AST_FLAG_ANCHORED_START); } /* An ALT group is only unsatisfiable if they ALL are. */ @@ -1294,7 +1461,16 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) } case AST_EXPR_REPEAT: - res = analysis_iter_reverse_anchoring(env, n->u.repeat.e); + if (n->u.repeat.min == 0) { + /* Call the child with a copy of the anchoring env, + * since any subtree repeated zero or more times can + * always be ignored. */ + struct anchoring_env child_env = *env; + res = analysis_iter_reverse_anchoring(&child_env, n->u.repeat.e); + } else { + res = analysis_iter_reverse_anchoring(env, n->u.repeat.e); + } + if (res == AST_ANALYSIS_UNSATISFIABLE) { if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, @@ -1312,6 +1488,18 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: REPEAT: analysis on child returned %d\n", __func__, res); return res; } + + /* If there's a repeat group that always matches anything before a start anchor, + * set its max count to 0. */ + if (env->before_start_anchor + && is_nullable(n) + && always_consumes_input(n)) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: repeated group that consumes input before ^, setting max count to 0\n", + __func__); + n->u.repeat.max = 0; + } + break; case AST_EXPR_GROUP: @@ -1371,6 +1559,10 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) return AST_ANALYSIS_UNSATISFIABLE; } + if (n->flags & AST_FLAG_CAN_CONSUME && matches_newline(n)) { + env->followed_by_consuming_newline = 1; + } + return AST_ANALYSIS_OK; } @@ -1404,7 +1596,7 @@ assign_firsts(struct ast_expr *n) assign_firsts(child); if (can_consume_input(child) || (child->flags & AST_FLAG_ANCHORED_START)) { - break; + if (!is_nullable(child)) { break; } } } break; @@ -1421,7 +1613,7 @@ assign_firsts(struct ast_expr *n) } case AST_EXPR_REPEAT: - if (n->u.repeat.max > 0) { + if (n->u.repeat.max >= 0) { set_flags(n, AST_FLAG_FIRST); } @@ -1486,7 +1678,7 @@ assign_lasts(struct ast_expr *n) struct ast_expr *child = n->u.concat.n[i - 1]; assign_lasts(child); if (can_consume_input(child) || (child->flags & AST_FLAG_ANCHORED_END)) { - break; + if (!is_nullable(child)) { break; } } } @@ -1504,7 +1696,7 @@ assign_lasts(struct ast_expr *n) } case AST_EXPR_REPEAT: - if (n->u.repeat.max > 0) { + if (n->u.repeat.max >= 0) { set_flags(n, AST_FLAG_LAST); } @@ -1539,6 +1731,616 @@ assign_lasts(struct ast_expr *n) } } +static int +is_only_anchors(struct ast_expr *expr) +{ + if (can_consume_input(expr)) { return 0; } + + switch (expr->type) { + case AST_EXPR_ANCHOR: + return 1; + + case AST_EXPR_CONCAT: + if (expr->u.concat.count == 0) { return 0; } + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (!is_only_anchors(expr->u.concat.n[i]) + && can_consume_input(expr->u.concat.n[i])) { + return 0; + } + } + return 1; + + case AST_EXPR_ALT: + assert(expr->u.alt.count > 0); + for (size_t i = 0; i < expr->u.alt.count; i++) { + /* earlier matches will shadow later ones */ + if (is_only_anchors(expr->u.alt.n[i])) { + return 1; + } + } + return 0; + + case AST_EXPR_REPEAT: + if (expr->u.repeat.min == 0 && expr->u.repeat.max == 0) { + return 0; + } + return is_only_anchors(expr->u.repeat.e); + + case AST_EXPR_GROUP: + return is_only_anchors(expr->u.group.e); + + default: + break; + } + + return 0; +} + +/* Check for certain special cases with repetition that need to be + * handled carefully or rejected as unsupported. */ +static enum ast_analysis_res +analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_parent, + int shadowed_by_previous_alt_case, struct ast_expr *repeat_plus_ancestor) +{ + enum ast_analysis_res res = AST_ANALYSIS_OK; + + LOG(3 - LOG_REPETITION_CASES, "%s: node %p, type %s, shadowed_by_previous_alt_case %d\n", + __func__, (void *)n, ast_node_type_name(n->type), shadowed_by_previous_alt_case); + + if (shadowed_by_previous_alt_case) { + assert(outermost_repeat_parent == NULL + || outermost_repeat_parent->type == AST_EXPR_ALT); + } + + switch (n->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + case AST_EXPR_ANCHOR: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + break; + + case AST_EXPR_CONCAT: { + /* If this CONCAT array of nodes always consumes input, then + * it cannot be repeated empty, so it cannot produce the + * special case that needs outermost_repeat_parent for + * AST_EXPR_REPEAT's case below. + * + * An example input that needs this is 'x(()x)*' for "xx", + * because the 'x' prevents the outermost group + * from repeating and matching empty again after consuming + * a run of "x"s. */ + if (always_consumes_input(n)) { + outermost_repeat_parent = NULL; + } + + for (size_t i = 0; i < n->u.concat.count; i++) { + res = analysis_iter_repetition(n->u.concat.n[i], outermost_repeat_parent, + shadowed_by_previous_alt_case, repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + } + break; + } + + case AST_EXPR_ALT: { + /* See AST_EXPR_GROUP below for why this matters. */ + int new_shadowed_by_previous_alt_case = shadowed_by_previous_alt_case; + + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT node %p, repeat_plus_ancestor %p\n", + __func__, (void *)n, (void *)repeat_plus_ancestor); + + for (size_t i = 0; i < n->u.alt.count; i++) { + /* If this is an ALT inside of a repeated subtree that contains + * a capture, this will need special handling. */ + if (outermost_repeat_parent != NULL) { + LOG(3 - LOG_REPETITION_CASES, + "%s: setting outermost_repeat_parent to %p for alt branch %zu, repeat_plus_ancestor %p\n", + __func__, (void *)n, i, (void *)repeat_plus_ancestor); + outermost_repeat_parent = n; + } + + if (is_nullable(n->u.alt.n[i]) || is_only_anchors(n->u.alt.n[i])) { + LOG(3 - LOG_REPETITION_CASES, + "%s: setting new_shadowed_by_previous_alt_case for alt branch %zu, repeat_plus_ancestor %p\n", + __func__, i, (void *)repeat_plus_ancestor); + new_shadowed_by_previous_alt_case = 1; + if (repeat_plus_ancestor != NULL) { + n->u.alt.nullable_alt_inside_plus_repeat = 1; + assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT); + assert(repeat_plus_ancestor->u.repeat.min == 1); + assert(repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); + + /* Repetition of an alt subtree which has a capture group child that + * only contains only* anchors is not handled properly yet. This + * isn't actually _useful_, it's just something that comes up + * in fuzzing, so reject it as an unsupported PCRE construct. + * + * An example input that triggers this is '^(($)|)+$' . */ + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + } + } + + res = analysis_iter_repetition(n->u.alt.n[i], + outermost_repeat_parent, + new_shadowed_by_previous_alt_case, + repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + } + break; + } + + case AST_EXPR_REPEAT: + { + struct ast_expr *child = n->u.repeat.e; + + LOG(3 - LOG_REPETITION_CASES, "%s: REPEAT node %p, min %u max %u nullable? %d, !cci %d\n", + __func__, (void *)n, n->u.repeat.min, n->u.repeat.max, + is_nullable(child), can_consume_input(child)); + + if (n->u.repeat.min == 1 && + n->u.repeat.max == AST_COUNT_UNBOUNDED) { + LOG(3 - LOG_REPETITION_CASES, "%s: setting repeat_plus_ancestor to %p\n", + __func__, (void *)n); + repeat_plus_ancestor = n; + } else { + repeat_plus_ancestor = NULL; + } + + /* Special cases for a repeated group that contains possibly empty captures, + * in order to correctly reflect their repeating one more time and capture + * at the end (but without an infinite loop). + * + * For example, '^((x?))*$' will always end up with capture groups 1 and 2 + * at the end of the input for any number of "x"s, since the outermost ()* + * can always repeat once more time, consuming nothing, and clobber the + * existing captures. We mark repeated groups so that the compiled capture + * program can move saving the captures after the repetition, instead + * behaving like `^((?:x?)*(())$`. + * + * However, if the repeated subtree always consumes input, such as with + * '^(()a)+b$', then clear any passed in outermost_repeat_parent, because + * having to consume input will prevent that extra repetition of the + * empty captures. */ + if (always_consumes_input(n)) { + res = analysis_iter_repetition(child, NULL, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + } else if (outermost_repeat_parent == NULL && n->u.repeat.max > 1) { + LOG(3 - LOG_REPETITION_CASES, "%s: recursing with outermost_repeat_parent set to %p\n", + __func__, (void *)n); + res = analysis_iter_repetition(child, n, 0, + repeat_plus_ancestor); + } else { + LOG(3 - LOG_REPETITION_CASES, "%s: recursing with outermost_repeat_parent %p\n", + __func__, (void *)outermost_repeat_parent); + res = analysis_iter_repetition(child, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + } + if (res != AST_ANALYSIS_OK) { return res; } + break; + } + + case AST_EXPR_GROUP: + LOG(3 - LOG_REPETITION_CASES, + "%s: GROUP %p, repeat_plus_ancestor %p\n", + __func__, (void *)n, (void *)repeat_plus_ancestor); + + + if (outermost_repeat_parent != NULL && (is_nullable(n) || !can_consume_input(n))) { + int should_mark_repeated = 1; + /* If the outermost_repeat_parent is an ALT node and a previous ALT subtree + * matching the empty string is shadowing this group, then do not mark it + * as repeated, because that can lead to incorrect handling in somewhat + * contrived regexes like '^(?:|(|x))*$'. */ + if (outermost_repeat_parent->type == AST_EXPR_ALT && shadowed_by_previous_alt_case) { + LOG(3 - LOG_REPETITION_CASES, + "%s: hit group shadowed_by_previous_alt_case, skipping\n", __func__); + should_mark_repeated = 0; + } + + if (n->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END)) { + LOG(3 - LOG_REPETITION_CASES, + "%s: hit repeating anchor, skipping\n", __func__); + should_mark_repeated = 0; + } + + if (should_mark_repeated) { + LOG(3 - LOG_REPETITION_CASES, "%s: setting group %u to repeated\n", + __func__, n->u.group.id); + n->u.group.repeated = 1; + assert(outermost_repeat_parent->type == AST_EXPR_REPEAT || + outermost_repeat_parent->type == AST_EXPR_ALT); + LOG(3 - LOG_REPETITION_CASES, "%s: setting contains_empty_groups on outermost_repeat_parent %p\n", + __func__, (void *)outermost_repeat_parent); + if (outermost_repeat_parent->type == AST_EXPR_REPEAT) { + outermost_repeat_parent->u.repeat.contains_empty_groups = 1; + } else if (outermost_repeat_parent->type == AST_EXPR_ALT) { + outermost_repeat_parent->u.alt.contains_empty_groups = 1; + } else { + assert(!"type mismatch"); + } + } + } + + if (repeat_plus_ancestor != NULL && (is_nullable(n) || !can_consume_input(n))) { + assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT + && repeat_plus_ancestor->u.repeat.min == 1 + && repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); + LOG(3 - LOG_REPETITION_CASES, + "%s: not yet implemented, skipping\n", __func__); + /* return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; */ + } + + res = analysis_iter_repetition(n->u.group.e, outermost_repeat_parent, + shadowed_by_previous_alt_case, repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + break; + + case AST_EXPR_SUBTRACT: + res = analysis_iter_repetition(n->u.subtract.a, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + res = analysis_iter_repetition(n->u.subtract.b, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + break; + + default: + assert(!"unreached"); + } + return res; +} + +struct pincer_anchors_env { + int always_consumes_or_anchored_start; + int always_consumes_or_anchored_end; + + int alloc_fail; + size_t used; + size_t ceil; + struct ast_expr **nodes; +}; + +enum analysis_direction { AD_FORWARD, AD_BACKWARD }; + +static int +subtree_contains_anchor(const struct ast_expr *expr, enum ast_anchor_type type) +{ + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + case AST_EXPR_TOMBSTONE: + return 0; + + case AST_EXPR_ANCHOR: + return expr->u.anchor.type == type; + + case AST_EXPR_CONCAT: + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (subtree_contains_anchor(expr->u.concat.n[i], type)) { + return 1; + } + } + return 0; + case AST_EXPR_ALT: + for (size_t i = 0; i < expr->u.alt.count; i++) { + if (subtree_contains_anchor(expr->u.alt.n[i], type)) { + return 1; + } + } + return 0; + + case AST_EXPR_REPEAT: + if (expr->u.repeat.min > 0) { + return subtree_contains_anchor(expr->u.repeat.e, type); + } + return 0; + + case AST_EXPR_GROUP: + return subtree_contains_anchor(expr->u.group.e, type); + case AST_EXPR_SUBTRACT: + return subtree_contains_anchor(expr->u.subtract.a, type); + + default: + assert(!"match fail"); + } +} + +static void +pincer_anchors_record_anchor(struct pincer_anchors_env *env, struct ast_expr *n) +{ + if (env->used == env->ceil) { + const size_t nceil = (env->ceil == 0 ? 8 : 2*env->ceil); + struct ast_expr **nnodes = realloc(env->nodes, + nceil * sizeof(nnodes[0])); + if (nnodes == NULL) { + env->alloc_fail = 1; + return; + } + env->ceil = nceil; + env->nodes = nnodes; + } + + env->nodes[env->used++] = n; +} + +/* Check for regexes with out-of-order start and end anchors, combined + * with alternate subtrees that must consume input. A minimal example of + * this is `(a|$)(b|^)` (tests/pcre-anchor/in92.re) -- Either an 'a' is + * consumed or matching must be at the end of input (with optional + * '\n'), followed by either consuming a 'b' or being at the start of + * input. This should only match "", "\n", and "ab", + * but because the anchors are out of order it takes a further analysis + * pass and extra AST_FLAG_CONSTRAINED_AT_{START,END} node flags to + * ensure that neither is linked to the unanchored start/end loops. */ +static void +analysis_iter_pincer_anchors(struct ast_expr *expr, struct pincer_anchors_env *env, enum analysis_direction dir) +{ + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p %s\n", + __func__, (void *)expr, ast_node_type_name(expr->type)); + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + return; + + case AST_EXPR_ANCHOR: + if (expr->u.anchor.type == AST_ANCHOR_START) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_START, flags start %d, end %d, dir %s\n", + __func__, (void *)expr, + env->always_consumes_or_anchored_start, + env->always_consumes_or_anchored_end, + dir == AD_FORWARD ? "FORWARD" : "BACKWARD"); + if (dir == AD_FORWARD && env->always_consumes_or_anchored_end) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_START: recording CONSTRAINED_AT_END\n", + __func__, (void *)expr); + /* record for later -- these should only be set if there's an anchored *pair* */ + pincer_anchors_record_anchor(env, expr); + } + } else if (expr->u.anchor.type == AST_ANCHOR_END) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_END, flags start %d, end %d, dir %s\n", + __func__, (void *)expr, + env->always_consumes_or_anchored_start, + env->always_consumes_or_anchored_end, + dir == AD_FORWARD ? "FORWARD" : "BACKWARD"); + if (dir == AD_BACKWARD && env->always_consumes_or_anchored_start) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_END: recording CONSTRAINED_AT_START\n", + __func__, (void *)expr); + pincer_anchors_record_anchor(env, expr); + } + } + break; + + case AST_EXPR_CONCAT: + if (dir == AD_FORWARD) { + for (size_t i = 0; i < expr->u.concat.count; i++) { + analysis_iter_pincer_anchors(expr->u.concat.n[i], env, dir); + } + } else if (dir == AD_BACKWARD) { + for (size_t i = expr->u.concat.count; i > 0; i--) { + analysis_iter_pincer_anchors(expr->u.concat.n[i - 1], env, dir); + } + } + break; + case AST_EXPR_ALT: + { + int all_always_consume_or_anchored_start = 1; + int all_always_consume_or_anchored_end = 1; + int has_always_consuming = 0; + int has_non_consuming_anchored_start = 0; + int has_non_consuming_anchored_end = 0; + + if (dir == AD_FORWARD) { + for (size_t i = 0; i < expr->u.alt.count; i++) { + struct ast_expr *child = expr->u.alt.n[i]; + LOG(3 - LOG_PINCER_ANCHORS, + "%s: ALT child %zu/%zu : %p\n", + __func__, i, expr->u.alt.count, (void *)child); + if (is_unsatisfiable_or_skipped(child)) { continue; } + + analysis_iter_pincer_anchors(child, env, dir); + if (child->flags & AST_FLAG_ALWAYS_CONSUMES) { + has_always_consuming = 1; + } else { + if (child->flags & AST_FLAG_ANCHORED_START + && subtree_contains_anchor(child, AST_ANCHOR_START)) { + has_non_consuming_anchored_start = 1; + } else { + all_always_consume_or_anchored_start = 0; + } + + if (child->flags & AST_FLAG_ANCHORED_END + && subtree_contains_anchor(child, AST_ANCHOR_END)) { + has_non_consuming_anchored_end = 1; + } else { + all_always_consume_or_anchored_end = 0; + } + } + LOG(3 - LOG_PINCER_ANCHORS, + "%s: flags all_acoa_start %d, all_acoa_end %d, hac %d, hnca_start %d, hnca_end %d\n", + __func__, + all_always_consume_or_anchored_start, + all_always_consume_or_anchored_end, + has_always_consuming, + has_non_consuming_anchored_start, + has_non_consuming_anchored_end); + } + } else if (dir == AD_BACKWARD) { + for (size_t i = expr->u.alt.count; i > 0; i--) { + struct ast_expr *child = expr->u.alt.n[i - 1]; + LOG(3 - LOG_PINCER_ANCHORS, + "%s: ALT child %zu/%zu : %p\n", + __func__, i, expr->u.alt.count, (void *)child); + if (is_unsatisfiable_or_skipped(child)) { continue; } + + analysis_iter_pincer_anchors(child, env, dir); + if (child->flags & AST_FLAG_ALWAYS_CONSUMES) { + has_always_consuming = 1; + } else { + if (child->flags & AST_FLAG_ANCHORED_START + && subtree_contains_anchor(child, AST_ANCHOR_START)) { + has_non_consuming_anchored_start = 1; + } else { + all_always_consume_or_anchored_start = 0; + } + + if (child->flags & AST_FLAG_ANCHORED_END + && subtree_contains_anchor(child, AST_ANCHOR_END)) { + has_non_consuming_anchored_end = 1; + } else { + all_always_consume_or_anchored_end = 0; + } + } + LOG(3 - LOG_PINCER_ANCHORS, + "%s: flags all_acoa_start %d, all_acoa_end %d, hac %d, hnca_start %d, hnca_end %d\n", + __func__, + all_always_consume_or_anchored_start, + all_always_consume_or_anchored_end, + has_always_consuming, + has_non_consuming_anchored_start, + has_non_consuming_anchored_end); + } + } + + if (has_always_consuming + && has_non_consuming_anchored_start + && all_always_consume_or_anchored_start) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: ALT %p: setting env->always_consumes_or_anchored_start\n", + __func__, (void *)expr); + env->always_consumes_or_anchored_start = 1; + LOG(3 - LOG_PINCER_ANCHORS, "%s: %p: setting AST_FLAG_CONSTRAINED_AT_START\n", + __func__, (void *)expr); + set_flags(expr, AST_FLAG_CONSTRAINED_AT_START); + } + if (has_always_consuming + && has_non_consuming_anchored_end + && all_always_consume_or_anchored_end) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: ALT %p: setting env->always_consumes_or_anchored_end\n", + __func__, (void *)expr); + env->always_consumes_or_anchored_end = 1; + LOG(3 - LOG_PINCER_ANCHORS, "%s: %p: setting AST_FLAG_CONSTRAINED_AT_END\n", + __func__, (void *)expr); + set_flags(expr, AST_FLAG_CONSTRAINED_AT_END); + } + + break; + } + + case AST_EXPR_REPEAT: + analysis_iter_pincer_anchors(expr->u.repeat.e, env, dir); + break; + + case AST_EXPR_GROUP: + analysis_iter_pincer_anchors(expr->u.group.e, env, dir); + break; + case AST_EXPR_SUBTRACT: + /* It could just return here, as long as we aren't + * subtracting anything besides character sets. */ + analysis_iter_pincer_anchors(expr->u.subtract.a, env, dir); + break; + + case AST_EXPR_RANGE: + /* nothing to do */ + break; + + case AST_EXPR_TOMBSTONE: + break; + default: + assert(!"match fail"); + } +} + +static void +analysis_pincer_anchors_apply_flags_iter(struct ast_expr *expr) +{ + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p %s\n", + __func__, (void *)expr, ast_node_type_name(expr->type)); + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_ANCHOR: + break; + + case AST_EXPR_CONCAT: + for (size_t i = 0; i < expr->u.concat.count; i++) { + analysis_pincer_anchors_apply_flags_iter(expr->u.concat.n[i]); + } + break; + case AST_EXPR_ALT: + for (size_t i = 0; i < expr->u.alt.count; i++) { + analysis_pincer_anchors_apply_flags_iter(expr->u.alt.n[i]); + } + break; + + case AST_EXPR_REPEAT: + analysis_pincer_anchors_apply_flags_iter(expr->u.repeat.e); + break; + + case AST_EXPR_GROUP: + analysis_pincer_anchors_apply_flags_iter(expr->u.group.e); + break; + + case AST_EXPR_SUBTRACT: + /* it could just return here as long as we aren't + * subtracting anything besides character sets */ + analysis_pincer_anchors_apply_flags_iter(expr->u.subtract.a); + analysis_pincer_anchors_apply_flags_iter(expr->u.subtract.b); + break; + + case AST_EXPR_RANGE: + /* nothing to do */ + break; + + case AST_EXPR_TOMBSTONE: + break; + default: + assert(!"match fail"); + } +} + +static enum ast_analysis_res +analysis_pincer_anchors_apply_flags(struct ast_expr *expr, struct pincer_anchors_env *env) +{ + if (env->alloc_fail) { + return AST_ANALYSIS_ERROR_MEMORY; + } + + analysis_pincer_anchors_apply_flags_iter(expr); + + int has_start = 0; + int has_end = 0; + for (size_t i = 0; i < env->used; i++) { + const struct ast_expr *n = env->nodes[i]; + assert(n->type == AST_EXPR_ANCHOR); + if (n->u.anchor.type == AST_ANCHOR_START) { + has_start = 1; + } else if (n->u.anchor.type == AST_ANCHOR_END) { + has_end = 1; + } + } + + if (!has_start || !has_end) { + /* skip: only meaningful when constrained on both sides */ + return AST_ANALYSIS_OK; + } + + for (size_t i = 0; i < env->used; i++) { + struct ast_expr *n = env->nodes[i]; + assert(n->type == AST_EXPR_ANCHOR); + if (n->u.anchor.type == AST_ANCHOR_START) { + set_flags(n, AST_FLAG_CONSTRAINED_AT_END); + } else if (n->u.anchor.type == AST_ANCHOR_END) { + set_flags(n, AST_FLAG_CONSTRAINED_AT_START); + } + } + + return AST_ANALYSIS_OK; +} + enum ast_analysis_res ast_analysis(struct ast *ast, enum re_flags flags) { @@ -1567,28 +2369,61 @@ ast_analysis(struct ast *ast, enum re_flags flags) */ { /* first anchoring analysis pass, sweeping forward */ - struct anchoring_env env = { .re_flags = flags }; + struct anchoring_env env = { + .re_flags = flags, + .newline_after_end_anchor_state = NAEAS_NONE, + }; res = analysis_iter_anchoring(&env, ast->expr); if (res != AST_ANALYSIS_OK) { return res; } /* another anchoring analysis pass, sweeping backward */ res = analysis_iter_reverse_anchoring(&env, ast->expr); if (res != AST_ANALYSIS_OK) { return res; } + } - /* - * Next passes, mark all nodes in a first and/or last - * position. This is informed by the anchoring flags, so - * that needs to happen first. - */ - assign_firsts(ast->expr); - assign_lasts(ast->expr); - - ast->has_unanchored_start = (analysis_iter_unanchored_start(ast->expr) != UA_NO); - ast->has_unanchored_end = (analysis_iter_unanchored_end(ast->expr) != UA_NO); - LOG(2 - LOG_UNANCHORED_FLAGS, - "%s: has_unanchored_start %d, has_unanchored_end %d\n", - __func__, ast->has_unanchored_start, ast->has_unanchored_end); + /* + * Next passes, mark all nodes in a first and/or last + * position. This is informed by the anchoring flags, so + * that needs to happen first. + */ + assign_firsts(ast->expr); + assign_lasts(ast->expr); + + /* Next pass, mark some cases that need special handling + * due to repetition. For example, with cases like + * ^((x?))*$ the inner capture will always need to repeat + * one more time to match () after any 'x's. + * + * This needs to happen after the anchoring passes. */ + res = analysis_iter_repetition(ast->expr, NULL, 0, NULL); + if (res != AST_ANALYSIS_OK) { return res; } + + /* Based on all analysis thus far, check for another obscure + * edge case that is otherwise difficult to handle properly. + * This may set an extra flag on start/end anchor nodes. */ + { + struct pincer_anchors_env pincer_anchors_env = { .ceil = 0, }; + LOG(3 - LOG_PINCER_ANCHORS, "%s: -> analysis_iter_pincer_anchors FORWARD\n", __func__); + analysis_iter_pincer_anchors(ast->expr, &pincer_anchors_env, AD_FORWARD); + + LOG(3 - LOG_PINCER_ANCHORS, "%s: -> analysis_iter_pincer_anchors BACKWARD\n", __func__); + pincer_anchors_env.always_consumes_or_anchored_start = 0; + pincer_anchors_env.always_consumes_or_anchored_end = 0; + analysis_iter_pincer_anchors(ast->expr, &pincer_anchors_env, AD_BACKWARD); + + res = analysis_pincer_anchors_apply_flags(ast->expr, &pincer_anchors_env); + free(pincer_anchors_env.nodes); + + if (res != AST_ANALYSIS_OK) { + return res; + } } + ast->has_unanchored_start = (analysis_iter_unanchored_start(ast->expr) != UA_NO); + ast->has_unanchored_end = (analysis_iter_unanchored_end(ast->expr) != UA_NO); + LOG(2 - LOG_UNANCHORED_FLAGS, + "%s: has_unanchored_start %d, has_unanchored_end %d\n", + __func__, ast->has_unanchored_start, ast->has_unanchored_end); + return res; } diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index aa0902d55..959ef7f7e 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -51,13 +51,21 @@ enum link_side { * Link to the unanchored self loop adjacent to the start/end * state (env->start_any_loop or env->end_any_loop), because * this node is in a FIRST or LAST position, but unanchored. + * + * - LINK_SKIP + * This is used to prune linkage when anchoring leads to + * unsatisfiability. */ enum link_types { LINK_TOP_DOWN, LINK_GLOBAL, LINK_GLOBAL_SELF_LOOP, + LINK_SKIP, }; +/* Special case: Don't link an epsilon edge */ +#define SKIP_EPSILON ((fsm_state_t)-1) + struct comp_env { struct fsm *fsm; enum re_flags re_flags; @@ -410,6 +418,22 @@ decide_linking(struct comp_env *env, return LINK_TOP_DOWN; } + + /* Special case for anchors that are more constrained than normal -- search + * for "pincer_anchors" in ast_analysis.c for details. */ + if (n->type == AST_EXPR_ANCHOR) { + if (n->u.anchor.type == AST_ANCHOR_START) { + if (side == LINK_END && n->flags & AST_FLAG_CONSTRAINED_AT_END) { + return LINK_GLOBAL; + } + + } else if (n->u.anchor.type == AST_ANCHOR_END) { + if (side == LINK_START && n->flags & AST_FLAG_CONSTRAINED_AT_START) { + return LINK_SKIP; + } + } + } + /* parent can be NULL, if we're at the root node, but it must * never be the same node. */ assert(parent != n); @@ -467,6 +491,9 @@ print_linkage(enum link_types t) case LINK_GLOBAL_SELF_LOOP: fprintf(stderr, "[SELF_LOOP]"); break; + case LINK_SKIP: + fprintf(stderr, "[SKIP]"); + break; default: assert(!"match fail"); break; @@ -477,8 +504,9 @@ print_linkage(enum link_types t) if (!fsm_addstate(env->fsm, &(NAME))) { return 0; } #define EPSILON(FROM, TO) \ - assert((FROM) != (TO)); \ - if (!fsm_addedge_epsilon(env->fsm, (FROM), (TO))) { return 0; } + assert((FROM) != (TO) || ((FROM) == SKIP_EPSILON && (TO) == SKIP_EPSILON)); \ + if ((FROM) != SKIP_EPSILON && (TO) != SKIP_EPSILON \ + && !fsm_addedge_epsilon(env->fsm, (FROM), (TO))) { return 0; } #define ANY(FROM, TO) \ if (!fsm_addedge_any(env->fsm, (FROM), (TO))) { return 0; } @@ -641,6 +669,10 @@ comp_iter(struct comp_env *env, x = env->start_any_loop; break; + case LINK_SKIP: + x = SKIP_EPSILON; + y = SKIP_EPSILON; + break; default: assert(!"unreachable"); } @@ -654,6 +686,15 @@ comp_iter(struct comp_env *env, return 0; } y = env->end_nl; + } else if (n->type == AST_EXPR_ANCHOR + && n->u.anchor.type == AST_ANCHOR_START + && n->flags & AST_FLAG_CONSTRAINED_AT_END) { + /* the start anchor is constrained, so only + * link directly to '\n''s end here */ + if (!intern_end_nl(env)) { + return 0; + } + y = env->end_nl; } else { y = env->end; } @@ -666,6 +707,10 @@ comp_iter(struct comp_env *env, y = env->end_any_loop; break; + case LINK_SKIP: + x = SKIP_EPSILON; + y = SKIP_EPSILON; + break; default: assert(!"unreachable"); } diff --git a/src/libre/print/tree.c b/src/libre/print/tree.c index ccf50cb23..87eabf54e 100644 --- a/src/libre/print/tree.c +++ b/src/libre/print/tree.c @@ -63,6 +63,8 @@ fprintf_flags(FILE *f, enum ast_flags flags) PR_FLAG(CAN_CONSUME, "c"); PR_FLAG(ALWAYS_CONSUMES, "C"); PR_FLAG(MATCHES_1NEWLINE, "n"); + PR_FLAG(CONSTRAINED_AT_START, "["); + PR_FLAG(CONSTRAINED_AT_END, "]"); #undef PR_FLAG From 89b8469e98a88115196f1de715aedf0eab88152d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:06:02 -0500 Subject: [PATCH 063/373] tests/pcre-anchor/in{99,100}.re: '^(?:$^|a)(?:$^|^)*' and '$[^a]*^|$^' `^(?:$^|a)(?:$^|^)*`: This should be equivalent to `^(?:^$|a)` because the first group is always either fully anchored (making the second redundant) or must match 'a' (making it being anchored at the start impossible), but that information wasn't being passed properly to the second group. `$[^a]*` was returning AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE while inside the repeat node (`[]*`), so it didn't flag the `^` immediately after. If a repeat node that can match 0 times is unsupported, unsatisfiable, etc., just set its max count to 0 and continue. With this change analysis finds the `^` and treats the whole regex overall like `$^|$^`, which reduces to `^$`. The fixes for both of these will be integrated in later commits -- I'm integrating changes from a working branch that had several intermediate steps committed to push to another computer for fuzzing. --- tests/pcre-anchor/in100.re | 1 + tests/pcre-anchor/in99.re | 1 + tests/pcre-anchor/out100.fsm | 4 ++++ tests/pcre-anchor/out99.fsm | 8 ++++++++ 4 files changed, 14 insertions(+) create mode 100644 tests/pcre-anchor/in100.re create mode 100644 tests/pcre-anchor/in99.re create mode 100644 tests/pcre-anchor/out100.fsm create mode 100644 tests/pcre-anchor/out99.fsm diff --git a/tests/pcre-anchor/in100.re b/tests/pcre-anchor/in100.re new file mode 100644 index 000000000..8fea9b9ec --- /dev/null +++ b/tests/pcre-anchor/in100.re @@ -0,0 +1 @@ +$[^a]*^|$^ \ No newline at end of file diff --git a/tests/pcre-anchor/in99.re b/tests/pcre-anchor/in99.re new file mode 100644 index 000000000..5c385ad3e --- /dev/null +++ b/tests/pcre-anchor/in99.re @@ -0,0 +1 @@ +^(?:$^|a)(?:$^|^)* \ No newline at end of file diff --git a/tests/pcre-anchor/out100.fsm b/tests/pcre-anchor/out100.fsm new file mode 100644 index 000000000..9ad9510cb --- /dev/null +++ b/tests/pcre-anchor/out100.fsm @@ -0,0 +1,4 @@ +0 -> 1 "\n"; + +start: 0; +end: 0, 1; diff --git a/tests/pcre-anchor/out99.fsm b/tests/pcre-anchor/out99.fsm new file mode 100644 index 000000000..43ecc6bce --- /dev/null +++ b/tests/pcre-anchor/out99.fsm @@ -0,0 +1,8 @@ +# '^(?:$^|a)(?:$^|^)*' is effectively the same as '^(?:$^|a)'. + +0 -> 2 "\n"; +0 -> 1 "a"; +1 -> 1 ?; + +start: 0; +end: 0, 1, 2; From 4e86c0c0587395993eb258601ba35c5d56e45baf Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:11:02 -0500 Subject: [PATCH 064/373] ast_analysis: Fix for tests/pcre-anchor/in100.fsm. If a repeat node's subtree returns an error but has a min count of 0, then prune it by setting its max count to 0. If its error is returned directly then the analysis won't see nodes that appear after it. --- src/libre/ast_analysis.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 829db0715..3f947c02a 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1094,9 +1094,15 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) return AST_ANALYSIS_UNSATISFIABLE; } } else if (res != AST_ANALYSIS_OK) { - LOG(3 - LOG_ANCHORING, - "%s: REPEAT: analysis on child returned %d\n", __func__, res); - return res; + if (n->u.repeat.min == 0) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: analysis on child returned %d, setting repeat count to 0\n", __func__, res); + n->u.repeat.max = 0; + } else { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: analysis on child returned %d\n", __func__, res); + return res; + } } if (can_consume_single_newline(n->u.repeat.e)) { From d6eecff9f3a852c066e716e0101e3567c6b5e98a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 30 Nov 2023 10:25:04 -0500 Subject: [PATCH 065/373] ast_analysis: Update incorrect assertion and comment. --- src/libre/ast_analysis.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 3f947c02a..5f3fd168a 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1259,8 +1259,11 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) break; case AST_ANCHOR_END: - /* should already be set during forward pass */ - assert(n->flags & AST_FLAG_ANCHORED_END); + /* This should already be set during forward pass, but + * may have been skipped when evaluation moving forward + * rejected this subtree as unsatisfiable (but evaluating + * moving in reverse hasn't). */ + set_flags(n, AST_FLAG_ANCHORED_END); if (env->followed_by_consuming) { if (env->followed_by_consuming_newline) { From f1f5ff6018538de1c93446c16f51af28468b4876 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:15:18 -0500 Subject: [PATCH 066/373] Silence warnings about unused (log-only) variables. --- src/adt/idmap.c | 2 ++ src/adt/internedstateset.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/adt/idmap.c b/src/adt/idmap.c index ca169b71e..342ecd136 100644 --- a/src/adt/idmap.c +++ b/src/adt/idmap.c @@ -387,6 +387,8 @@ idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, } } assert(block_count == check_count); +#else + (void)block_count; #endif } } diff --git a/src/adt/internedstateset.c b/src/adt/internedstateset.c index e6e3a2b8c..a4f6c5bd1 100644 --- a/src/adt/internedstateset.c +++ b/src/adt/internedstateset.c @@ -364,6 +364,8 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, #if LOG_ISS > 3 fprintf(stderr, "%s: miss after %zd probes\n", __func__, probes); +#else + (void)probes; #endif From 1dc2b2b47d022d7d9c91a7ca26473408d34cf1c7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:17:40 -0500 Subject: [PATCH 067/373] ast_analysis: Always check `analysis_iter`'s result. Found while fuzzing. --- src/libre/ast_analysis.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 5f3fd168a..91d7e326b 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -117,7 +117,9 @@ analysis_iter(struct ast_expr *n) (void *)n, (void *)child); set_flags(child, AST_FLAG_NULLABLE); } - analysis_iter(child); + enum ast_analysis_res child_res = analysis_iter(child); + if (child_res != AST_ANALYSIS_OK) { return child_res; } + if (can_consume_input(child)) { any_can_consume = 1; } @@ -156,7 +158,8 @@ analysis_iter(struct ast_expr *n) for (i = 0; i < n->u.alt.count; i++) { struct ast_expr *child = n->u.alt.n[i]; - analysis_iter(child); + enum ast_analysis_res child_res = analysis_iter(child); + if (child_res != AST_ANALYSIS_OK) { return child_res; } /* spread nullability upward */ if (is_nullable(child)) { @@ -203,7 +206,8 @@ analysis_iter(struct ast_expr *n) set_flags(n, AST_FLAG_NULLABLE); } - analysis_iter(e); + enum ast_analysis_res child_res = analysis_iter(e); + if (child_res != AST_ANALYSIS_OK) { return child_res; } set_flags(n, e->flags & AST_FLAG_CAN_CONSUME); if (n->u.repeat.min > 0 && e->flags & AST_FLAG_ALWAYS_CONSUMES) { @@ -242,7 +246,8 @@ analysis_iter(struct ast_expr *n) set_flags(e, AST_FLAG_NULLABLE); } - analysis_iter(e); + enum ast_analysis_res child_res = analysis_iter(e); + if (child_res != AST_ANALYSIS_OK) { return child_res; } set_flags(n, e->flags & (AST_FLAG_CAN_CONSUME | AST_FLAG_ALWAYS_CONSUMES)); if (is_nullable(e)) { From 411dd38d728e5ae6a2d303dad65337d9f2ada37c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:18:20 -0500 Subject: [PATCH 068/373] re.c: `AST_ANALYSIS_ERROR_MEMORY` => Set err->e to `RE_EERRNO`. This should normally be set when allocation fails, but when built for fuzzing analysis can pre-emptively reject inputs that would need excessive memory due to large repetition counts (for example), so in that case make sure an error code is set to avoid triggering an assertion later on. --- src/libre/re.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libre/re.c b/src/libre/re.c index 5e4604aa3..19703c0c8 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -128,6 +128,12 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, if (err != NULL) { if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { err->e = RE_EUNSUPPPCRE; + } else if (res == AST_ANALYSIS_ERROR_MEMORY) { + /* This case comes up during fuzzing. */ + if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + errno = ENOMEM; + } } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE) { err->e = RE_EUNSUPCAPTUR; } else if (err->e == RE_ESUCCESS) { From 3474d9e29c3df25460effe6b5ef393981a52a945 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:45:56 -0500 Subject: [PATCH 069/373] ast_compile: ALT nodes should pass top-down linking along unmodified. Lots of other bugs in linking have been indirect consequences of an ancestor ALT node that changed its top-down x,y linkage to global or the global start loop instead. Avoiding modification here eliminates a lot of noise. While trying to simultaneously satisfy the pcre-anchor tests 92, 99, and 100 I took a step back, thought through the system as a whole, and realized this was where their linkage was getting messed up. --- src/libre/ast_compile.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 959ef7f7e..b64edd290 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -418,6 +418,11 @@ decide_linking(struct comp_env *env, return LINK_TOP_DOWN; } + /* An alt node should always pass along the top-down (x,y) pair + * unmodified, individual subtrees may override the linking. */ + if (n->type == AST_EXPR_ALT) { + return LINK_TOP_DOWN; + } /* Special case for anchors that are more constrained than normal -- search * for "pincer_anchors" in ast_analysis.c for details. */ From 94843197a9ca22c63dca81380cde53738aac44b7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 11:56:11 -0500 Subject: [PATCH 070/373] ast.h: Add comments for AST_FLAG_CONSTRAINED_AT_{START,END}. --- src/libre/ast.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libre/ast.h b/src/libre/ast.h index 7583af6a9..c488ee392 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -86,6 +86,13 @@ enum ast_anchor_type { * ends with the PCRE end anchor that implicitly matches a single * trailing newline. * + * - AST_FLAG_CONSTRAINED_AT_START + * The anchor needs more restrictive linkage on its start side, + * see ast_analysis's "pincer_anchors" analysis for details. + * + * - AST_FLAG_CONSTRAINED_AT_END + * End counterpart to AST_FLAG_CONSTRAINED_AT_START. + * * Not all are valid for all node types. */ enum ast_flags { @@ -99,8 +106,6 @@ enum ast_flags { AST_FLAG_ANCHORED_END = 1 << 7, AST_FLAG_END_NL = 1 << 8, AST_FLAG_MATCHES_1NEWLINE= 1 << 9, - - /* FIXME: These need a description above */ AST_FLAG_CONSTRAINED_AT_START = 1 << 10, AST_FLAG_CONSTRAINED_AT_END = 1 << 11, From 95b1915b06dc85bb7a22bb5aed082bd4bb6c3bc2 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 12:16:17 -0500 Subject: [PATCH 071/373] ast_analysis: Remove tautological comparisons (unsigned >= 0). --- src/libre/ast_analysis.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 91d7e326b..987e0b86f 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1627,9 +1627,7 @@ assign_firsts(struct ast_expr *n) } case AST_EXPR_REPEAT: - if (n->u.repeat.max >= 0) { - set_flags(n, AST_FLAG_FIRST); - } + set_flags(n, AST_FLAG_FIRST); /* Don't recurse. * @@ -1710,9 +1708,7 @@ assign_lasts(struct ast_expr *n) } case AST_EXPR_REPEAT: - if (n->u.repeat.max >= 0) { - set_flags(n, AST_FLAG_LAST); - } + set_flags(n, AST_FLAG_LAST); /* Don't recurse. * From 0f5a966c2493bc00d742dbeeb46e05241019b7ab Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 20 Dec 2023 12:16:44 -0500 Subject: [PATCH 072/373] ast_analysis: Add `prune_repeat_node`. Rather than modifying this directly, add a function, which has its own asserts and logging. --- src/libre/ast_analysis.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 987e0b86f..bc5bc6bed 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -86,6 +86,15 @@ is_unsatisfiable_or_skipped(const struct ast_expr *n) && n->u.repeat.max == 0); } +static void +prune_repeat_node(struct ast_expr *n) +{ + assert(n->type == AST_EXPR_REPEAT && n->u.repeat.min == 0); + LOG(3, "%s: setting REPEAT node %p's max count to 0\n", + __func__, (void *)n); + n->u.repeat.max = 0; +} + static enum ast_analysis_res analysis_iter(struct ast_expr *n) { @@ -937,7 +946,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (child->type == AST_EXPR_REPEAT && (child->flags & AST_FLAG_UNSATISFIABLE) && child->u.repeat.min == 0) { - child->u.repeat.max = 0; + prune_repeat_node(child); } } else if (!after_end_anchor && child->flags & AST_FLAG_ANCHORED_END @@ -1090,7 +1099,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, "%s: REPEAT: UNSATISFIABLE but can be repeated 0 times, ignoring\n", __func__); - n->u.repeat.max = 0; + prune_repeat_node(n); break; } else if (n->u.repeat.min > 0) { set_flags(n, AST_FLAG_UNSATISFIABLE); @@ -1102,7 +1111,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, "%s: REPEAT: analysis on child returned %d, setting repeat count to 0\n", __func__, res); - n->u.repeat.max = 0; + prune_repeat_node(n); } else { LOG(3 - LOG_ANCHORING, "%s: REPEAT: analysis on child returned %d\n", __func__, res); @@ -1134,7 +1143,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (orig_after_end_anchor && always_consumes_input(n) && n->u.repeat.min == 0) { - n->u.repeat.max = 0; + prune_repeat_node(n); } break; @@ -1489,7 +1498,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, "%s: REPEAT: UNSATISFIABLE but can be repeated 0 times, ignoring\n", __func__); - n->u.repeat.max = 0; /* skip */ + prune_repeat_node(n); break; } else if (n->u.repeat.min > 0) { LOG(3 - LOG_ANCHORING, @@ -1511,7 +1520,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) LOG(3 - LOG_ANCHORING, "%s: REPEAT: repeated group that consumes input before ^, setting max count to 0\n", __func__); - n->u.repeat.max = 0; + prune_repeat_node(n); } break; From 7dd550ab0ed57fe925f12ee463eaba2d576c2408 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 19 Dec 2023 13:42:11 -0500 Subject: [PATCH 073/373] Improve performance when hashing sets of state IDs. Add probe count logging to `to_set_htab_check`. Switch from adding each PHI_64*(id+1) to xor-ing, it has better collision behavior. This change makes time ./re -rpcre 'a[^bz][^bz][^bz][^b][^bz][^bz][^bz][^bz][^bz][^bz][^b][^bz][^bz][^bz][^bz][^bz]' go from taking ~14 seconds to ~6 when I run it locally, because adding leads to much higher probe counts on average. --- src/adt/internedstateset.c | 2 +- src/libfsm/determinise.c | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/adt/internedstateset.c b/src/adt/internedstateset.c index a4f6c5bd1..a39f09a19 100644 --- a/src/adt/internedstateset.c +++ b/src/adt/internedstateset.c @@ -203,7 +203,7 @@ hash_state_ids(size_t count, const fsm_state_t *ids) { uint64_t h = 0; for (size_t i = 0; i < count; i++) { - h = hash_id(ids[i]) + (FSM_PHI_64 * h); + h ^= hash_id(ids[i]); } return h; } diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 9c77f3a42..b6c688ac4 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1315,7 +1315,7 @@ to_set_hash(size_t count, const fsm_state_t *ids) { uint64_t h = hash_id(count); for (size_t i = 0; i < count; i++) { - h += hash_id(ids[i]); + h ^= hash_id(ids[i]); } return h; } @@ -1333,19 +1333,28 @@ to_set_htab_check(struct analyze_closures_env *env, return 0; } + +#if LOG_TO_SET_HTAB + size_t probes = 0; +#endif + int res = 0; + const uint64_t mask = bcount - 1; for (size_t b_i = 0; b_i < bcount; b_i++) { +#if LOG_TO_SET_HTAB + probes++; +#endif const struct to_set_bucket *b = &htab->buckets[(h + b_i) & mask]; if (b->count == 0) { - return 0; /* empty bucket -> not found */ + goto done; /* empty bucket -> not found */ } else if (b->count == count) { assert(env->to_sets.buf != NULL); assert(b->offset + count <= env->to_sets.used); const fsm_state_t *ids = &env->to_sets.buf[b->offset]; if (0 == memcmp(ids, dst, count * sizeof(dst[0]))) { *out_offset = b->offset; - - return 1; /* cache hit */ + res = 1; /* cache hit */ + goto done; } else { continue; /* collision */ } @@ -1354,7 +1363,13 @@ to_set_htab_check(struct analyze_closures_env *env, } } - return 0; +done: +#if LOG_TO_SET_HTAB + fprintf(stderr, "%s: result %d, %zu probes, htab: used %zu/%zu ceil\n", + __func__, res, probes, htab->buckets_used, htab->bucket_count); +#endif + + return res; } static int From 84d7da40d04799e94202a4f19ab69b20289c3508 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 3 Jan 2024 14:39:00 -0500 Subject: [PATCH 074/373] hash_id: Change to xorshift*. katef's testing with words.sh found some suspicious timing, profiling with callgrind showed there's still some kind of bad collision behavior doing PHI64(a) ^ PHI64(b) with exactly two IDs. It's probably still a bad idea to combine multiple Fibonacci hashes, even with xor-ing rather than adding. Changing to xorshift* (another fast, high quality hash function for 64-bit ints) immediately makes the issue go away, so do that. --- include/adt/hash.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/adt/hash.h b/include/adt/hash.h index d4ed1227d..74ee4a890 100644 --- a/include/adt/hash.h +++ b/include/adt/hash.h @@ -15,14 +15,17 @@ #define FSM_PHI_32 0x9e3779b9UL #define FSM_PHI_64 (uint64_t)0x9e3779b97f4a7c15UL -/* A suitable hash function for individual sequentially allocated - * identifiers. See Knuth 6.4, Fibonacci hashing. */ - SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static __inline__ uint64_t hash_id(unsigned id) { - return FSM_PHI_64 * (uint64_t)(id + (unsigned)1); + /* xorshift* A1(12,25,27), + * from http://vigna.di.unimi.it/ftp/papers/xorshift.pdf */ + uint64_t x = id + 1; + x ^= x >> 12; // a + x ^= x << 25; // b + x ^= x >> 27; // c + return x * 2685821657736338717LLU; } /* FNV-1a hash function, 32 and 64 bit versions. This is in the public From 18972777028e0c3f41f9ca1df0ec67cee75bc448 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 4 Jan 2024 15:43:56 +0000 Subject: [PATCH 075/373] Handling for the initial token being TOK_UNKNOWN. My intended idea was to have all syntax errors go through the same actions driven by parser.sid. But in this case SID's generated parser bails out immediately on TOK_UNKNOWN, and we never reach an ## alt. So we never reach , and thus never about it. That's what the assertion was about; the generated parser should never return with a NULL AST. My workaround for this is just to recreate the equivalent syntax error at the top-level, which isn't very satisfying at all. But it does what we need here. Spotted by @cinco-de-mayonnaise, thank you. --- src/lx/parser.act | 14 +++++++++++++- src/lx/parser.c | 16 ++++++++++++++-- src/lx/parser.h | 2 +- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/lx/parser.act b/src/lx/parser.act index 78254e123..f9b0b32f9 100644 --- a/src/lx/parser.act +++ b/src/lx/parser.act @@ -869,7 +869,19 @@ lx->free(lx->buf_opaque); - assert(ast != NULL); + /* + * This can happen when the first token is TOK_UNKNOWN. + * SID's generated parser bails out immediately. + * So we never reach , and never about it. + * + * Really I wanted this handled along with the usual syntax error + * case, from ## alts inside parser.sid, instead of reproducing + * the same error here. + */ + if (ast == NULL) { + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + } return ast; } diff --git a/src/lx/parser.c b/src/lx/parser.c index cbc2067fd..dfcf36b8f 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -3364,11 +3364,23 @@ ZL1:; lx->free(lx->buf_opaque); - assert(ast != NULL); + /* + * This can happen when the first token is TOK_UNKNOWN. + * SID's generated parser bails out immediately. + * So we never reach , and never about it. + * + * Really I wanted this handled along with the usual syntax error + * case, from ## alts inside parser.sid, instead of reproducing + * the same error here. + */ + if (ast == NULL) { + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + } return ast; } -#line 3373 "src/lx/parser.c" +#line 3385 "src/lx/parser.c" /* END OF FILE */ diff --git a/src/lx/parser.h b/src/lx/parser.h index fbe704fc4..bde7c36ab 100644 --- a/src/lx/parser.h +++ b/src/lx/parser.h @@ -29,7 +29,7 @@ extern void p_lx(lex_state, act_state, ast *); /* BEGINNING OF TRAILER */ -#line 877 "src/lx/parser.act" +#line 889 "src/lx/parser.act" #line 36 "src/lx/parser.h" From 16957a21baaedecc990a2943aee26b98d9f8e9f1 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jan 2024 09:11:07 -0500 Subject: [PATCH 076/373] Remove duplicated struct field introduced during merge. --- src/libre/ast_analysis.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 535f96a12..850735b30 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -664,7 +664,6 @@ struct anchoring_env { /* Corresponding flag for end anchors while sweeping backward. */ int followed_by_consuming; - int followed_by_consuming_newline; /* Special case for detecting '$[^a]', which matches "\n" with * a capture group 0 of (0,1) in PCRE. */ From ad0695e64b56beae4683cb0b756518091a96f18e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 3 Jan 2024 16:05:40 -0500 Subject: [PATCH 077/373] hash_id: Rather than adding the IDs, combine into one uint64_t. - hash_id should take a uint64_t argument, rather than unsigned. - Instead of adding them or hashing them separately and combining, pack both into the uint64_t argument for hash_id, since each is a 32-bit ID. Further experimentation supports that this has better collision behavior. --- include/adt/hash.h | 2 +- src/libfsm/determinise.c | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/include/adt/hash.h b/include/adt/hash.h index 74ee4a890..c2b7a0683 100644 --- a/include/adt/hash.h +++ b/include/adt/hash.h @@ -17,7 +17,7 @@ SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static __inline__ uint64_t -hash_id(unsigned id) +hash_id(uint64_t id) { /* xorshift* A1(12,25,27), * from http://vigna.di.unimi.it/ftp/papers/xorshift.pdf */ diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index b6c688ac4..5f6a984bc 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1645,16 +1645,9 @@ hash_pair(fsm_state_t a, fsm_state_t b) assert(b & RESULT_BIT); a &=~ RESULT_BIT; b &=~ RESULT_BIT; - - /* Don't hash a and b separately and combine them with - * hash_id, because it's common to have adjacent pairs of - * result IDs, and with how hash_id works that leads to - * multiples of similar hash values bunching up. - * - * This could be replaced with a better hash function later, - * but use LOG_CACHE_HTAB to ensure there aren't visually obvious - * runs of collisions appearing in the tables. */ - const uint64_t res = hash_id(a + b); + assert(a != b); + const uint64_t ab = ((uint64_t)a << 32) | (uint64_t)b; + const uint64_t res = hash_id(ab); /* fprintf(stderr, "%s: a %d, b %d -> %016lx\n", __func__, a, b, res); */ return res; } From 2b443515b30fc1ebaca5d1771ee5efac0e4b7010 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 4 Jan 2024 14:14:46 -0500 Subject: [PATCH 078/373] stateset: Remove unused `state_set_hash`. --- include/adt/stateset.h | 3 --- src/adt/stateset.c | 16 ---------------- 2 files changed, 19 deletions(-) diff --git a/include/adt/stateset.h b/include/adt/stateset.h index 83e835467..3ec91794b 100644 --- a/include/adt/stateset.h +++ b/include/adt/stateset.h @@ -72,8 +72,5 @@ state_set_rebase(struct state_set **set, fsm_state_t base); void state_set_replace(struct state_set **set, fsm_state_t old, fsm_state_t new); -unsigned long -state_set_hash(const struct state_set *set); - #endif diff --git a/src/adt/stateset.c b/src/adt/stateset.c index 8d73a9b5f..c2cc8b8c0 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -668,19 +668,3 @@ state_set_replace(struct state_set **setp, fsm_state_t old, fsm_state_t new) } } } - -unsigned long -state_set_hash(const struct state_set *set) -{ - if (set == NULL) { - return 0; /* empty */ - } - - if (IS_SINGLETON(set)) { - fsm_state_t state; - state = SINGLETON_DECODE(set); - return hashrec(&state, sizeof state); - } - - return hashrec(set->a, set->i * sizeof *set->a); -} From c776e62b2d705c1383a0185451757c8424cfe2c7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 4 Jan 2024 14:42:42 -0500 Subject: [PATCH 079/373] Add include/adt/hash.h `hash_ids` and consolidate uses. - Add a function to hash a series of IDs, rather than doing it with a loop combining the intermediate hashes in inconsistent ways. - Add a #define to log a probe count and a limit to assert, and rework the hashing code control flow for tracking/asserting on the probe count. - Remove `hash_fnv1a_64` from a logging statement (normally not compiled), since that's the only remaining use remove the FNV1a functions from hash.h. - Remove FSM_PHI_32 and FSM_PHI_64, now unused. - parser.act still uses 32-bit FNV1a, but that's for character-by- character hashing, so that still makes sense, and it has its own copy of fnv1a. --- include/adt/hash.h | 39 +++++++---------------------------- src/adt/idmap.c | 27 +++++++++++++++++------- src/adt/internedstateset.c | 18 ++++++++-------- src/libfsm/determinise.c | 42 ++++++++++++++++++++++++++------------ 4 files changed, 65 insertions(+), 61 deletions(-) diff --git a/include/adt/hash.h b/include/adt/hash.h index c2b7a0683..bd1cf2907 100644 --- a/include/adt/hash.h +++ b/include/adt/hash.h @@ -10,10 +10,10 @@ #include #include "adt/common.h" +#include "fsm/fsm.h" -/* 32 and 64-bit approximations of the golden ratio. */ -#define FSM_PHI_32 0x9e3779b9UL -#define FSM_PHI_64 (uint64_t)0x9e3779b97f4a7c15UL +#define HASH_LOG_PROBES 0 +/* #define HASH_PROBE_LIMIT 100 */ SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static __inline__ uint64_t @@ -28,38 +28,13 @@ hash_id(uint64_t id) return x * 2685821657736338717LLU; } -/* FNV-1a hash function, 32 and 64 bit versions. This is in the public - * domain. For details, see: - * - * http://www.isthe.com/chongo/tech/comp/fnv/index.html - */ - -SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() -static __inline__ uint32_t -hash_fnv1a_32(const uint8_t *buf, size_t length) -{ -#define FNV1a_32_OFFSET_BASIS 0x811c9dc5UL -#define FNV1a_32_PRIME 0x01000193UL - uint32_t h = FNV1a_32_OFFSET_BASIS; - size_t i; - for (i = 0; i < length; i++) { - h ^= buf[i]; - h *= FNV1a_32_PRIME; - } - return h; -} - SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static __inline__ uint64_t -hash_fnv1a_64(const uint8_t *buf, size_t length) +hash_ids(size_t count, const fsm_state_t *ids) { -#define FNV1a_64_OFFSET_BASIS 0xcbf29ce484222325UL -#define FNV1a_64_PRIME 0x100000001b3UL - uint64_t h = FNV1a_64_OFFSET_BASIS; - size_t i; - for (i = 0; i < length; i++) { - h ^= buf[i]; - h *= FNV1a_64_PRIME; + uint64_t h = 0; + for (size_t i = 0; i < count; i++) { + h = hash_id(h ^ ids[i]); } return h; } diff --git a/src/adt/idmap.c b/src/adt/idmap.c index 342ecd136..208af3c5a 100644 --- a/src/adt/idmap.c +++ b/src/adt/idmap.c @@ -203,12 +203,15 @@ idmap_set(struct idmap *m, fsm_state_t state_id, } const uint64_t mask = m->bucket_count - 1; - for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + int res = 0; + size_t b_i; + for (b_i = 0; b_i < m->bucket_count; b_i++) { struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; if (b->state == state_id) { assert(b->values != NULL); u64bitset_set(b->values, value); - return 1; + res = 1; + goto done; } else if (b->state == NO_STATE) { b->state = state_id; assert(b->values == NULL); @@ -217,20 +220,30 @@ idmap_set(struct idmap *m, fsm_state_t state_id, b->values = f_calloc(m->alloc, vw, sizeof(b->values[0])); if (b->values == NULL) { - return 0; + goto done; } m->buckets_used++; u64bitset_set(b->values, value); - return 1; + res = 1; + goto done; } else { continue; /* collision */ } - } - assert(!"unreachable"); - return 0; +done: +#if HASH_LOG_PROBES + fprintf(stderr, "%s: res %d after %zu probes (%u buckets)\n", + __func__, res, b_i, m->bucket_count); +#endif + +#ifdef HASH_PROBE_LIMIT + assert(b_i < HASH_PROBE_LIMIT); +#endif + assert(b_i < m->bucket_count); + + return res; } static const struct idmap_bucket * diff --git a/src/adt/internedstateset.c b/src/adt/internedstateset.c index a39f09a19..9daf50ae7 100644 --- a/src/adt/internedstateset.c +++ b/src/adt/internedstateset.c @@ -197,15 +197,10 @@ dump_tables(FILE *f, const struct interned_state_set_pool *pool) #endif } -SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static uint64_t hash_state_ids(size_t count, const fsm_state_t *ids) { - uint64_t h = 0; - for (size_t i = 0; i < count; i++) { - h ^= hash_id(ids[i]); - } - return h; + return hash_ids(count, ids); } static bool @@ -329,8 +324,13 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, fprintf(stderr, "%s: htab[(0x%lx + %lu) & 0x%lx => %d\n", __func__, h, b_i, mask, *b); #endif + +#ifdef HASH_PROBE_LIMIT + assert(probes < HASH_PROBE_LIMIT); +#endif + if (*b == NO_ID) { -#if LOG_ISS > 3 +#if LOG_ISS > 3 || HASH_LOG_PROBES fprintf(stderr, "%s: empty bucket (%zd probes)\n", __func__, probes); #endif dst_bucket = b; @@ -351,7 +351,7 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, if (0 == memcmp(states, buf, s->length * sizeof(buf[0]))) { *result = id; -#if LOG_ISS > 3 +#if LOG_ISS > 3 || HASH_LOG_PROBES fprintf(stderr, "%s: reused %u (%zd probes)\n", __func__, id, probes); #endif return true; @@ -362,7 +362,7 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, } assert(dst_bucket != NULL); -#if LOG_ISS > 3 +#if LOG_ISS > 3 || HASH_LOG_PROBES fprintf(stderr, "%s: miss after %zd probes\n", __func__, probes); #else (void)probes; diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 5f6a984bc..582d769f6 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -396,7 +396,7 @@ hash_iss(interned_state_set_id iss) { /* Just hashing the ID directly is fine here -- since they're * interned, they're identified by pointer equality. */ - return FSM_PHI_64 * (uintptr_t)iss; + return hash_id((uintptr_t)iss); } static struct mapping * @@ -1309,15 +1309,10 @@ build_output_from_cached_analysis(struct analyze_closures_env *env, fsm_state_t #define LOG_TO_SET_HTAB 0 -SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static uint64_t to_set_hash(size_t count, const fsm_state_t *ids) { - uint64_t h = hash_id(count); - for (size_t i = 0; i < count; i++) { - h ^= hash_id(ids[i]); - } - return h; + return hash_ids(count, ids); } static int @@ -1333,15 +1328,14 @@ to_set_htab_check(struct analyze_closures_env *env, return 0; } - -#if LOG_TO_SET_HTAB +#if LOG_TO_SET_HTAB || HASH_LOG_PROBES || defined(HASH_PROBE_LIMIT) size_t probes = 0; #endif int res = 0; const uint64_t mask = bcount - 1; for (size_t b_i = 0; b_i < bcount; b_i++) { -#if LOG_TO_SET_HTAB +#if LOG_TO_SET_HTAB || HASH_LOG_PROBES || defined(HASH_PROBE_LIMIT) probes++; #endif const struct to_set_bucket *b = &htab->buckets[(h + b_i) & mask]; @@ -1364,10 +1358,16 @@ to_set_htab_check(struct analyze_closures_env *env, } done: -#if LOG_TO_SET_HTAB +#if LOG_TO_SET_HTAB || HASH_LOG_PROBES fprintf(stderr, "%s: result %d, %zu probes, htab: used %zu/%zu ceil\n", __func__, res, probes, htab->buckets_used, htab->bucket_count); #endif +#ifdef HASH_PROBE_LIMIT + if (probes >= HASH_PROBE_LIMIT) { + fprintf(stderr, "-- %zd probes, limit exceeded\n", probes); + } + assert(probes < HASH_PROBE_LIMIT); +#endif return res; } @@ -1431,6 +1431,22 @@ to_set_htab_save(struct analyze_closures_env *env, b->count = count; b->offset = offset; htab->buckets_used++; +#if HASH_LOG_PROBES + fprintf(stderr, "%s: [", __func__); + const fsm_state_t *ids = &env->to_sets.buf[offset]; + for (size_t i = 0; i < count; i++) { + fprintf(stderr, "%s%d", + i > 0 ? " " : "", ids[i]); + } + + fprintf(stderr, "] -> hash %lx -> b %zu (%zu/%zu used), %zd probes\n", + hash, (hash + b_i) & mask, + htab->buckets_used, htab->bucket_count, b_i); +#endif +#if HASH_PROBE_LIMIT + assert(b_i < HASH_PROBE_LIMIT); +#endif + return 1; } else if (b->count == count) { assert(b->offset != offset); /* no duplicates */ @@ -1586,8 +1602,8 @@ commit_buffered_result(struct analyze_closures_env *env, uint32_t *cache_result_ memcpy(&nr->entries[0], env->results.buffer.entries, nr->count * sizeof(env->results.buffer.entries[0])); #if LOG_GROUPING || LOG_COMMIT_BUFFERED_GROUP - fprintf(stderr, "%s: alloc %zu, hash 0x%016lx\n", - __func__, alloc_sz, hash_fnv1a_64((const uint8_t *)nr, alloc_sz)); + fprintf(stderr, "%s: alloc %zu, count %u\n", + __func__, alloc_sz, nr->count)); #endif } From 60a2f8ca7328b290c7404abb172967eb748a5721 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jan 2024 09:26:53 -0500 Subject: [PATCH 080/373] hash_pair: Ensure pair order is consistent for hashing. --- src/libfsm/determinise.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 582d769f6..c0bca3b64 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1659,10 +1659,14 @@ hash_pair(fsm_state_t a, fsm_state_t b) assert(a != b); assert(a & RESULT_BIT); assert(b & RESULT_BIT); - a &=~ RESULT_BIT; - b &=~ RESULT_BIT; - assert(a != b); - const uint64_t ab = ((uint64_t)a << 32) | (uint64_t)b; + const uint64_t ma = (uint64_t)(a & ~RESULT_BIT); /* m: masked */ + const uint64_t mb = (uint64_t)(b & ~RESULT_BIT); + assert(ma != mb); + + /* Left-shift the smaller ID, so the pair order is consistent for hashing. */ + const uint64_t ab = (ma < mb) + ? ((ma << 32) | mb) + : ((mb << 32) | ma); const uint64_t res = hash_id(ab); /* fprintf(stderr, "%s: a %d, b %d -> %016lx\n", __func__, a, b, res); */ return res; From cad3ecd43447da70452bf771882ec91bf8c38641 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jan 2024 09:30:36 -0500 Subject: [PATCH 081/373] hash_ids: Add EXPENSIVE_CHECK for ids[] being in ascending order. --- include/adt/hash.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/adt/hash.h b/include/adt/hash.h index bd1cf2907..65c8dc897 100644 --- a/include/adt/hash.h +++ b/include/adt/hash.h @@ -12,6 +12,10 @@ #include "adt/common.h" #include "fsm/fsm.h" +#if EXPENSIVE_CHECKS +#include +#endif + #define HASH_LOG_PROBES 0 /* #define HASH_PROBE_LIMIT 100 */ @@ -35,6 +39,11 @@ hash_ids(size_t count, const fsm_state_t *ids) uint64_t h = 0; for (size_t i = 0; i < count; i++) { h = hash_id(h ^ ids[i]); +#if EXPENSIVE_CHECKS + if (i > 0) { + assert(ids[i-1] <= ids[i]); + } +#endif } return h; } From 78081b8fbf2b3263c58771c7a564b0e5e70e8feb Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jan 2024 14:17:45 -0500 Subject: [PATCH 082/373] determinise: Ensure that the analysis_cache pair ordering is consistent. --- src/libfsm/determinise.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index c0bca3b64..7c5612b36 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -1672,6 +1672,15 @@ hash_pair(fsm_state_t a, fsm_state_t b) return res; } +static int eq_bucket_pair(const struct result_pair_bucket *b, fsm_state_t pa, fsm_state_t pb) +{ + if ((pa & ~RESULT_BIT) < (pb & ~RESULT_BIT)) { + return b->ids[0] == pa && b->ids[1] == pb; + } else { + return b->ids[0] == pb && b->ids[1] == pa; + } +} + static int analysis_cache_check_pair(struct analyze_closures_env *env, fsm_state_t pa, fsm_state_t pb, fsm_state_t *result_id) @@ -1697,7 +1706,7 @@ analysis_cache_check_pair(struct analyze_closures_env *env, fsm_state_t pa, fsm_ struct result_pair_bucket *b = &htab->buckets[(h + i) & mask]; if (b->t == RPBT_UNUSED) { break; /* not found */ - } else if (b->t == RPBT_PAIR && b->ids[0] == pa && b->ids[1] == pb) { + } else if (b->t == RPBT_PAIR && eq_bucket_pair(b, pa, pb)) { *result_id = b->result_id; /* hit */ #if LOG_GROUPING > 1 fprintf(stderr, "%s: cache hit for pair { %d_R, %d_R } => %d\n", @@ -1831,6 +1840,7 @@ pair_cache_save(struct analyze_closures_env *env, assert(pa != pb); assert(pa & RESULT_BIT); assert(pb & RESULT_BIT); + assert((pa & ~RESULT_BIT) < (pb & ~RESULT_BIT)); } #if LOG_GROUPING > 1 @@ -1868,6 +1878,8 @@ pair_cache_save(struct analyze_closures_env *env, : hash_pair(ob->ids[0], ob->ids[1])); if (ob->t == RPBT_SINGLE_STATE) { assert(ob->ids[0] == ob->ids[1]); + } else { + assert((ob->ids[0] & ~RESULT_BIT) < (ob->ids[1] & ~RESULT_BIT)); } for (size_t nb_i = 0; nb_i < ncount; nb_i++) { @@ -1902,8 +1914,9 @@ pair_cache_save(struct analyze_closures_env *env, #endif if (b->t == RPBT_UNUSED) { /* empty */ - b->ids[0] = pa; - b->ids[1] = pb; + /* Ensure pair ordering is consistent */ + b->ids[0] = pa < pb ? pa : pb; + b->ids[1] = pa < pb ? pb : pa; b->t = type; b->result_id = result_id; htab->buckets_used++; @@ -1947,6 +1960,16 @@ analysis_cache_save_pair(struct analyze_closures_env *env, fsm_state_t a, fsm_state_t b, fsm_state_t result_id) { assert(a != b); + assert((a & RESULT_BIT) == 0); + assert((b & RESULT_BIT) == 0); + + if (a > b) { /* if necessary, swap to ensure a < b */ + const fsm_state_t tmp = a; + a = b; + b = tmp; + } + assert(a < b); + return pair_cache_save(env, RPBT_PAIR, a | RESULT_BIT, b | RESULT_BIT, result_id); } From f5b130063434506dd756e210bf71b52831e6379b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 8 Jan 2024 17:46:18 -0500 Subject: [PATCH 083/373] ast_analysis.c: Add assert for unreachable branch. Ranges using named endpoints are currently rejected with "Unsupported operator". --- src/libre/ast_analysis.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 850735b30..24db8b6d7 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -603,7 +603,8 @@ can_consume_single_newline(struct ast_expr *n) return n->u.range.from.u.codepoint.u <= '\n' && n->u.range.to.u.codepoint.u >= '\n'; } else if (n->u.range.from.type == AST_ENDPOINT_NAMED) { - /* TODO: unreachable? */ + /* currently unreachable, named ranges are unsupported */ + assert(!"unreachable"); break; } break; From 32c8995a6ae33093f3b8664432ab156cfb4f1b06 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 9 Jan 2024 00:43:44 +0000 Subject: [PATCH 084/373] Missing free. Spotted by ASAN. --- src/libfsm/print/dot.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index f7a351c0d..25f6ea4a7 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -177,7 +177,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) const size_t count = fsm_getendidcount(fsm, i); if (count > 0) { ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); + sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); assert(ids != NULL); if (ids == NULL) { return -1; @@ -203,6 +203,8 @@ print_dotfrag(FILE *f, const struct fsm *fsm) } fprintf(f, " ];\n"); + + f_free(fsm->opt->alloc, ids); } /* TODO: show example here, unless !opt->comments */ From 8e7e4df5d8aafd693db963003797156d2ad29f99 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 9 Jan 2024 09:56:58 -0500 Subject: [PATCH 085/373] determinise: Use alt approach for small sort_and_dedup_dst_buf inputs. I'm still experimenting with the limit, but it's common for sort_and_dedup_dst_buf to be called with a fairly small input array. The bitset approach's cost is proportional to the max - min value delta, so when there's only a few entries it may be faster to do insertion sort instead -- 16 entries will probably fit in a single cache line. Changing SMALL_INPUT_LIMIT changes cutoff, commenting it out disables that approach outright. Setting SMALL_INPUT_CHECK to 1 enables a pass at the end that verifies the array is actually sorted and unique. Also, check upfront whether the input array is already sorted and unique. Experiments so far suggest that this doesn't pay off very often, but it's very cheap since we're already sweeping over the input to the min and max values. --- src/libfsm/determinise.c | 88 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 7c5612b36..f53f5f880 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2065,6 +2065,71 @@ cmp_fsm_state_t(const void *pa, const void *pb) return a < b ? -1 : a > b ? 1 : 0; } +#define SMALL_INPUT_LIMIT 15 +#define SMALL_INPUT_CHECK 0 + +#ifdef SMALL_INPUT_LIMIT +static void +small_input_sort_and_dedup(fsm_state_t *buf, size_t *pused) +{ + /* Alternate sort_and_dedup_dst_buf implementation for when + * the input is small enough to fit in a few cache lines. */ + + const size_t orig_used = *pused; + assert(orig_used > 0 && orig_used <= SMALL_INPUT_LIMIT); + size_t used = 0; + fsm_state_t tmp[SMALL_INPUT_LIMIT]; + + const fsm_state_t MOVED = (fsm_state_t)-1; + + tmp[used++] = buf[0]; + buf[0] = MOVED; + + /* First, insert unique ascending values into a tmp buffer. + * If the input is already mostly sorted, great. */ + for (size_t i = 1; i < orig_used; i++) { + fsm_state_t cur = buf[i]; + if (cur > tmp[used - 1]) { + tmp[used++] = cur; + buf[i] = MOVED; + } + } + + /* Then, do insertion sort for entries that haven't already been + * moved into tmp. They must go somewhere in the middle, because + * entries > the other entries in tmp would have already been + * appended in the first pass. */ + for (size_t i = 1; i < orig_used; i++) { + const fsm_state_t cur = buf[i]; + if (cur == MOVED) { continue; } + for (size_t j = 0; j < used; j++) { + const fsm_state_t other = tmp[j]; + if (cur < other) { /* shift the rest down */ + const size_t to_move = used - j; + memmove(&tmp[j+1], &tmp[j], to_move * sizeof(tmp[j])); + tmp[j] = cur; + used++; + break; + } else if (cur == other) { + break; /* discard duplicate */ + } + } + } + + /* Finally, copy the sorted/dedup'd input back into the buffer. */ + for (size_t i = 0; i < used; i++) { + buf[i] = tmp[i]; + } + *pused = used; + + if (SMALL_INPUT_CHECK) { + for (size_t i = 1; i < used; i++) { + assert(buf[i - 1] < buf[i]); + } + } +} +#endif + static void sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) { @@ -2074,15 +2139,38 @@ sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) return; /* no change */ } +#ifdef SMALL_INPUT_LIMIT + if (orig_used <= SMALL_INPUT_LIMIT) { + small_input_sort_and_dedup(buf, used); + return; + } +#endif + /* Figure out what the min and max values are, because * when the difference between them is not too large it * can be significantly faster to avoid qsort here. */ fsm_state_t min = (fsm_state_t)-1; fsm_state_t max = 0; + fsm_state_t prev; + int already_sorted_and_unique = 1; + for (size_t i = 0; i < orig_used; i++) { const fsm_state_t cur = buf[i]; if (cur < min) { min = cur; } if (cur > max) { max = cur; } + + if (i > 0) { + if (cur <= prev) { + already_sorted_and_unique = 0; + } + } + prev = cur; + } + + /* If the buffer is already sorted and unique, we're done. */ + if (already_sorted_and_unique) { + *used = orig_used; + return; } /* If there's only one unique value, then we're done. */ From 259d5076cb864bf309bd3b200ac99f0415b83810 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 9 Jan 2024 13:02:31 -0500 Subject: [PATCH 086/373] determinise: Adjust SMALL_INPUT_LIMIT. I benchmarked with a range of values 8 to 4096, somewhere around 32 - 256 worked particularly well, and it started to become slower beyond that, so set it to <32 for now. --- src/libfsm/determinise.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index f53f5f880..f0ae42f2a 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2065,7 +2065,7 @@ cmp_fsm_state_t(const void *pa, const void *pb) return a < b ? -1 : a > b ? 1 : 0; } -#define SMALL_INPUT_LIMIT 15 +#define SMALL_INPUT_LIMIT 255 #define SMALL_INPUT_CHECK 0 #ifdef SMALL_INPUT_LIMIT From 692eee7c5be157d293dd1c91cc1f61617721b067 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 4 Apr 2024 15:14:44 -0400 Subject: [PATCH 087/373] Minimisation should not combine end states with distinct sets of endids. This was adapted from the group capture integration branch. It isn't specific to group capture. Currently, metadata on end states will not prohibit merging, which can lead to false positives when using endids to detect which of the original regexes are matching when several regexes are compiled and then unioned into a single DFA. This adds a pass before the main minimisation loop that splits the ECs of end states with distinct sets of end IDs, which is sufficient to prevent them from merging later. I reworked the test tests/endids/endids2_union_many_endids.c somewhat, since it was previously checking that the end states WERE merged. Unfortunately the other checks are somewhat weakened -- I removed the part checking that `endids[j] == j+1` because that ordering isn't guaranteed anymore: the patterns aren't anchored, so some examples will match more than one of them, and the endids collected can be nonconsecutive. I added a separate test, endids10_minimise_partial_overlap.c, which checks the minimal case that when the two regexes /^abc$/ and /^ab*c$/ are combined and a distinct endid is set on each, matching "ac" "abc" "abbc" gets one or both endids as appropriate. Ideally, we'd have a property test that checks that any arbitrary set of regex patterns has the same set of inputs matching -> endids when they are run individually vs. when combined into a single FSM, determinised, and minimised. --- src/libfsm/minimise.c | 323 ++++++++++++++++++ .../endids10_minimise_partial_overlap.c | 82 +++++ tests/endids/endids2_union_many_endids.c | 18 +- 3 files changed, 412 insertions(+), 11 deletions(-) create mode 100644 tests/endids/endids10_minimise_partial_overlap.c diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 60cec48c5..f1ff9fffc 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "internal.h" #include "capture.h" @@ -38,6 +39,21 @@ #include "minimise_internal.h" #include "minimise_test_oracle.h" +static int +split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm); + +#define DEF_CAPTURE_ID_CEIL 4 +struct end_metadata { + struct end_metadata_end { + unsigned count; + fsm_end_id_t *ids; + } end; +}; + +static int +collect_end_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_end *e); + int fsm_minimise(struct fsm *fsm) { @@ -253,6 +269,12 @@ build_minimised_mapping(const struct fsm *fsm, goto cleanup; } + /* This only needs to be run once, but must run before the main + * fixpoint loop below, because it potentially refines ECs. */ + if (!split_ecs_by_end_metadata(&env, fsm)) { + goto cleanup; + } + #if LOG_INIT for (i = 0; i < env.ec_count; i++) { fprintf(stderr, "# --ec[%lu]: %d\n", i, env.ecs[i]); @@ -646,6 +668,307 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, #endif } +SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() +static void +incremental_hash_of_ids(uint64_t *accum, fsm_end_id_t id) +{ + (*accum) += hash_id(id); +} + +static int +same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) +{ + if (a->end.count != b->end.count) { + return 0; + } + + /* compare -- these must be sorted */ + + for (size_t i = 0; i < a->end.count; i++) { + if (a->end.ids[i] != b->end.ids[i]) { + return 0; + } + } + + return 1; +} + +static int +split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) +{ + int res = 0; + + struct end_metadata *end_md; + fsm_state_t *htab = NULL; + + const size_t state_count = fsm_countstates(fsm); + +#if EXPENSIVE_INTEGRITY_CHECKS + /* Invariant: For each EC, either all or none of the states + * are end states. We only partition the set(s) of end states + * here. */ + all_end_states_are_currently_together(env); +#endif + + /* Use the hash table to assign to new groups. */ + + end_md = f_calloc(fsm->opt->alloc, + state_count, sizeof(end_md[0])); + if (end_md == NULL) { + goto cleanup; + } + + size_t bucket_count = 1; + while (bucket_count < state_count) { + bucket_count *= 2; /* power of 2 ceiling */ + } + const size_t mask = bucket_count - 1; + + htab = f_malloc(fsm->opt->alloc, + bucket_count * sizeof(htab[0])); + if (htab == NULL) { + goto cleanup; + } + + /* First pass: collect end state metadata */ + for (size_t ec_i = 0; ec_i < env->ec_count; ec_i++) { + fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); +#if LOG_ECS + fprintf(stderr, "## EC %zu\n", ec_i); +#endif + while (s != NO_ID) { + struct end_metadata *e = &end_md[s]; + if (!fsm_isend(fsm, s)) { + break; /* this EC has non-end states, skip */ + } + + if (!collect_end_ids(fsm, s, &e->end)) { + goto cleanup; + } + + s = env->jump[s]; + } + } + +#if LOG_ECS + fprintf(stderr, "==== BEFORE PARTITIONING BY END METADATA\n"); + dump_ecs(stderr, env); + fprintf(stderr, "====\n"); +#endif + + /* Second pass: partition ECs into groups with identical end IDs. + * for each group with different end IDs, unlink them. */ + const size_t max_ec = env->ec_count; + for (size_t ec_i = 0; ec_i < max_ec; ec_i++) { + fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); + fsm_state_t prev = NO_ID; + + for (size_t i = 0; i < bucket_count; i++) { + htab[i] = NO_ID; /* reset hash table */ + } + + while (s != NO_ID) { + const struct end_metadata *s_md = &end_md[s]; + + uint64_t hash = 0; + const fsm_state_t next = env->jump[s]; + + for (size_t eid_i = 0; eid_i < s_md->end.count; eid_i++) { + incremental_hash_of_ids(&hash, s_md->end.ids[eid_i]); + } + + for (size_t b_i = 0; b_i < bucket_count; b_i++) { + fsm_state_t *b = &htab[(b_i + hash) & mask]; + const fsm_state_t other = *b; + const struct end_metadata *other_md = &end_md[other]; + + if (other == NO_ID) { /* empty hash bucket */ + *b = s; + if (prev == NO_ID) { + /* keep the first state, along with other states + * with matching end IDs, in this EC. no-op. */ +#if LOG_ECS + fprintf(stderr, " -- keeping state s %d in EC %u\n", + s, env->state_ecs[s]); +#endif + prev = s; + } else { /* not first (prev is set), so it landed somewhere else */ + /* unlink and assign new EC */ +#if LOG_ECS + fprintf(stderr, " -- moving state s %d from EC %u to EC %u\n", + s, env->state_ecs[s], env->ec_count); +#endif + env->jump[prev] = env->jump[s]; /* unlink */ + env->ecs[env->ec_count] = s; /* head of new EC */ + env->state_ecs[s] = env->ec_count; + env->jump[s] = NO_ID; + env->ec_count++; + } + break; + } else if (same_end_metadata(s_md, other_md)) { + if (env->state_ecs[other] == ec_i) { + /* keep in the current EC -- no-op */ +#if LOG_ECS + fprintf(stderr, " -- keeping state s %d in EC %u\n", + s, env->state_ecs[s]); +#endif + prev = s; + } else { + /* unlink and link to other state's EC */ +#if LOG_ECS + fprintf(stderr, " -- appending s %d to EC %u, after state %d, before %d\n", + s, env->state_ecs[other], other, env->jump[other]); +#endif + assert(prev != NO_ID); + env->jump[prev] = env->jump[s]; /* unlink */ + env->state_ecs[s] = env->state_ecs[other]; + env->jump[s] = env->jump[other]; + env->jump[other] = s; /* link after other */ + } + break; + } else { + continue; /* collision */ + } + } + + s = next; + } + + /* If this EC only has one entry and it's before the + * done_ec_offset, then set that here so that invariants + * will be restored while sweeping forward after this loop. */ + + if (env->jump[MASK_EC_HEAD(env->ecs[ec_i])] == NO_ID && ec_i < env->done_ec_offset) { + env->done_ec_offset = ec_i; /* will be readjusted later */ + } + +#if LOG_ECS + fprintf(stderr, "==== AFTER PARTITIONING BY END METADATA -- EC %zu\n", ec_i); + dump_ecs(stderr, env); + fprintf(stderr, "==== (done_ec_offset: %d)\n", env->done_ec_offset); +#endif + } + +#if LOG_ECS + fprintf(stderr, "==== AFTER PARTITIONING BY END IDs\n"); + dump_ecs(stderr, env); + fprintf(stderr, "==== (done_ec_offset: %d)\n", env->done_ec_offset); +#endif + + /* Sweep forward and swap ECs as necessary so all single-entry + * ECs are at the end -- they're done. */ + size_t ec_i = env->done_ec_offset; + + while (ec_i < env->ec_count) { + const fsm_state_t head = MASK_EC_HEAD(env->ecs[ec_i]); + if (env->jump[head] == NO_ID) { + /* offset stays where it is */ +#if LOG_ECS + fprintf(stderr, "ec_i: %zu / %u -- branch a\n", ec_i, env->ec_count); +#endif + env->ecs[ec_i] = SET_SMALL_EC_FLAG(head); + } else { + /* this EC has more than one state, but is after + * the done_ec_offset, so swap it with an EC at + * the boundary. */ + const fsm_state_t n_ec_i = env->done_ec_offset; +#if LOG_ECS + fprintf(stderr, "ec_i: %zu / %u -- branch b -- swap %ld and %d\n", + ec_i, env->ec_count, ec_i, n_ec_i); +#endif + + /* swap ec[n_ec_i] and ec[ec_i] */ + const fsm_state_t tmp = env->ecs[ec_i]; + env->ecs[ec_i] = env->ecs[n_ec_i]; + env->ecs[n_ec_i] = tmp; + /* note: this may set the SMALL_EC_FLAG. */ + update_ec_links(env, ec_i); + update_ec_links(env, n_ec_i); + env->done_ec_offset++; + } + ec_i++; + } + +#if LOG_ECS + fprintf(stderr, "==== (done_ec_offset is now: %d, ec_count %u)\n", env->done_ec_offset, env->ec_count); + dump_ecs(stderr, env); +#endif + + /* check that all ECs are before/after done_ec_offset */ + for (size_t ec_i = 0; ec_i < env->ec_count; ec_i++) { + const fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); +#if LOG_ECS + fprintf(stderr, " -- ec_i %zu: s %d\n", ec_i, s); +#endif + if (ec_i < env->done_ec_offset) { + assert(env->jump[s] != NO_ID); + } else { + assert(env->jump[s] == NO_ID); + } + } + + res = 1; + +cleanup: + if (htab != NULL) { + f_free(fsm->opt->alloc, htab); + } + if (end_md != NULL) { + size_t i; + for (i = 0; i < state_count; i++) { + struct end_metadata *e = &end_md[i]; + if (e->end.ids != NULL) { + f_free(fsm->opt->alloc, e->end.ids); + } + } + f_free(fsm->opt->alloc, end_md); + } + + return res; +} + +static int +cmp_end_ids(const void *pa, const void *pb) +{ + const fsm_end_id_t a = *(fsm_end_id_t *)pa; + const fsm_end_id_t b = *(fsm_end_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static int +collect_end_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_end *e) +{ + e->count = fsm_getendidcount(fsm, s); + + if (e->count > 0) { + e->ids = f_malloc(fsm->opt->alloc, + e->count * sizeof(e->ids[0])); + if (e->ids == NULL) { + return 0; + } + + size_t written; + enum fsm_getendids_res res = fsm_getendids(fsm, s, + e->count, e->ids, &written); + assert(res == FSM_GETENDIDS_FOUND); + assert(written == e->count); + + /* sort, to make comparison easier later */ + qsort(e->ids, e->count, + sizeof(e->ids[0]), cmp_end_ids); + +#if LOG_ECS + fprintf(stderr, "%d:", s); + for (size_t i = 0; i < written; i++) { + fprintf(stderr, " %u", e->ids[i]); + } + fprintf(stderr, "\n"); +#endif + } + return 1; +} + #if EXPENSIVE_INTEGRITY_CHECKS static void check_done_ec_offset(const struct min_env *env) diff --git a/tests/endids/endids10_minimise_partial_overlap.c b/tests/endids/endids10_minimise_partial_overlap.c new file mode 100644 index 000000000..6c002f368 --- /dev/null +++ b/tests/endids/endids10_minimise_partial_overlap.c @@ -0,0 +1,82 @@ +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include "endids_utils.h" + +#define ENDID_AB_STAR_C 1 +#define ENDID_ABC 2 + +int main(void) +{ + /* Union /^ab*c$/ and /^abc$/ with distinct endids set on each + * and verify that inputs of "ac", "abc", "abbc" get the endids + * associated with only the regexes that match. + * + * In other words, check that minimisation does not merge them + * and cause false positives. */ + + const char *regex_ab_star_c = "^ab*c$"; + const char *regex_abc = "^abc$"; + + struct fsm *fsm_ab_star_c = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_ab_star_c, NULL, 0, NULL); + assert(fsm_ab_star_c != NULL); + if (!fsm_setendid(fsm_ab_star_c, ENDID_AB_STAR_C)) { assert(!"setendid"); } + + if (!fsm_determinise(fsm_ab_star_c)) { assert(!"determinise"); } + if (!fsm_minimise(fsm_ab_star_c)) { assert(!"minimise"); } + + struct fsm *fsm_abc = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_abc, NULL, 0, NULL); + assert(fsm_abc != NULL); + if (!fsm_setendid(fsm_abc, ENDID_ABC)) { assert(!"setendid"); } + + if (!fsm_determinise(fsm_abc)) { assert(!"determinise"); } + if (!fsm_minimise(fsm_abc)) { assert(!"minimise"); } + + struct fsm *combined = fsm_union(fsm_ab_star_c, fsm_abc, NULL); + assert(combined != NULL); + + int ret = fsm_determinise(combined); + assert(ret != 0); + + ret = fsm_minimise(combined); + assert(ret != 0); + + size_t written; + fsm_end_id_t *endids = NULL; + if (match_string(combined, "ac", NULL, &endids, &written) != 1) { + assert(!"'ac' should match"); + } + assert(written == 1); + assert(endids[0] == ENDID_AB_STAR_C); + + if (match_string(combined, "abc", NULL, &endids, &written) != 1) { + assert(!"'abc' should match"); + } + assert(written == 2); + /* result is not sorted */ + assert((endids[0] == ENDID_AB_STAR_C && endids[1] == ENDID_ABC) || + (endids[1] == ENDID_AB_STAR_C && endids[0] == ENDID_ABC)); + + if (match_string(combined, "abbc", NULL, &endids, &written) != 1) { + assert(!"'abbc' should match"); + } + assert(written == 1); + assert(endids[0] == ENDID_AB_STAR_C); + + fsm_free(combined); +} diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index 8e39ca93d..46f2a189e 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -22,6 +22,7 @@ // to exercise reallocs in the ID list // // Each pattern gets 5 end ids +// Note: These are unanchored and therefore partially overlap static const char *patterns[] = { "abc", // 1-5 "def", // 6-10 @@ -175,7 +176,7 @@ int main(void) ret = fsm_determinise(fsm); assert(ret != 0); - // find end states, make sure we have two end states and they each have endids + // find end states, make sure we have multiple end states and they each have endids nstates = fsm_countstates(fsm); for (state_ind = 0; state_ind < nstates; state_ind++) { @@ -238,19 +239,18 @@ int main(void) } } - /* fsm_minimise currently collapses all end states to the same state. This should - * create a single end state with NUM_ENDIDS_TOTAL end ids - */ ret = fsm_minimise(fsm); assert(ret != 0); - assert( fsm_count(fsm, fsm_isend) == 1 ); + /* fsm_minimise should not collapse all the end states to a + * single end state, because they have distinct endids. */ + assert( fsm_count(fsm, fsm_isend) > 1); nstates = fsm_countstates(fsm); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, num_endids, j; + size_t nwritten, num_endids; enum fsm_getendids_res ret; memset(&endids[0], 0, sizeof endids); @@ -258,7 +258,7 @@ int main(void) nwritten = 0; num_endids = fsm_getendidcount(fsm, state_ind); - assert(num_endids == NUM_ENDIDS_TOTAL); + assert(num_endids <= NUM_ENDIDS_TOTAL); ret = fsm_getendids( fsm, @@ -269,10 +269,6 @@ int main(void) assert(ret == FSM_GETENDIDS_FOUND); assert(nwritten == num_endids); - - for (j=0; j < num_endids; j++) { - assert( endids[j] == j+1 ); - } } } From 76ccc7a081096c73fd612ef8816ca3b2a420a298 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Apr 2024 08:06:38 -0400 Subject: [PATCH 088/373] minimise: Integrate a few functions only called with EXPENSIVE_CHECKS. These got missed. Also, rename EXPENSIVE_INTEGRITY_CHECKS in minimise.c to EXPENSIVE_CHECKS and move the header inclusion. --- src/libfsm/minimise.c | 51 ++++++++++++++++++++++++++-------- src/libfsm/minimise_internal.h | 4 --- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index f1ff9fffc..96d92b4e8 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -17,10 +17,6 @@ #include #include -#if EXPENSIVE_CHECKS -#include -#endif - #include #include #include @@ -39,6 +35,16 @@ #include "minimise_internal.h" #include "minimise_test_oracle.h" +#if EXPENSIVE_CHECKS +#include + +static void +check_done_ec_offset(const struct min_env *env); + +static int +all_end_states_are_currently_together(const struct min_env *env); +#endif + static int split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm); @@ -351,7 +357,7 @@ build_minimised_mapping(const struct fsm *fsm, } } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS check_done_ec_offset(&env); #endif } @@ -425,7 +431,7 @@ dump_ecs(FILE *f, const struct min_env *env) #endif } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS static void check_descending_EC_counts(const struct min_env *env) { @@ -633,7 +639,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, /* The dead state is not a member of any EC. */ env->state_ecs[env->dead_state] = NO_ID; -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS check_descending_EC_counts(env); #endif @@ -703,7 +709,7 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) const size_t state_count = fsm_countstates(fsm); -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Invariant: For each EC, either all or none of the states * are end states. We only partition the set(s) of end states * here. */ @@ -969,7 +975,7 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, return 1; } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS static void check_done_ec_offset(const struct min_env *env) { @@ -984,13 +990,34 @@ check_done_ec_offset(const struct min_env *env) * worth the added complexity to avoid checking ECs 0 and 1. */ for (i = 0; i < env->ec_count; i++) { const fsm_state_t head = MASK_EC_HEAD(env->ecs[i]); - if (i >= done_ec_offset) { + if (i >= env->done_ec_offset) { assert(head == NO_ID || env->jump[head] == NO_ID); } else if (i >= 2) { assert(env->jump[head] != NO_ID); } } } + +static int +all_end_states_are_currently_together(const struct min_env *env) +{ + /* For each EC, either all or none of the states in it + * are end states. */ + for (size_t i = 0; i < env->ec_count; i++) { + const fsm_state_t head = MASK_EC_HEAD(env->ecs[i]); + const int ec_first_is_end = fsm_isend(env->fsm, head); + + fsm_state_t s = env->jump[head]; + while (s != NO_ID) { + if (fsm_isend(env->fsm, s) != ec_first_is_end) { + return 0; + } + s = env->jump[s]; + } + } + + return 1; +} #endif static int @@ -1140,7 +1167,7 @@ try_partition(struct min_env *env, unsigned char label, const unsigned dead_state_ec = env->state_ecs[env->dead_state]; const struct fsm_state *states = env->fsm->states; -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Count states here, to compare against the partitioned * EC' counts later. */ size_t state_count = 0, psrc_count, pdst_count; @@ -1238,7 +1265,7 @@ try_partition(struct min_env *env, unsigned char label, } } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Count how many states were split into each EC * and check that the sum matches the original count. */ psrc_count = 0; diff --git a/src/libfsm/minimise_internal.h b/src/libfsm/minimise_internal.h index 6621d5040..f14352803 100644 --- a/src/libfsm/minimise_internal.h +++ b/src/libfsm/minimise_internal.h @@ -3,10 +3,6 @@ #define NO_ID ((fsm_state_t)-1) -/* If set to non-zero, do extra intensive - * integrity checks inside some inner loops. */ -#define EXPENSIVE_INTEGRITY_CHECKS 0 - #define DEF_INITIAL_COUNT_CEIL 8 struct min_env { From b5f8fb713ef26902e1985321f2c5509546e8b57a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Apr 2024 08:11:36 -0400 Subject: [PATCH 089/373] asan: Fix memory leaks in test. --- tests/endids/endids10_minimise_partial_overlap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/endids/endids10_minimise_partial_overlap.c b/tests/endids/endids10_minimise_partial_overlap.c index 6c002f368..5a72d05d9 100644 --- a/tests/endids/endids10_minimise_partial_overlap.c +++ b/tests/endids/endids10_minimise_partial_overlap.c @@ -63,6 +63,7 @@ int main(void) } assert(written == 1); assert(endids[0] == ENDID_AB_STAR_C); + free(endids); if (match_string(combined, "abc", NULL, &endids, &written) != 1) { assert(!"'abc' should match"); @@ -71,12 +72,14 @@ int main(void) /* result is not sorted */ assert((endids[0] == ENDID_AB_STAR_C && endids[1] == ENDID_ABC) || (endids[1] == ENDID_AB_STAR_C && endids[0] == ENDID_ABC)); + free(endids); if (match_string(combined, "abbc", NULL, &endids, &written) != 1) { assert(!"'abbc' should match"); } assert(written == 1); assert(endids[0] == ENDID_AB_STAR_C); + free(endids); fsm_free(combined); } From a54a40c9442110ec6559651c8ed0c88f110a0c2c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 10 Apr 2024 17:28:32 -0400 Subject: [PATCH 090/373] minimise_test_oracle: Add upfront pass based on endids. If two end states have distinct sets of endids associated with them, mark them as distinguishable before entering the main fixpoint loop. This brings the results from the test oracle back in line with fsm_minimise. --- src/libfsm/minimise_test_oracle.c | 91 +++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index c7d1473c8..529ae6c07 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -107,6 +108,14 @@ fsm_minimise_test_oracle(const struct fsm *fsm) fsm_state_t *tmp_map = NULL; fsm_state_t *mapping = NULL; + /* endid_group_assignments[X] = Y: state X is in endid group Y + * endid_group_leaders[X] = Y: see end state Y for endid group X */ + unsigned *endid_group_assignments = NULL; + size_t endid_group_count = 1; /* group 0 is the empty set */ + unsigned *endid_group_leaders = NULL; + fsm_end_id_t *endid_buf_a = NULL; + fsm_end_id_t *endid_buf_b = NULL; + table = calloc(row_words * table_states, sizeof(table[0])); if (table == NULL) { goto cleanup; } @@ -116,15 +125,30 @@ fsm_minimise_test_oracle(const struct fsm *fsm) mapping = malloc(state_count * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; } + endid_group_assignments = calloc(state_count, sizeof(tmp_map[0])); + if (endid_group_assignments == NULL) { goto cleanup; } + + endid_group_leaders = calloc(state_count, sizeof(tmp_map[0])); + if (endid_group_leaders == NULL) { goto cleanup; } + /* macros for NxN bit table */ #define POS(X,Y) ((X*table_states) + Y) #define MARK(X,Y) u64bitset_set(table, POS(X,Y)) #define CLEAR(X,Y) u64bitset_clear(table, POS(X,Y)) #define CHECK(X,Y) u64bitset_get(table, POS(X,Y)) + size_t max_endid_count = 0; + /* Mark all pairs of states where one is final and one is not. * This includes the dead state. */ for (size_t i = 0; i < table_states; i++) { + if (i < state_count && fsm_isend(fsm, i)) { + const size_t count = fsm_getendidcount(fsm, i); + if (count > max_endid_count) { + max_endid_count = count; + } + } + for (size_t j = 0; j < i; j++) { const bool end_i = i == dead_state ? false : fsm_isend(fsm, i); @@ -141,6 +165,69 @@ fsm_minimise_test_oracle(const struct fsm *fsm) } } + endid_buf_a = malloc(max_endid_count * sizeof(endid_buf_a[0])); + if (endid_buf_a == NULL) { goto cleanup; } + endid_buf_b = malloc(max_endid_count * sizeof(endid_buf_b[0])); + if (endid_buf_b == NULL) { goto cleanup; } + + /* For every end state, check if it has endids. If not, assign it + * to endid group 0 (none). Otherwise, check if its endids match + * any of the other end states. If so, assign it to the same endid + * group, otherwise assign a new one and mark it as the leader. */ + for (size_t i = 0; i < state_count; i++) { + if (fsm_isend(fsm, i)) { + size_t written_a; + enum fsm_getendids_res eres = fsm_getendids(fsm, i, + max_endid_count, endid_buf_a, &written_a); + assert(eres != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); + if (eres == FSM_GETENDIDS_NOT_FOUND) { + assert(written_a == 0); + } else { + assert(written_a > 0); + bool found = false; + /* note: skipping eg 0 here since that's the empty set */ + for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { + size_t written_b; + eres = fsm_getendids(fsm, endid_group_leaders[eg_i], + max_endid_count, endid_buf_b, &written_b); + assert(eres != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); + if (written_b != written_a) { continue; } + if (0 == memcmp(endid_buf_a, endid_buf_b, written_a * sizeof(endid_buf_a[0]))) { + found = true; + endid_group_assignments[i] = eg_i; + break; + } else { + continue; + } + } + + if (!found) { + endid_group_assignments[i] = endid_group_count; + endid_group_leaders[endid_group_count] = i; + endid_group_count++; + } + } + } else { + endid_group_assignments[i] = 0; /* none */ + } + } + + /* Any end states that have not been assigned the same endid + * group must be distinguishable. */ + for (size_t i = 0; i < state_count; i++) { + if (fsm_isend(fsm, i)) { + const unsigned i_group = endid_group_assignments[i]; + for (size_t j = 0; j < i; j++) { + if (fsm_isend(fsm, j)) { + const unsigned j_group = endid_group_assignments[j]; + if (i_group != j_group) { + MARK(i, j); + } + } + } + } + } + bool changed; do { /* run until fixpoint */ changed = false; @@ -273,6 +360,10 @@ fsm_minimise_test_oracle(const struct fsm *fsm) if (table != NULL) { free(table); } if (tmp_map != NULL) { free(tmp_map); } if (mapping != NULL) { free(mapping); } + if (endid_group_assignments != NULL) { free(endid_group_assignments); } + if (endid_group_leaders != NULL) { free(endid_group_leaders); } + if (endid_buf_a != NULL) { free(endid_buf_a); } + if (endid_buf_b != NULL) { free(endid_buf_b); } if (res != NULL) { fsm_free(res); } return NULL; } From 269c973b7883e2245f30b08f65f871a5d61093bd Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 11 Apr 2024 13:06:29 -0400 Subject: [PATCH 091/373] minimise_test_oracle: Fix memory leak. --- src/libfsm/minimise_test_oracle.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 529ae6c07..9045ce67b 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -353,6 +353,10 @@ fsm_minimise_test_oracle(const struct fsm *fsm) free(table); free(tmp_map); free(mapping); + free(endid_group_assignments); + free(endid_group_leaders); + free(endid_buf_a); + free(endid_buf_b); return res; From 993888e1d6389de5d4593a931c512a141384cf66 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 26 Apr 2024 01:28:44 +0100 Subject: [PATCH 092/373] Bugfix: Skipping group #0 only applied for pcre dialect. Here I'd assumed #0 would always be present at the root, but that's actually only the case for a dialect which provides capturing groups. Instead of switching on the dialect, I'm just coding this to treat #0 as optional. This does have the semantic effect of removing the "whatever we parsed, it wasn't a literal" part. But I actually can't find a way to reach that path anyway. --- src/libre/re.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libre/re.c b/src/libre/re.c index 19703c0c8..ae4e4d7c3 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -223,6 +223,7 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, enum re_literal_category *category, char **s, size_t *n) { struct ast *ast; + struct ast_expr *root; const struct dialect *m; int unsatisfiable; int r; @@ -262,8 +263,10 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, /* * Literals have an enclosing group #0, and we skip it for our purposes. - * Parsing a satisfiable expression is required to produce group #0. - * If this doesn't exist, whatever we parsed, it's not a literal. + * Parsing a satisfiable expression is required to produce group #0, + * so we grab the root node from inside #0. But this is only true for + * dialects with group capture, and I dont want to explicitly handle + * all those dialects here. So this is conditional on group #0. * * I'm not doing this as an assertion because AST rewriting is free to * transform as it wishes, and I don't want to assume a particular @@ -272,14 +275,15 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, * This is outside of ast_expr_is_literal() because that function may * apply to sub-trees within a larger AST. */ - if (ast->expr->type != AST_EXPR_GROUP || ast->expr->u.group.id != 0) { - r = 0; - goto done; + if (ast->expr->type == AST_EXPR_GROUP && ast->expr->u.group.id == 0) { + root = ast->expr->u.group.e; + } else { + root = ast->expr; } int anchor_start, anchor_end; - r = ast_expr_is_literal(ast->expr->u.group.e, &anchor_start, &anchor_end, s, n); + r = ast_expr_is_literal(root, &anchor_start, &anchor_end, s, n); if (r == -1) { goto error; } From d80644a3bb3d42533aaf950fce97d1330e05af41 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 26 Apr 2024 02:28:14 +0100 Subject: [PATCH 093/373] Allow bases=NULL for fsm_union_array() --- include/fsm/bool.h | 2 +- src/libfsm/union.c | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/fsm/bool.h b/include/fsm/bool.h index f9807f9f9..d35d1c425 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -44,7 +44,7 @@ fsm_union(struct fsm *a, struct fsm *b, * starting states and (if used) capture bases. * * bases[] is expected to have the same count as fsms[], and - * will be initialized by the function. + * will be initialized by the function. May be NULL. * * On success, returns the unioned fsm and updates bases[]. * On failure, returns NULL; all fsms are freed. */ diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 60e1fbaff..7a882be01 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -101,7 +101,9 @@ fsm_union_array(size_t fsm_count, struct fsm *res = fsms[0]; fsms[0] = NULL; - memset(bases, 0x00, fsm_count * sizeof(bases[0])); + if (bases != NULL) { + memset(bases, 0x00, fsm_count * sizeof(bases[0])); + } for (i = 1; i < fsm_count; i++) { struct fsm_combine_info ci; @@ -117,9 +119,14 @@ fsm_union_array(size_t fsm_count, return NULL; } + res = combined; + + if (bases == NULL) { + continue; + } + bases[i].state = ci.base_b; bases[i].capture = ci.capture_base_b; - res = combined; /* If the first argument didn't get its states put first * in the union, then shift the bases for everything @@ -134,9 +141,11 @@ fsm_union_array(size_t fsm_count, } #if LOG_UNION_ARRAY - for (i = 0; i < fsm_count; i++) { - fprintf(stderr, "union_array: bases %u: %zu, %zu\n", - i, bases[i].state, bases[i].capture); + if (bases != NULL) { + for (i = 0; i < fsm_count; i++) { + fprintf(stderr, "union_array: bases %u: %zu, %zu\n", + i, bases[i].state, bases[i].capture); + } } #endif From 4b8276aae4ed6a8cce880e15894a749a48a15127 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 26 Apr 2024 12:43:28 +0100 Subject: [PATCH 094/373] Construct group #0 at the AST root for all dialects. My thinking here is that capturing the entire expression is a property of the runtime, regardless of whether a particular dialect offers syntax for group capture for subexpressions. Practically, it seems like a user might want to know what a pattern matched (i.e. ignoring the unanchored prefix and suffix). --- src/libre/dialect/glob/parser.c | 50 ++- src/libre/dialect/glob/parser.sid | 14 +- src/libre/dialect/like/parser.c | 50 ++- src/libre/dialect/like/parser.sid | 14 +- src/libre/dialect/literal/parser.c | 58 ++- src/libre/dialect/literal/parser.sid | 14 +- src/libre/dialect/native/parser.c | 258 +++++------ src/libre/dialect/native/parser.sid | 6 +- src/libre/dialect/pcre/parser.c | 622 +++++++++++++-------------- src/libre/dialect/pcre/parser.sid | 10 +- src/libre/dialect/sql/parser.c | 252 ++++++----- src/libre/dialect/sql/parser.sid | 6 +- src/libre/re.c | 18 +- 13 files changed, 738 insertions(+), 634 deletions(-) diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 6c039b0a4..2f0e05149 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -537,7 +537,19 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast return; } { - /* BEGINNING OF INLINE: 119 */ + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 915 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 550 "src/libre/dialect/glob/parser.c" + } + /* END OF ACTION: make-group-id */ + /* BEGINNING OF INLINE: 120 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): @@ -546,15 +558,15 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { #line 894 "src/libre/parser.act" - (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 555 "src/libre/dialect/glob/parser.c" +#line 567 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ - p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); + p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -567,20 +579,32 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { #line 887 "src/libre/parser.act" - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 576 "src/libre/dialect/glob/parser.c" +#line 588 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } break; } } - /* END OF INLINE: 119 */ - /* BEGINNING OF INLINE: 120 */ + /* END OF INLINE: 120 */ + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 945 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 605 "src/libre/dialect/glob/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 121 */ { { switch (CURRENT_TERMINAL) { @@ -603,13 +627,13 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } goto ZL1; -#line 607 "src/libre/dialect/glob/parser.c" +#line 631 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 120 */ + /* END OF INLINE: 121 */ } goto ZL0; ZL1:; @@ -779,6 +803,6 @@ ZL0:; return NULL; } -#line 783 "src/libre/dialect/glob/parser.c" +#line 807 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.sid b/src/libre/dialect/glob/parser.sid index 3c0699b9a..d60c415ee 100644 --- a/src/libre/dialect/glob/parser.sid +++ b/src/libre/dialect/glob/parser.sid @@ -15,7 +15,7 @@ ast_count; ast_class_id; !endpoint; - !group_id; + group_id; %terminals% @@ -84,7 +84,7 @@ !: () -> (:ast_count); !: (:unsigned, :pos, :unsigned, :pos) -> (:ast_count); - !: () -> (:group_id); + : () -> (:group_id); !: () -> (:char); !: () -> (:char); !: () -> (:char); @@ -97,7 +97,7 @@ : () -> (:ast_expr); !: () -> (:ast_expr); : (:ast_expr, :ast_count) -> (:ast_expr); - !: (:ast_expr, :group_id) -> (:ast_expr); + : (:ast_expr, :group_id) -> (:ast_expr); !: () -> (:re_flags); !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); @@ -165,12 +165,14 @@ }; re_glob: () -> (node :ast_expr) = { + id = ; { - node = ; - list-of-atoms(node); + e = ; + list-of-atoms(e); || - node = ; + e = ; }; + node = (e, id); { EOF; diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index eb42f63d4..edda875c8 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -537,7 +537,19 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast return; } { - /* BEGINNING OF INLINE: 119 */ + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 915 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 550 "src/libre/dialect/like/parser.c" + } + /* END OF ACTION: make-group-id */ + /* BEGINNING OF INLINE: 120 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): @@ -546,15 +558,15 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { #line 894 "src/libre/parser.act" - (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 555 "src/libre/dialect/like/parser.c" +#line 567 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ - p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); + p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -567,20 +579,32 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { #line 887 "src/libre/parser.act" - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 576 "src/libre/dialect/like/parser.c" +#line 588 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } break; } } - /* END OF INLINE: 119 */ - /* BEGINNING OF INLINE: 120 */ + /* END OF INLINE: 120 */ + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 945 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 605 "src/libre/dialect/like/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 121 */ { { switch (CURRENT_TERMINAL) { @@ -603,13 +627,13 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } goto ZL1; -#line 607 "src/libre/dialect/like/parser.c" +#line 631 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 120 */ + /* END OF INLINE: 121 */ } goto ZL0; ZL1:; @@ -779,6 +803,6 @@ ZL0:; return NULL; } -#line 783 "src/libre/dialect/like/parser.c" +#line 807 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.sid b/src/libre/dialect/like/parser.sid index 26cc5c90f..56d14a23a 100644 --- a/src/libre/dialect/like/parser.sid +++ b/src/libre/dialect/like/parser.sid @@ -15,7 +15,7 @@ ast_count; ast_class_id; !endpoint; - !group_id; + group_id; %terminals% @@ -84,7 +84,7 @@ !: () -> (:ast_count); !: (:unsigned, :pos, :unsigned, :pos) -> (:ast_count); - !: () -> (:group_id); + : () -> (:group_id); !: () -> (:char); !: () -> (:char); !: () -> (:char); @@ -97,7 +97,7 @@ : () -> (:ast_expr); !: () -> (:ast_expr); : (:ast_expr, :ast_count) -> (:ast_expr); - !: (:ast_expr, :group_id) -> (:ast_expr); + : (:ast_expr, :group_id) -> (:ast_expr); !: () -> (:re_flags); !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); @@ -165,12 +165,14 @@ }; re_like: () -> (node :ast_expr) = { + id = ; { - node = ; - list-of-atoms(node); + e = ; + list-of-atoms(e); || - node = ; + e = ; }; + node = (e, id); { EOF; diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 9b85216e3..dc00dab39 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -363,7 +363,19 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ return; } { - /* BEGINNING OF INLINE: 117 */ + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 915 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 376 "src/libre/dialect/literal/parser.c" + } + /* END OF ACTION: make-group-id */ + /* BEGINNING OF INLINE: 118 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): @@ -372,15 +384,15 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { #line 894 "src/libre/parser.act" - (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 381 "src/libre/dialect/literal/parser.c" +#line 393 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ - p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); + p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -393,20 +405,32 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { #line 887 "src/libre/parser.act" - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 402 "src/libre/dialect/literal/parser.c" +#line 414 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } break; } } - /* END OF INLINE: 117 */ - /* BEGINNING OF INLINE: 118 */ + /* END OF INLINE: 118 */ + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 945 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 431 "src/libre/dialect/literal/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 120 */ { { switch (CURRENT_TERMINAL) { @@ -429,13 +453,13 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ } goto ZL1; -#line 433 "src/libre/dialect/literal/parser.c" +#line 457 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 118 */ + /* END OF INLINE: 120 */ } goto ZL0; ZL1:; @@ -475,7 +499,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 479 "src/libre/dialect/literal/parser.c" +#line 503 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -492,7 +516,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 496 "src/libre/dialect/literal/parser.c" +#line 520 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -508,7 +532,7 @@ ZL1:; } goto ZL2; -#line 512 "src/libre/dialect/literal/parser.c" +#line 536 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -520,7 +544,7 @@ ZL1:; goto ZL2; } -#line 524 "src/libre/dialect/literal/parser.c" +#line 548 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -692,6 +716,6 @@ ZL0:; return NULL; } -#line 696 "src/libre/dialect/literal/parser.c" +#line 720 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.sid b/src/libre/dialect/literal/parser.sid index 9bf204d38..99f8f69d9 100644 --- a/src/libre/dialect/literal/parser.sid +++ b/src/libre/dialect/literal/parser.sid @@ -15,7 +15,7 @@ !ast_count; !ast_class_id; !endpoint; - !group_id; + group_id; %terminals% @@ -84,7 +84,7 @@ !: () -> (:ast_count); !: (:unsigned, :pos, :unsigned, :pos) -> (:ast_count); - !: () -> (:group_id); + : () -> (:group_id); !: () -> (:char); !: () -> (:char); !: () -> (:char); @@ -97,7 +97,7 @@ : () -> (:ast_expr); !: () -> (:ast_expr); !: (:ast_expr, :ast_count) -> (:ast_expr); - !: (:ast_expr, :group_id) -> (:ast_expr); + : (:ast_expr, :group_id) -> (:ast_expr); !: () -> (:re_flags); !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); @@ -155,12 +155,14 @@ }; re_literal: () -> (node :ast_expr) = { + id = ; { - node = ; - list-of-atoms(node); + e = ; + list-of-atoms(e); || - node = ; + e = ; }; + node = (e, id); { EOF; diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 1ddf136f3..aa675da00 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -298,7 +298,7 @@ /* BEGINNING OF FUNCTION DECLARATIONS */ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); -static void p_268(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_267(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags, lex_state, act_state, err, t_ast__expr); static void p_155(flags, lex_state, act_state, err); static void p_expr_C_Clist_Hof_Hpieces(flags, lex_state, act_state, err, t_ast__expr); @@ -316,9 +316,9 @@ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags, lex_state, act_ static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); static void p_expr_C_Cpiece_C_Catom(flags, lex_state, act_state, err, t_ast__expr *); -static void p_249(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); +static void p_248(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); static void p_expr_C_Calt(flags, lex_state, act_state, err, t_ast__expr *); -static void p_253(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_252(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); /* BEGINNING OF STATIC VARIABLES */ @@ -538,7 +538,7 @@ ZL0:; } static void -p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI266, t_unsigned *ZIm, t_ast__count *ZOc) +p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI265, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; @@ -566,7 +566,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 { #line 791 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); #line 573 "src/libre/dialect/native/parser.c" @@ -583,13 +583,13 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI266)); + AST_POS_OF_LX_POS(ast_start, (*ZI265)); AST_POS_OF_LX_POS(ast_end, (ZIend)); (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); @@ -662,7 +662,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 { #line 791 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); #line 669 "src/libre/dialect/native/parser.c" @@ -679,13 +679,13 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI266)); + AST_POS_OF_LX_POS(ast_start, (*ZI265)); AST_POS_OF_LX_POS(ast_end, (ZIend)); (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); @@ -1087,9 +1087,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI262; + t_char ZI261; + t_pos ZI262; t_pos ZI263; - t_pos ZI264; /* BEGINNING OF EXTRACT: CHAR */ { @@ -1098,19 +1098,19 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI263 = lex_state->lx.start; - ZI264 = lex_state->lx.end; + ZI262 = lex_state->lx.start; + ZI263 = lex_state->lx.end; + (void) ZI262; (void) ZI263; - (void) ZI264; - ZI262 = lex_state->buf.a[0]; + ZI261 = lex_state->buf.a[0]; #line 1110 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI262, &ZI263, &ZInode); + p_252 (flags, lex_state, act_state, err, &ZI261, &ZI262, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1119,9 +1119,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_ESC): { - t_char ZI250; + t_char ZI249; + t_pos ZI250; t_pos ZI251; - t_pos ZI252; /* BEGINNING OF EXTRACT: ESC */ { @@ -1131,31 +1131,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(lex_state->buf.a[1] != '\0'); assert(lex_state->buf.a[2] == '\0'); - ZI250 = lex_state->buf.a[1]; - - switch (ZI250) { - case 'a': ZI250 = '\a'; break; - case 'b': ZI250 = '\b'; break; - case 'e': ZI250 = '\033'; break; - case 'f': ZI250 = '\f'; break; - case 'n': ZI250 = '\n'; break; - case 'r': ZI250 = '\r'; break; - case 't': ZI250 = '\t'; break; - case 'v': ZI250 = '\v'; break; + ZI249 = lex_state->buf.a[1]; + + switch (ZI249) { + case 'a': ZI249 = '\a'; break; + case 'b': ZI249 = '\b'; break; + case 'e': ZI249 = '\033'; break; + case 'f': ZI249 = '\f'; break; + case 'n': ZI249 = '\n'; break; + case 'r': ZI249 = '\r'; break; + case 't': ZI249 = '\t'; break; + case 'v': ZI249 = '\v'; break; default: break; } - ZI251 = lex_state->lx.start; - ZI252 = lex_state->lx.end; + ZI250 = lex_state->lx.start; + ZI251 = lex_state->lx.end; + (void) ZI250; (void) ZI251; - (void) ZI252; #line 1155 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI250, &ZI251, &ZInode); + p_252 (flags, lex_state, act_state, err, &ZI249, &ZI250, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1164,9 +1164,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_HEX): { - t_char ZI258; + t_char ZI257; + t_pos ZI258; t_pos ZI259; - t_pos ZI260; /* BEGINNING OF EXTRACT: HEX */ { @@ -1179,11 +1179,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI259 = lex_state->lx.start; - ZI260 = lex_state->lx.end; + ZI258 = lex_state->lx.start; + ZI259 = lex_state->lx.end; + (void) ZI258; (void) ZI259; - (void) ZI260; errno = 0; @@ -1216,13 +1216,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI258 = (char) (unsigned char) u; + ZI257 = (char) (unsigned char) u; #line 1222 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI258, &ZI259, &ZInode); + p_252 (flags, lex_state, act_state, err, &ZI257, &ZI258, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1231,31 +1231,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_NAMED__CLASS): { - t_ast__class__id ZI246; + t_ast__class__id ZI245; + t_pos ZI246; t_pos ZI247; - t_pos ZI248; /* BEGINNING OF EXTRACT: NAMED_CLASS */ { #line 661 "src/libre/parser.act" - ZI246 = DIALECT_CLASS(lex_state->buf.a); - if (ZI246 == NULL) { + ZI245 = DIALECT_CLASS(lex_state->buf.a); + if (ZI245 == NULL) { /* syntax error -- unrecognized class */ goto ZL1; } - ZI247 = lex_state->lx.start; - ZI248 = lex_state->lx.end; + ZI246 = lex_state->lx.start; + ZI247 = lex_state->lx.end; + (void) ZI246; (void) ZI247; - (void) ZI248; #line 1255 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; - p_249 (flags, lex_state, act_state, err, &ZI246, &ZI247, &ZInode); + p_248 (flags, lex_state, act_state, err, &ZI245, &ZI246, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1264,9 +1264,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_OCT): { - t_char ZI254; + t_char ZI253; + t_pos ZI254; t_pos ZI255; - t_pos ZI256; /* BEGINNING OF EXTRACT: OCT */ { @@ -1279,11 +1279,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI255 = lex_state->lx.start; - ZI256 = lex_state->lx.end; + ZI254 = lex_state->lx.start; + ZI255 = lex_state->lx.end; + (void) ZI254; (void) ZI255; - (void) ZI256; errno = 0; @@ -1311,13 +1311,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI254 = (char) (unsigned char) u; + ZI253 = (char) (unsigned char) u; #line 1317 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI254, &ZI255, &ZInode); + p_252 (flags, lex_state, act_state, err, &ZI253, &ZI254, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2222,30 +2222,48 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a return; } { - /* BEGINNING OF INLINE: 233 */ + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ { - { - p_expr (flags, lex_state, act_state, err, &ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } +#line 915 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 2235 "src/libre/dialect/native/parser.c" } - /* END OF INLINE: 233 */ - /* BEGINNING OF INLINE: 234 */ + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 945 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 2252 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 233 */ { { switch (CURRENT_TERMINAL) { case (TOK_EOF): break; default: - goto ZL4; + goto ZL3; } ADVANCE_LEXER; } - goto ZL3; - ZL4:; + goto ZL2; + ZL3:; { /* BEGINNING OF ACTION: err-expected-eof */ { @@ -2256,13 +2274,13 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } goto ZL1; -#line 2260 "src/libre/dialect/native/parser.c" +#line 2278 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } - ZL3:; + ZL2:; } - /* END OF INLINE: 234 */ + /* END OF INLINE: 233 */ } goto ZL0; ZL1:; @@ -2289,7 +2307,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2293 "src/libre/dialect/native/parser.c" +#line 2311 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2311,7 +2329,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2315 "src/libre/dialect/native/parser.c" +#line 2333 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2326,7 +2344,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI198; (void) ZI199; -#line 2330 "src/libre/dialect/native/parser.c" +#line 2348 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2342,7 +2360,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2346 "src/libre/dialect/native/parser.c" +#line 2364 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ @@ -2380,7 +2398,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2384 "src/libre/dialect/native/parser.c" +#line 2402 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2467,7 +2485,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL1; } -#line 2471 "src/libre/dialect/native/parser.c" +#line 2489 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 231 */ @@ -2499,7 +2517,7 @@ ZL1:; } goto ZL4; -#line 2503 "src/libre/dialect/native/parser.c" +#line 2521 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2518,21 +2536,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { + t_pos ZI265; t_pos ZI266; - t_pos ZI267; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { #line 389 "src/libre/parser.act" - ZI266 = lex_state->lx.start; - ZI267 = lex_state->lx.end; + ZI265 = lex_state->lx.start; + ZI266 = lex_state->lx.end; + (void) ZI265; (void) ZI266; - (void) ZI267; -#line 2536 "src/libre/dialect/native/parser.c" +#line 2554 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2560,7 +2578,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2564 "src/libre/dialect/native/parser.c" +#line 2582 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2568,7 +2586,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_268 (flags, lex_state, act_state, err, &ZI266, &ZIm, &ZIc); + p_267 (flags, lex_state, act_state, err, &ZI265, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2584,7 +2602,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, 1, NULL); -#line 2588 "src/libre/dialect/native/parser.c" +#line 2606 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2598,7 +2616,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 2602 "src/libre/dialect/native/parser.c" +#line 2620 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2612,7 +2630,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 2616 "src/libre/dialect/native/parser.c" +#line 2634 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2625,7 +2643,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 2629 "src/libre/dialect/native/parser.c" +#line 2647 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2645,7 +2663,7 @@ ZL1:; } goto ZL2; -#line 2649 "src/libre/dialect/native/parser.c" +#line 2667 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -2654,7 +2672,7 @@ ZL1:; (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 2658 "src/libre/dialect/native/parser.c" +#line 2676 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2684,7 +2702,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2688 "src/libre/dialect/native/parser.c" +#line 2706 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -2696,7 +2714,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2700 "src/libre/dialect/native/parser.c" +#line 2718 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2713,7 +2731,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2717 "src/libre/dialect/native/parser.c" +#line 2735 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2730,7 +2748,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 2734 "src/libre/dialect/native/parser.c" +#line 2752 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2747,7 +2765,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2751 "src/libre/dialect/native/parser.c" +#line 2769 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2771,7 +2789,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2775 "src/libre/dialect/native/parser.c" +#line 2793 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2811,7 +2829,7 @@ ZL1:; } goto ZL2; -#line 2815 "src/libre/dialect/native/parser.c" +#line 2833 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2823,7 +2841,7 @@ ZL1:; goto ZL2; } -#line 2827 "src/libre/dialect/native/parser.c" +#line 2845 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2836,7 +2854,7 @@ ZL0:; } static void -p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI246, t_pos *ZI247, t_ast__expr *ZOnode) +p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI245, t_pos *ZI246, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2847,12 +2865,12 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { #line 1071 "src/libre/parser.act" - (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI246)); + (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI245)); if ((ZInode) == NULL) { goto ZL1; } -#line 2856 "src/libre/dialect/native/parser.c" +#line 2874 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2868,9 +2886,9 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla #line 878 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; - (ZIlower).u.named.class = (*ZI246); + (ZIlower).u.named.class = (*ZI245); -#line 2874 "src/libre/dialect/native/parser.c" +#line 2892 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_155 (flags, lex_state, act_state, err); @@ -2883,10 +2901,10 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { #line 786 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI247)); + mark(&act_state->rangestart, &(*ZI246)); mark(&act_state->rangeend, &(ZIend)); -#line 2890 "src/libre/dialect/native/parser.c" +#line 2908 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ @@ -2896,7 +2914,7 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI247)); + AST_POS_OF_LX_POS(ast_start, (*ZI246)); AST_POS_OF_LX_POS(ast_end, (ZIend)); if ((ZIlower).type != AST_ENDPOINT_LITERAL || @@ -2924,7 +2942,7 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2928 "src/libre/dialect/native/parser.c" +#line 2946 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2958,7 +2976,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2962 "src/libre/dialect/native/parser.c" +#line 2980 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2976,7 +2994,7 @@ ZL0:; } static void -p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI250, t_pos *ZI251, t_ast__expr *ZOnode) +p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI249, t_pos *ZI250, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2987,12 +3005,12 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { #line 908 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI250)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI249)); if ((ZInode) == NULL) { goto ZL1; } -#line 2996 "src/libre/dialect/native/parser.c" +#line 3014 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3008,9 +3026,9 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI #line 873 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZI250); + (ZIlower).u.literal.c = (unsigned char)(*ZI249); -#line 3014 "src/libre/dialect/native/parser.c" +#line 3032 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_155 (flags, lex_state, act_state, err); @@ -3023,10 +3041,10 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { #line 786 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI251)); + mark(&act_state->rangestart, &(*ZI250)); mark(&act_state->rangeend, &(ZIend)); -#line 3030 "src/libre/dialect/native/parser.c" +#line 3048 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ @@ -3036,7 +3054,7 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI251)); + AST_POS_OF_LX_POS(ast_start, (*ZI250)); AST_POS_OF_LX_POS(ast_end, (ZIend)); if ((ZIlower).type != AST_ENDPOINT_LITERAL || @@ -3064,7 +3082,7 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3068 "src/libre/dialect/native/parser.c" +#line 3086 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3240,6 +3258,6 @@ ZL0:; return NULL; } -#line 3244 "src/libre/dialect/native/parser.c" +#line 3262 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.sid b/src/libre/dialect/native/parser.sid index 457fe7a15..20f0912a3 100644 --- a/src/libre/dialect/native/parser.sid +++ b/src/libre/dialect/native/parser.sid @@ -466,9 +466,9 @@ }; re_native: () -> (node :ast_expr) = { - { - node = expr; - }; + id = ; + e = expr; + node = (e, id); { EOF; diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index bccd5bbba..98afd8ec2 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -301,7 +301,7 @@ static void p_expr_C_Cflags_C_Cflag__set(flags, lex_state, act_state, err, t_re_ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Clist_Hof_Hpieces(flags, lex_state, act_state, err, t_ast__expr); -static void p_296(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); +static void p_295(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); static void p_169(flags, lex_state, act_state, err); static void p_expr_C_Cliteral(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_state, err, t_ast__expr *); @@ -309,12 +309,12 @@ static void p_expr_C_Ccomment(flags, lex_state, act_state, err); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags, lex_state, act_state, err, t_endpoint *, t_pos *); +static void p_319(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); -static void p_320(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); +static void p_322(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_195(flags, lex_state, act_state, err, t_ast__expr *); static void p_323(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); -static void p_324(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Cflags(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags, lex_state, act_state, err, t_ast__expr, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); @@ -879,7 +879,7 @@ ZL1:; } static void -p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI293, t_pos *ZI294, t_ast__expr *ZOnode) +p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI292, t_pos *ZI293, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -890,7 +890,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { #line 1071 "src/libre/parser.act" - (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI293)); + (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI292)); if ((ZInode) == NULL) { goto ZL1; } @@ -911,7 +911,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla #line 878 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; - (ZIlower).u.named.class = (*ZI293); + (ZIlower).u.named.class = (*ZI292); #line 917 "src/libre/dialect/pcre/parser.c" } @@ -926,7 +926,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { #line 786 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI294)); + mark(&act_state->rangestart, &(*ZI293)); mark(&act_state->rangeend, &(ZIend)); #line 933 "src/libre/dialect/pcre/parser.c" @@ -939,7 +939,7 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI294)); + AST_POS_OF_LX_POS(ast_start, (*ZI293)); AST_POS_OF_LX_POS(ast_end, (ZIend)); if ((ZIlower).type != AST_ENDPOINT_LITERAL || @@ -1388,9 +1388,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI309; + t_char ZI308; + t_pos ZI309; t_pos ZI310; - t_pos ZI311; /* BEGINNING OF EXTRACT: CHAR */ { @@ -1399,19 +1399,19 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI310 = lex_state->lx.start; - ZI311 = lex_state->lx.end; + ZI309 = lex_state->lx.start; + ZI310 = lex_state->lx.end; + (void) ZI309; (void) ZI310; - (void) ZI311; - ZI309 = lex_state->buf.a[0]; + ZI308 = lex_state->buf.a[0]; #line 1411 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI309, &ZI310, &ZInode); + p_319 (flags, lex_state, act_state, err, &ZI308, &ZI309, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1420,9 +1420,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_CONTROL): { - t_char ZI313; + t_char ZI312; + t_pos ZI313; t_pos ZI314; - t_pos ZI315; /* BEGINNING OF EXTRACT: CONTROL */ { @@ -1433,7 +1433,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(lex_state->buf.a[2] != '\0'); assert(lex_state->buf.a[3] == '\0'); - ZI313 = lex_state->buf.a[2]; + ZI312 = lex_state->buf.a[2]; /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to @@ -1443,7 +1443,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ */ { - unsigned char cc = (unsigned char)ZI313; + unsigned char cc = (unsigned char)ZI312; if (cc > 126 || cc < 32) { goto ZL1; } @@ -1454,14 +1454,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ cc ^= 0x40; - ZI313 = cc; + ZI312 = cc; } - ZI314 = lex_state->lx.start; - ZI315 = lex_state->lx.end; + ZI313 = lex_state->lx.start; + ZI314 = lex_state->lx.end; + (void) ZI313; (void) ZI314; - (void) ZI315; #line 1467 "src/libre/dialect/pcre/parser.c" } @@ -1479,7 +1479,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ #line 1480 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ - p_320 (flags, lex_state, act_state, err, &ZI313, &ZI314, &ZInode); + p_319 (flags, lex_state, act_state, err, &ZI312, &ZI313, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1488,9 +1488,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_ESC): { - t_char ZI297; + t_char ZI296; + t_pos ZI297; t_pos ZI298; - t_pos ZI299; /* BEGINNING OF EXTRACT: ESC */ { @@ -1500,31 +1500,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(lex_state->buf.a[1] != '\0'); assert(lex_state->buf.a[2] == '\0'); - ZI297 = lex_state->buf.a[1]; - - switch (ZI297) { - case 'a': ZI297 = '\a'; break; - case 'b': ZI297 = '\b'; break; - case 'e': ZI297 = '\033'; break; - case 'f': ZI297 = '\f'; break; - case 'n': ZI297 = '\n'; break; - case 'r': ZI297 = '\r'; break; - case 't': ZI297 = '\t'; break; - case 'v': ZI297 = '\v'; break; + ZI296 = lex_state->buf.a[1]; + + switch (ZI296) { + case 'a': ZI296 = '\a'; break; + case 'b': ZI296 = '\b'; break; + case 'e': ZI296 = '\033'; break; + case 'f': ZI296 = '\f'; break; + case 'n': ZI296 = '\n'; break; + case 'r': ZI296 = '\r'; break; + case 't': ZI296 = '\t'; break; + case 'v': ZI296 = '\v'; break; default: break; } - ZI298 = lex_state->lx.start; - ZI299 = lex_state->lx.end; + ZI297 = lex_state->lx.start; + ZI298 = lex_state->lx.end; + (void) ZI297; (void) ZI298; - (void) ZI299; #line 1524 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI297, &ZI298, &ZInode); + p_319 (flags, lex_state, act_state, err, &ZI296, &ZI297, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1533,9 +1533,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_HEX): { - t_char ZI305; + t_char ZI304; + t_pos ZI305; t_pos ZI306; - t_pos ZI307; /* BEGINNING OF EXTRACT: HEX */ { @@ -1548,11 +1548,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI306 = lex_state->lx.start; - ZI307 = lex_state->lx.end; + ZI305 = lex_state->lx.start; + ZI306 = lex_state->lx.end; + (void) ZI305; (void) ZI306; - (void) ZI307; errno = 0; @@ -1585,13 +1585,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI305 = (char) (unsigned char) u; + ZI304 = (char) (unsigned char) u; #line 1591 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI305, &ZI306, &ZInode); + p_319 (flags, lex_state, act_state, err, &ZI304, &ZI305, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1600,31 +1600,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_NAMED__CLASS): { - t_ast__class__id ZI293; + t_ast__class__id ZI292; + t_pos ZI293; t_pos ZI294; - t_pos ZI295; /* BEGINNING OF EXTRACT: NAMED_CLASS */ { #line 661 "src/libre/parser.act" - ZI293 = DIALECT_CLASS(lex_state->buf.a); - if (ZI293 == NULL) { + ZI292 = DIALECT_CLASS(lex_state->buf.a); + if (ZI292 == NULL) { /* syntax error -- unrecognized class */ goto ZL1; } - ZI294 = lex_state->lx.start; - ZI295 = lex_state->lx.end; + ZI293 = lex_state->lx.start; + ZI294 = lex_state->lx.end; + (void) ZI293; (void) ZI294; - (void) ZI295; #line 1624 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; - p_296 (flags, lex_state, act_state, err, &ZI293, &ZI294, &ZInode); + p_295 (flags, lex_state, act_state, err, &ZI292, &ZI293, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1673,9 +1673,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_OCT): { - t_char ZI301; + t_char ZI300; + t_pos ZI301; t_pos ZI302; - t_pos ZI303; /* BEGINNING OF EXTRACT: OCT */ { @@ -1688,11 +1688,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI302 = lex_state->lx.start; - ZI303 = lex_state->lx.end; + ZI301 = lex_state->lx.start; + ZI302 = lex_state->lx.end; + (void) ZI301; (void) ZI302; - (void) ZI303; errno = 0; @@ -1720,13 +1720,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI301 = (char) (unsigned char) u; + ZI300 = (char) (unsigned char) u; #line 1726 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI301, &ZI302, &ZInode); + p_319 (flags, lex_state, act_state, err, &ZI300, &ZI301, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1735,9 +1735,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_UNSUPPORTED): { - t_char ZI317; + t_char ZI316; + t_pos ZI317; t_pos ZI318; - t_pos ZI319; /* BEGINNING OF EXTRACT: UNSUPPORTED */ { @@ -1745,17 +1745,17 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { - ZI317 = lex_state->buf.a[1]; + ZI316 = lex_state->buf.a[1]; } else { - ZI317 = lex_state->buf.a[0]; + ZI316 = lex_state->buf.a[0]; } - ZI318 = lex_state->lx.start; - ZI319 = lex_state->lx.end; + ZI317 = lex_state->lx.start; + ZI318 = lex_state->lx.end; + (void) ZI317; (void) ZI318; - (void) ZI319; #line 1761 "src/libre/dialect/pcre/parser.c" } @@ -1773,7 +1773,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ #line 1774 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ - p_320 (flags, lex_state, act_state, err, &ZI317, &ZI318, &ZInode); + p_319 (flags, lex_state, act_state, err, &ZI316, &ZI317, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2420,80 +2420,7 @@ ZL0:; } static void -p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) -{ - t_ast__expr ZInode; - - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - t_ast__expr ZIe; - - p_expr_C_Cpiece_C_Catom (flags, lex_state, act_state, err, &ZIe); - /* BEGINNING OF INLINE: 267 */ - { - switch (CURRENT_TERMINAL) { - case (TOK_OPT): case (TOK_PLUS): case (TOK_STAR): case (TOK_OPENCOUNT): - { - p_expr_C_Cpiece_C_Clist_Hof_Hcounts (flags, lex_state, act_state, err, ZIe, &ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - break; - default: - { - t_ast__count ZIc; - - /* BEGINNING OF ACTION: count-one */ - { -#line 849 "src/libre/parser.act" - - (ZIc) = ast_make_count(1, NULL, 1, NULL); - -#line 2457 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: count-one */ - /* BEGINNING OF ACTION: ast-make-piece */ - { -#line 931 "src/libre/parser.act" - - if ((ZIc).min == 0 && (ZIc).max == 0) { - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - } else if ((ZIc).min == 1 && (ZIc).max == 1) { - (ZInode) = (ZIe); - } else { - (ZInode) = ast_make_expr_repeat(act_state->poolp, *flags, (ZIe), (ZIc)); - } - if ((ZInode) == NULL) { - err->e = RE_EXEOF; - goto ZL1; - } - -#line 2476 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-piece */ - } - break; - case (ERROR_TERMINAL): - RESTORE_LEXER; - goto ZL1; - } - } - /* END OF INLINE: 267 */ - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOnode = ZInode; -} - -static void -p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI317, t_pos *ZI318, t_ast__expr *ZOnode) +p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI316, t_pos *ZI317, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2504,12 +2431,12 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { #line 908 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI317)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI316)); if ((ZInode) == NULL) { goto ZL1; } -#line 2513 "src/libre/dialect/pcre/parser.c" +#line 2440 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2525,9 +2452,9 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI #line 873 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZI317); + (ZIlower).u.literal.c = (unsigned char)(*ZI316); -#line 2531 "src/libre/dialect/pcre/parser.c" +#line 2458 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_169 (flags, lex_state, act_state, err); @@ -2540,10 +2467,10 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { #line 786 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI318)); + mark(&act_state->rangestart, &(*ZI317)); mark(&act_state->rangeend, &(ZIend)); -#line 2547 "src/libre/dialect/pcre/parser.c" +#line 2474 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ @@ -2553,7 +2480,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI318)); + AST_POS_OF_LX_POS(ast_start, (*ZI317)); AST_POS_OF_LX_POS(ast_end, (ZIend)); if ((ZIlower).type != AST_ENDPOINT_LITERAL || @@ -2581,7 +2508,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2585 "src/libre/dialect/pcre/parser.c" +#line 2512 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2597,6 +2524,79 @@ ZL0:; *ZOnode = ZInode; } +static void +p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) +{ + t_ast__expr ZInode; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + t_ast__expr ZIe; + + p_expr_C_Cpiece_C_Catom (flags, lex_state, act_state, err, &ZIe); + /* BEGINNING OF INLINE: 267 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_OPT): case (TOK_PLUS): case (TOK_STAR): case (TOK_OPENCOUNT): + { + p_expr_C_Cpiece_C_Clist_Hof_Hcounts (flags, lex_state, act_state, err, ZIe, &ZInode); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + default: + { + t_ast__count ZIc; + + /* BEGINNING OF ACTION: count-one */ + { +#line 849 "src/libre/parser.act" + + (ZIc) = ast_make_count(1, NULL, 1, NULL); + +#line 2562 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: count-one */ + /* BEGINNING OF ACTION: ast-make-piece */ + { +#line 931 "src/libre/parser.act" + + if ((ZIc).min == 0 && (ZIc).max == 0) { + (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); + } else if ((ZIc).min == 1 && (ZIc).max == 1) { + (ZInode) = (ZIe); + } else { + (ZInode) = ast_make_expr_repeat(act_state->poolp, *flags, (ZIe), (ZIc)); + } + if ((ZInode) == NULL) { + err->e = RE_EXEOF; + goto ZL1; + } + +#line 2581 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-make-piece */ + } + break; + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + } + } + /* END OF INLINE: 267 */ + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOnode = ZInode; +} + static void p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { @@ -2660,6 +2660,91 @@ ZL0:; *ZOnode = ZInode; } +static void +p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI320, t_unsigned *ZIm, t_ast__count *ZOc) +{ + t_ast__count ZIc; + + switch (CURRENT_TERMINAL) { + case (TOK_CLOSECOUNT): + { + t_pos ZI258; + t_pos ZIend; + + /* BEGINNING OF EXTRACT: CLOSECOUNT */ + { +#line 397 "src/libre/parser.act" + + ZI258 = lex_state->lx.start; + ZIend = lex_state->lx.end; + + (void) ZI258; + (void) ZIend; + +#line 2685 "src/libre/dialect/pcre/parser.c" + } + /* END OF EXTRACT: CLOSECOUNT */ + ADVANCE_LEXER; + /* BEGINNING OF ACTION: mark-count */ + { +#line 791 "src/libre/parser.act" + + mark(&act_state->countstart, &(*ZI320)); + mark(&act_state->countend, &(ZIend)); + +#line 2696 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: mark-count */ + /* BEGINNING OF ACTION: count-range */ + { +#line 853 "src/libre/parser.act" + + struct ast_pos ast_start, ast_end; + + if ((*ZIm) < (*ZIm)) { + err->e = RE_ENEGCOUNT; + err->m = (*ZIm); + err->n = (*ZIm); + + mark(&act_state->countstart, &(*ZI320)); + mark(&act_state->countend, &(ZIend)); + + goto ZL1; + } + + AST_POS_OF_LX_POS(ast_start, (*ZI320)); + AST_POS_OF_LX_POS(ast_end, (ZIend)); + + (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + +#line 2721 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: count-range */ + } + break; + case (TOK_SEP): + { + ADVANCE_LEXER; + p_323 (flags, lex_state, act_state, err, ZI320, ZIm, &ZIc); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + case (ERROR_TERMINAL): + return; + default: + goto ZL1; + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOc = ZIc; +} + static void p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) { @@ -2683,7 +2768,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI196; -#line 2687 "src/libre/dialect/pcre/parser.c" +#line 2772 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2701,7 +2786,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2705 "src/libre/dialect/pcre/parser.c" +#line 2790 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2722,7 +2807,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(ZIc); -#line 2726 "src/libre/dialect/pcre/parser.c" +#line 2811 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2737,7 +2822,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI199; (void) ZI200; -#line 2741 "src/libre/dialect/pcre/parser.c" +#line 2826 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2781,7 +2866,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2785 "src/libre/dialect/pcre/parser.c" +#line 2870 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2797,7 +2882,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2801 "src/libre/dialect/pcre/parser.c" +#line 2886 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2814,92 +2899,7 @@ ZL1:; } static void -p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI321, t_unsigned *ZIm, t_ast__count *ZOc) -{ - t_ast__count ZIc; - - switch (CURRENT_TERMINAL) { - case (TOK_CLOSECOUNT): - { - t_pos ZI258; - t_pos ZIend; - - /* BEGINNING OF EXTRACT: CLOSECOUNT */ - { -#line 397 "src/libre/parser.act" - - ZI258 = lex_state->lx.start; - ZIend = lex_state->lx.end; - - (void) ZI258; - (void) ZIend; - -#line 2838 "src/libre/dialect/pcre/parser.c" - } - /* END OF EXTRACT: CLOSECOUNT */ - ADVANCE_LEXER; - /* BEGINNING OF ACTION: mark-count */ - { -#line 791 "src/libre/parser.act" - - mark(&act_state->countstart, &(*ZI321)); - mark(&act_state->countend, &(ZIend)); - -#line 2849 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-count */ - /* BEGINNING OF ACTION: count-range */ - { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - if ((*ZIm) < (*ZIm)) { - err->e = RE_ENEGCOUNT; - err->m = (*ZIm); - err->n = (*ZIm); - - mark(&act_state->countstart, &(*ZI321)); - mark(&act_state->countend, &(ZIend)); - - goto ZL1; - } - - AST_POS_OF_LX_POS(ast_start, (*ZI321)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); - -#line 2874 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: count-range */ - } - break; - case (TOK_SEP): - { - ADVANCE_LEXER; - p_324 (flags, lex_state, act_state, err, ZI321, ZIm, &ZIc); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - break; - case (ERROR_TERMINAL): - return; - default: - goto ZL1; - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOc = ZIc; -} - -static void -p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI321, t_unsigned *ZIm, t_ast__count *ZOc) +p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI320, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; @@ -2928,7 +2928,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 { #line 791 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); #line 2935 "src/libre/dialect/pcre/parser.c" @@ -2954,13 +2954,13 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI321)); + AST_POS_OF_LX_POS(ast_start, (*ZI320)); AST_POS_OF_LX_POS(ast_end, (ZIend)); (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); @@ -3026,7 +3026,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 { #line 791 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); #line 3033 "src/libre/dialect/pcre/parser.c" @@ -3043,13 +3043,13 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI321)); + AST_POS_OF_LX_POS(ast_start, (*ZI320)); AST_POS_OF_LX_POS(ast_end, (ZIend)); (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); @@ -3678,19 +3678,19 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { + t_pos ZI320; t_pos ZI321; - t_pos ZI322; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { #line 389 "src/libre/parser.act" - ZI321 = lex_state->lx.start; - ZI322 = lex_state->lx.end; + ZI320 = lex_state->lx.start; + ZI321 = lex_state->lx.end; + (void) ZI320; (void) ZI321; - (void) ZI322; #line 3696 "src/libre/dialect/pcre/parser.c" } @@ -3728,7 +3728,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_323 (flags, lex_state, act_state, err, &ZI321, &ZIm, &ZIc); + p_322 (flags, lex_state, act_state, err, &ZI320, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -3824,28 +3824,25 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast return; } { - /* BEGINNING OF INLINE: 282 */ - { - { - t_group__id ZIid; - t_ast__expr ZIe; + t_group__id ZIid; + t_ast__expr ZIe; - /* BEGINNING OF ACTION: make-group-id */ - { + /* BEGINNING OF ACTION: make-group-id */ + { #line 915 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3840 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: make-group-id */ - p_expr (flags, lex_state, act_state, err, &ZIe); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - /* BEGINNING OF ACTION: ast-make-group */ - { +#line 3837 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { #line 945 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); @@ -3853,25 +3850,22 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 3857 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-group */ - } +#line 3854 "src/libre/dialect/pcre/parser.c" } - /* END OF INLINE: 282 */ - /* BEGINNING OF INLINE: 283 */ + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 282 */ { { switch (CURRENT_TERMINAL) { case (TOK_EOF): break; default: - goto ZL4; + goto ZL3; } ADVANCE_LEXER; } - goto ZL3; - ZL4:; + goto ZL2; + ZL3:; { /* BEGINNING OF ACTION: err-expected-eof */ { @@ -3882,13 +3876,13 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } goto ZL1; -#line 3886 "src/libre/dialect/pcre/parser.c" +#line 3880 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } - ZL3:; + ZL2:; } - /* END OF INLINE: 283 */ + /* END OF INLINE: 282 */ } goto ZL0; ZL1:; @@ -3916,7 +3910,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3920 "src/libre/dialect/pcre/parser.c" +#line 3914 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -3928,7 +3922,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 3932 "src/libre/dialect/pcre/parser.c" +#line 3926 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3945,7 +3939,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 3949 "src/libre/dialect/pcre/parser.c" +#line 3943 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3965,7 +3959,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3969 "src/libre/dialect/pcre/parser.c" +#line 3963 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3988,7 +3982,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3992 "src/libre/dialect/pcre/parser.c" +#line 3986 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ @@ -4000,7 +3994,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4004 "src/libre/dialect/pcre/parser.c" +#line 3998 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ @@ -4012,7 +4006,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4016 "src/libre/dialect/pcre/parser.c" +#line 4010 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ @@ -4021,7 +4015,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIcr) = '\r'; -#line 4025 "src/libre/dialect/pcre/parser.c" +#line 4019 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ @@ -4033,7 +4027,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4037 "src/libre/dialect/pcre/parser.c" +#line 4031 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ @@ -4042,7 +4036,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZInl) = '\n'; -#line 4046 "src/libre/dialect/pcre/parser.c" +#line 4040 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ @@ -4054,7 +4048,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4058 "src/libre/dialect/pcre/parser.c" +#line 4052 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ @@ -4065,7 +4059,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4069 "src/libre/dialect/pcre/parser.c" +#line 4063 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ @@ -4076,7 +4070,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4080 "src/libre/dialect/pcre/parser.c" +#line 4074 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ @@ -4088,7 +4082,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4092 "src/libre/dialect/pcre/parser.c" +#line 4086 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -4099,7 +4093,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4103 "src/libre/dialect/pcre/parser.c" +#line 4097 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -4110,7 +4104,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4114 "src/libre/dialect/pcre/parser.c" +#line 4108 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4128,7 +4122,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIflags) = *flags; -#line 4132 "src/libre/dialect/pcre/parser.c" +#line 4126 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ @@ -4137,7 +4131,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 4141 "src/libre/dialect/pcre/parser.c" +#line 4135 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4151,7 +4145,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e *flags = (ZIflags); -#line 4155 "src/libre/dialect/pcre/parser.c" +#line 4149 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ @@ -4163,7 +4157,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4167 "src/libre/dialect/pcre/parser.c" +#line 4161 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4187,7 +4181,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4191 "src/libre/dialect/pcre/parser.c" +#line 4185 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4246,7 +4240,7 @@ ZL1:; } goto ZL2; -#line 4250 "src/libre/dialect/pcre/parser.c" +#line 4244 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -4258,7 +4252,7 @@ ZL1:; goto ZL2; } -#line 4262 "src/libre/dialect/pcre/parser.c" +#line 4256 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4291,7 +4285,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 4295 "src/libre/dialect/pcre/parser.c" +#line 4289 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4312,7 +4306,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 4316 "src/libre/dialect/pcre/parser.c" +#line 4310 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4355,7 +4349,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t goto ZL1; } -#line 4359 "src/libre/dialect/pcre/parser.c" +#line 4353 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -4366,7 +4360,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t goto ZL1; } -#line 4370 "src/libre/dialect/pcre/parser.c" +#line 4364 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-expr */ @@ -4386,7 +4380,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t (ZInode)->u.class.end = ast_end; */ -#line 4390 "src/libre/dialect/pcre/parser.c" +#line 4384 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -4558,6 +4552,6 @@ ZL0:; return NULL; } -#line 4562 "src/libre/dialect/pcre/parser.c" +#line 4556 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.sid b/src/libre/dialect/pcre/parser.sid index 7d3e90c54..3a1d34cbc 100644 --- a/src/libre/dialect/pcre/parser.sid +++ b/src/libre/dialect/pcre/parser.sid @@ -465,7 +465,7 @@ e = ; || END_NL; - e = ; + e = ; || ANY; a = ; @@ -644,11 +644,9 @@ }; re_pcre: () -> (node :ast_expr) = { - { - id = ; - e = expr; - node = (e, id); - }; + id = ; + e = expr; + node = (e, id); { EOF; diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index 993fcfc26..4592ebd14 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -306,9 +306,9 @@ static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_stat static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); -static void p_206(flags, lex_state, act_state, err, t_ast__expr *); -static void p_210(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); -static void p_213(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_205(flags, lex_state, act_state, err, t_ast__expr *); +static void p_209(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_212(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); static void p_expr_C_Cpiece_C_Catom(flags, lex_state, act_state, err, t_ast__expr *); @@ -326,21 +326,21 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_INVERT): { - t_char ZI205; + t_char ZI204; /* BEGINNING OF EXTRACT: INVERT */ { #line 321 "src/libre/parser.act" - ZI205 = '^'; + ZI204 = '^'; - (void) ZI205; + (void) ZI204; #line 340 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; - p_206 (flags, lex_state, act_state, err, ZIclass); + p_205 (flags, lex_state, act_state, err, ZIclass); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -415,30 +415,48 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ return; } { - /* BEGINNING OF INLINE: 192 */ + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ { - { - p_expr (flags, lex_state, act_state, err, &ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } +#line 915 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 428 "src/libre/dialect/sql/parser.c" } - /* END OF INLINE: 192 */ - /* BEGINNING OF INLINE: 193 */ + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 945 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 445 "src/libre/dialect/sql/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 192 */ { { switch (CURRENT_TERMINAL) { case (TOK_EOF): break; default: - goto ZL4; + goto ZL3; } ADVANCE_LEXER; } - goto ZL3; - ZL4:; + goto ZL2; + ZL3:; { /* BEGINNING OF ACTION: err-expected-eof */ { @@ -449,13 +467,13 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } goto ZL1; -#line 453 "src/libre/dialect/sql/parser.c" +#line 471 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } - ZL3:; + ZL2:; } - /* END OF INLINE: 193 */ + /* END OF INLINE: 192 */ } goto ZL0; ZL1:; @@ -491,7 +509,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL4; } -#line 495 "src/libre/dialect/sql/parser.c" +#line 513 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -507,7 +525,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } goto ZL1; -#line 511 "src/libre/dialect/sql/parser.c" +#line 529 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } @@ -567,7 +585,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL1; } -#line 571 "src/libre/dialect/sql/parser.c" +#line 589 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 184 */ @@ -601,9 +619,9 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI207; + t_char ZI206; + t_pos ZI207; t_pos ZI208; - t_pos ZI209; /* BEGINNING OF EXTRACT: CHAR */ { @@ -612,19 +630,19 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI208 = lex_state->lx.start; - ZI209 = lex_state->lx.end; + ZI207 = lex_state->lx.start; + ZI208 = lex_state->lx.end; + (void) ZI207; (void) ZI208; - (void) ZI209; - ZI207 = lex_state->buf.a[0]; + ZI206 = lex_state->buf.a[0]; -#line 624 "src/libre/dialect/sql/parser.c" +#line 642 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_210 (flags, lex_state, act_state, err, &ZI207, &ZI208, &ZInode); + p_209 (flags, lex_state, act_state, err, &ZI206, &ZI207, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -679,7 +697,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 683 "src/libre/dialect/sql/parser.c" +#line 701 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -696,7 +714,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 700 "src/libre/dialect/sql/parser.c" +#line 718 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; @@ -724,7 +742,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 728 "src/libre/dialect/sql/parser.c" +#line 746 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; @@ -735,7 +753,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIopen__end)); -#line 739 "src/libre/dialect/sql/parser.c" +#line 757 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ /* BEGINNING OF ACTION: mark-expr */ @@ -755,7 +773,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 759 "src/libre/dialect/sql/parser.c" +#line 777 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ ZInode = ZIclass; @@ -775,7 +793,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI159; -#line 779 "src/libre/dialect/sql/parser.c" +#line 797 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; @@ -788,7 +806,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL3; } -#line 792 "src/libre/dialect/sql/parser.c" +#line 810 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; @@ -820,7 +838,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 824 "src/libre/dialect/sql/parser.c" +#line 842 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -835,7 +853,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIclose__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 839 "src/libre/dialect/sql/parser.c" +#line 857 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ /* BEGINNING OF ACTION: mark-expr */ @@ -855,7 +873,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 859 "src/libre/dialect/sql/parser.c" +#line 877 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ /* BEGINNING OF ACTION: mark-expr */ @@ -875,7 +893,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZImask__tmp)->u.class.end = ast_end; */ -#line 879 "src/libre/dialect/sql/parser.c" +#line 897 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ } @@ -891,7 +909,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } goto ZL3; -#line 895 "src/libre/dialect/sql/parser.c" +#line 913 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -907,7 +925,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL3; } -#line 911 "src/libre/dialect/sql/parser.c" +#line 929 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -930,7 +948,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } goto ZL1; -#line 934 "src/libre/dialect/sql/parser.c" +#line 952 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -942,7 +960,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 946 "src/libre/dialect/sql/parser.c" +#line 964 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -992,7 +1010,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 996 "src/libre/dialect/sql/parser.c" +#line 1014 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1022,7 +1040,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 1026 "src/libre/dialect/sql/parser.c" +#line 1044 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -1043,7 +1061,7 @@ ZL1:; } goto ZL2; -#line 1047 "src/libre/dialect/sql/parser.c" +#line 1065 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -1055,7 +1073,7 @@ ZL1:; goto ZL2; } -#line 1059 "src/libre/dialect/sql/parser.c" +#line 1077 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1068,7 +1086,7 @@ ZL0:; } static void -p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZIclass) +p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZIclass) { switch (CURRENT_TERMINAL) { case (TOK_RANGE): @@ -1090,7 +1108,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI118; (void) ZI119; -#line 1094 "src/libre/dialect/sql/parser.c" +#line 1112 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1103,7 +1121,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1107 "src/libre/dialect/sql/parser.c" +#line 1125 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ @@ -1114,7 +1132,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1118 "src/libre/dialect/sql/parser.c" +#line 1136 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ @@ -1157,7 +1175,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1161 "src/libre/dialect/sql/parser.c" +#line 1179 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1204,7 +1222,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1208 "src/libre/dialect/sql/parser.c" +#line 1226 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1219,7 +1237,7 @@ ZL1:; } static void -p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI207, t_pos *ZI208, t_ast__expr *ZOnode) +p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI206, t_pos *ZI207, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -1230,12 +1248,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { #line 908 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI207)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI206)); if ((ZInode) == NULL) { goto ZL1; } -#line 1239 "src/libre/dialect/sql/parser.c" +#line 1257 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1256,9 +1274,9 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI #line 873 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; - (ZIa).u.literal.c = (unsigned char)(*ZI207); + (ZIa).u.literal.c = (unsigned char)(*ZI206); -#line 1262 "src/libre/dialect/sql/parser.c" +#line 1280 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -1273,7 +1291,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (void) ZI138; (void) ZI139; -#line 1277 "src/libre/dialect/sql/parser.c" +#line 1295 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1294,7 +1312,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ZIcz = lex_state->buf.a[0]; -#line 1298 "src/libre/dialect/sql/parser.c" +#line 1316 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1309,17 +1327,17 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char)(ZIcz); -#line 1313 "src/libre/dialect/sql/parser.c" +#line 1331 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { #line 786 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI208)); + mark(&act_state->rangestart, &(*ZI207)); mark(&act_state->rangeend, &(ZIend)); -#line 1323 "src/libre/dialect/sql/parser.c" +#line 1341 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ @@ -1329,7 +1347,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI208)); + AST_POS_OF_LX_POS(ast_start, (*ZI207)); AST_POS_OF_LX_POS(ast_end, (ZIend)); if ((ZIa).type != AST_ENDPOINT_LITERAL || @@ -1357,7 +1375,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1361 "src/libre/dialect/sql/parser.c" +#line 1379 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1374,7 +1392,7 @@ ZL0:; } static void -p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI211, t_unsigned *ZIm, t_ast__count *ZOc) +p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI210, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; @@ -1394,7 +1412,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI177; (void) ZIend; -#line 1398 "src/libre/dialect/sql/parser.c" +#line 1416 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -1402,10 +1420,10 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 { #line 791 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1409 "src/libre/dialect/sql/parser.c" +#line 1427 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ @@ -1419,18 +1437,18 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI211)); + AST_POS_OF_LX_POS(ast_start, (*ZI210)); AST_POS_OF_LX_POS(ast_end, (ZIend)); (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); -#line 1434 "src/libre/dialect/sql/parser.c" +#line 1452 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1466,7 +1484,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1470 "src/libre/dialect/sql/parser.c" +#line 1488 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1486,7 +1504,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI180; (void) ZIend; -#line 1490 "src/libre/dialect/sql/parser.c" +#line 1508 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1498,10 +1516,10 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 { #line 791 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1505 "src/libre/dialect/sql/parser.c" +#line 1523 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ @@ -1515,18 +1533,18 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI211)); + AST_POS_OF_LX_POS(ast_start, (*ZI210)); AST_POS_OF_LX_POS(ast_end, (ZIend)); (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); -#line 1530 "src/libre/dialect/sql/parser.c" +#line 1548 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1567,7 +1585,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL1; } -#line 1571 "src/libre/dialect/sql/parser.c" +#line 1589 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 190 */ @@ -1599,7 +1617,7 @@ ZL1:; } goto ZL4; -#line 1603 "src/libre/dialect/sql/parser.c" +#line 1621 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1618,21 +1636,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { + t_pos ZI210; t_pos ZI211; - t_pos ZI212; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { #line 389 "src/libre/parser.act" - ZI211 = lex_state->lx.start; - ZI212 = lex_state->lx.end; + ZI210 = lex_state->lx.start; + ZI211 = lex_state->lx.end; + (void) ZI210; (void) ZI211; - (void) ZI212; -#line 1636 "src/libre/dialect/sql/parser.c" +#line 1654 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1660,7 +1678,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1664 "src/libre/dialect/sql/parser.c" +#line 1682 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1668,7 +1686,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_213 (flags, lex_state, act_state, err, &ZI211, &ZIm, &ZIc); + p_212 (flags, lex_state, act_state, err, &ZI210, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1684,7 +1702,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, 1, NULL); -#line 1688 "src/libre/dialect/sql/parser.c" +#line 1706 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1698,7 +1716,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 1702 "src/libre/dialect/sql/parser.c" +#line 1720 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1712,7 +1730,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 1716 "src/libre/dialect/sql/parser.c" +#line 1734 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1725,7 +1743,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 1729 "src/libre/dialect/sql/parser.c" +#line 1747 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1745,7 +1763,7 @@ ZL1:; } goto ZL2; -#line 1749 "src/libre/dialect/sql/parser.c" +#line 1767 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -1754,7 +1772,7 @@ ZL1:; (ZIc) = ast_make_count(1, NULL, 1, NULL); -#line 1758 "src/libre/dialect/sql/parser.c" +#line 1776 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1784,7 +1802,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1788 "src/libre/dialect/sql/parser.c" +#line 1806 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -1796,7 +1814,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1800 "src/libre/dialect/sql/parser.c" +#line 1818 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1822,7 +1840,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1826 "src/libre/dialect/sql/parser.c" +#line 1844 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1835,7 +1853,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1839 "src/libre/dialect/sql/parser.c" +#line 1857 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1854,7 +1872,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1858 "src/libre/dialect/sql/parser.c" +#line 1876 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ @@ -1866,7 +1884,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1870 "src/libre/dialect/sql/parser.c" +#line 1888 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ @@ -1875,7 +1893,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); -#line 1879 "src/libre/dialect/sql/parser.c" +#line 1897 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -1894,7 +1912,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1898 "src/libre/dialect/sql/parser.c" +#line 1916 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1911,7 +1929,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 1915 "src/libre/dialect/sql/parser.c" +#line 1933 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1928,7 +1946,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1932 "src/libre/dialect/sql/parser.c" +#line 1950 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1966,7 +1984,7 @@ ZL1:; } goto ZL2; -#line 1970 "src/libre/dialect/sql/parser.c" +#line 1988 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -1978,7 +1996,7 @@ ZL1:; goto ZL2; } -#line 1982 "src/libre/dialect/sql/parser.c" +#line 2000 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2021,7 +2039,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI129; (void) ZI130; -#line 2025 "src/libre/dialect/sql/parser.c" +#line 2043 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2038,7 +2056,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act goto ZL1; } -#line 2042 "src/libre/dialect/sql/parser.c" +#line 2060 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2068,7 +2086,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2072 "src/libre/dialect/sql/parser.c" +#line 2090 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2089,7 +2107,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2093 "src/libre/dialect/sql/parser.c" +#line 2111 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2265,6 +2283,6 @@ ZL0:; return NULL; } -#line 2269 "src/libre/dialect/sql/parser.c" +#line 2287 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.sid b/src/libre/dialect/sql/parser.sid index c051c2a2c..cc54ffecd 100644 --- a/src/libre/dialect/sql/parser.sid +++ b/src/libre/dialect/sql/parser.sid @@ -412,9 +412,9 @@ }; re_sql: () -> (node :ast_expr) = { - { - node = expr; - }; + id = ; + e = expr; + node = (e, id); { EOF; diff --git a/src/libre/re.c b/src/libre/re.c index ae4e4d7c3..499bbf194 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -223,7 +223,6 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, enum re_literal_category *category, char **s, size_t *n) { struct ast *ast; - struct ast_expr *root; const struct dialect *m; int unsatisfiable; int r; @@ -263,10 +262,10 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, /* * Literals have an enclosing group #0, and we skip it for our purposes. - * Parsing a satisfiable expression is required to produce group #0, - * so we grab the root node from inside #0. But this is only true for - * dialects with group capture, and I dont want to explicitly handle - * all those dialects here. So this is conditional on group #0. + * Parsing a satisfiable expression is required to produce group #0. + * All dialects do this, regardless of whether their syntax provides + * for group capture of subexpressions. + * If this doesn't exist, whatever we parsed, it's not a literal. * * I'm not doing this as an assertion because AST rewriting is free to * transform as it wishes, and I don't want to assume a particular @@ -275,15 +274,14 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, * This is outside of ast_expr_is_literal() because that function may * apply to sub-trees within a larger AST. */ - if (ast->expr->type == AST_EXPR_GROUP && ast->expr->u.group.id == 0) { - root = ast->expr->u.group.e; - } else { - root = ast->expr; + if (ast->expr->type != AST_EXPR_GROUP || ast->expr->u.group.id != 0) { + r = 0; + goto done; } int anchor_start, anchor_end; - r = ast_expr_is_literal(root, &anchor_start, &anchor_end, s, n); + r = ast_expr_is_literal(ast->expr->u.group.e, &anchor_start, &anchor_end, s, n); if (r == -1) { goto error; } From 3d4beb1c476fc9111a51f929d410440497e7d0a3 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 22 Apr 2024 16:55:47 -0400 Subject: [PATCH 095/373] re_strings: add support for endids. This adds an extra parameter to `re_strings_add_str` and `re_strings_add_raw` that (if non-NULL) will associate a single endid with the string being added. When `re_strings_build` constructs the DFA it will produce a separate end state for each end. This needs further testing with multiple overlapping patterns. When multiple literal strings appear in the input only the latest match will be reported. --- include/fsm/fsm.h | 6 ++++++ include/re/strings.h | 6 +++--- src/libfsm/endids.c | 10 ++++++++++ src/libfsm/libfsm.syms | 1 + src/libre/ac.c | 12 ++++++++++-- src/libre/ac.h | 5 ++++- src/libre/re_strings.c | 10 +++++----- 7 files changed, 39 insertions(+), 11 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 7c3883749..2b4c438f1 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -207,6 +207,12 @@ fsm_setend(struct fsm *fsm, fsm_state_t state, int end); int fsm_setendid(struct fsm *fsm, fsm_end_id_t id); +/* Associate a numeric ID with a specific end state in an fsm. + * Returns 1 on success, 0 on error. + * */ +int +fsm_setendidstate(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); + /* Get the end IDs associated with an end state, if any. * If id_buf has enough cells to store all the end IDs (according * to id_buf_count) then they are written into id_buf[] and diff --git a/include/re/strings.h b/include/re/strings.h index 06387a54c..fe4f7adc2 100644 --- a/include/re/strings.h +++ b/include/re/strings.h @@ -7,7 +7,7 @@ #ifndef RE_STRINGS_H #define RE_STRINGS_H -struct fsm; +#include struct fsm_options; struct re_strings; @@ -42,10 +42,10 @@ void re_strings_free(struct re_strings *g); int -re_strings_add_raw(struct re_strings *g, const void *p, size_t n); +re_strings_add_raw(struct re_strings *g, const void *p, size_t n, const fsm_end_id_t *endid); int -re_strings_add_str(struct re_strings *g, const char *s); +re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endid); struct fsm * re_strings_build(struct re_strings *g, diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 444ccbc2e..1fc98ae68 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -84,6 +84,16 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) return 1; } +int +fsm_setendidstate(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id) +{ + enum fsm_endid_set_res sres = fsm_endid_set(fsm, end_state, id); + if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + return 0; + } + return 1; +} + enum fsm_getendids_res fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, size_t id_buf_count, fsm_end_id_t *id_buf, diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 415bffbea..a2570b8c9 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -89,6 +89,7 @@ fsm_getendids fsm_setendid fsm_mapendids fsm_increndids +fsm_setendidstate fsm_countedges fsm_countstates diff --git a/src/libre/ac.c b/src/libre/ac.c index 2ebdb4f7e..d5f8e6f4b 100644 --- a/src/libre/ac.c +++ b/src/libre/ac.c @@ -16,6 +16,7 @@ #include "ac.h" +#define ENDID_NONE ((fsm_end_id_t)-1) enum { POOL_BLOCK_SIZE = 256 }; struct trie_state { @@ -25,6 +26,7 @@ struct trie_state { unsigned int index; unsigned int output:1; unsigned int have_st:1; + fsm_end_id_t endid; /* or ENDID_NONE */ }; struct trie_pool { @@ -126,7 +128,7 @@ trie_create(void) } struct trie_graph * -trie_add_word(struct trie_graph *g, const char *w, size_t n) +trie_add_word(struct trie_graph *g, const char *w, size_t n, const fsm_end_id_t *endid) { struct trie_state *st; size_t i; @@ -159,6 +161,7 @@ trie_add_word(struct trie_graph *g, const char *w, size_t n) g->depth = n; } + st->endid = (endid == NULL ? ENDID_NONE : *endid); return g; } @@ -278,7 +281,7 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, assert(fsm != NULL); assert(q != NULL); - if (ts->output && have_end) { + if (ts->output && have_end && ts->endid == ENDID_NONE) { *q = single_end; return 1; } @@ -315,6 +318,11 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, if (ts->output) { fsm_setend(fsm, st, 1); + if (ts->endid != ENDID_NONE) { + if (!fsm_setendidstate(fsm, st, ts->endid)) { + return 0; + } + } } *q = st; diff --git a/src/libre/ac.h b/src/libre/ac.h index 480c76bfb..edabca27c 100644 --- a/src/libre/ac.h +++ b/src/libre/ac.h @@ -7,6 +7,8 @@ #ifndef AC_H #define AC_H +#include "fsm/fsm.h" + struct fsm; struct fsm_state; struct fsm_options; @@ -20,7 +22,8 @@ void trie_free(struct trie_graph *g); struct trie_graph * -trie_add_word(struct trie_graph *g, const char *w, size_t n); +trie_add_word(struct trie_graph *g, const char *w, size_t n, + const fsm_end_id_t *endid); int trie_add_failure_edges(struct trie_graph *g); diff --git a/src/libre/re_strings.c b/src/libre/re_strings.c index 06b7dc772..b2f04fec1 100644 --- a/src/libre/re_strings.c +++ b/src/libre/re_strings.c @@ -32,7 +32,7 @@ re_strings(const struct fsm_options *opt, const char *a[], size_t n, } for (i = 0; i < n; i++) { - if (!re_strings_add_str(g, a[i])) { + if (!re_strings_add_str(g, a[i], NULL)) { goto error; } } @@ -64,20 +64,20 @@ re_strings_free(struct re_strings *g) } int -re_strings_add_raw(struct re_strings *g, const void *p, size_t n) +re_strings_add_raw(struct re_strings *g, const void *p, size_t n, const fsm_end_id_t *endid) { assert(p != NULL); assert(n > 0); - return trie_add_word((struct trie_graph *) g, p, n) != NULL; + return trie_add_word((struct trie_graph *) g, p, n, endid) != NULL; } int -re_strings_add_str(struct re_strings *g, const char *s) +re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endid) { assert(s != NULL); - return re_strings_add_raw(g, s, strlen(s)); + return re_strings_add_raw(g, s, strlen(s), endid); } struct fsm * From 986144b5a10429f74fe1e99b346bca55569a8e16 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 29 Apr 2024 10:57:06 -0400 Subject: [PATCH 096/373] Add tests/re_strings. Switch to a set of endids. This uses a `struct state_set` since sizeof(fsm_state) == sizeof(fsm_end_id_t), and it's probably not worth making a separate ADT just for these. The second test checks that duplicated strings get all their endids set. The previous implementation (a single endid, or ENDID_NONE) dropped all but the last endid defined. --- Makefile | 1 + src/libre/ac.c | 22 ++++++++--- tests/re_strings/Makefile | 26 +++++++++++++ tests/re_strings/re_strings1.c | 21 +++++++++++ tests/re_strings/re_strings2.c | 17 +++++++++ tests/re_strings/testutil.c | 69 ++++++++++++++++++++++++++++++++++ tests/re_strings/testutil.h | 11 ++++++ 7 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 tests/re_strings/Makefile create mode 100644 tests/re_strings/re_strings1.c create mode 100644 tests/re_strings/re_strings2.c create mode 100644 tests/re_strings/testutil.c create mode 100644 tests/re_strings/testutil.h diff --git a/Makefile b/Makefile index f1f4f1396..499239fd9 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,7 @@ SUBDIR += tests/pcre-flags SUBDIR += tests/pcre-repeat SUBDIR += tests/pred SUBDIR += tests/re_literal +SUBDIR += tests/re_strings SUBDIR += tests/reverse SUBDIR += tests/trim SUBDIR += tests/union diff --git a/src/libre/ac.c b/src/libre/ac.c index d5f8e6f4b..121eed487 100644 --- a/src/libre/ac.c +++ b/src/libre/ac.c @@ -13,10 +13,10 @@ #include #include +#include #include "ac.h" -#define ENDID_NONE ((fsm_end_id_t)-1) enum { POOL_BLOCK_SIZE = 256 }; struct trie_state { @@ -26,7 +26,9 @@ struct trie_state { unsigned int index; unsigned int output:1; unsigned int have_st:1; - fsm_end_id_t endid; /* or ENDID_NONE */ + + /* use a state set as an endid set */ + struct state_set *endids; }; struct trie_pool { @@ -75,6 +77,7 @@ newstate(struct trie_graph *g) st->index = ++g->nstates; st->output = 0; + st->endids = NULL; return st; } @@ -161,7 +164,9 @@ trie_add_word(struct trie_graph *g, const char *w, size_t n, const fsm_end_id_t g->depth = n; } - st->endid = (endid == NULL ? ENDID_NONE : *endid); + if (endid != NULL) { + state_set_add(&st->endids, NULL, (fsm_state_t)*endid); + } return g; } @@ -281,7 +286,7 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, assert(fsm != NULL); assert(q != NULL); - if (ts->output && have_end && ts->endid == ENDID_NONE) { + if (ts->output && have_end && state_set_empty(ts->endids)) { *q = single_end; return 1; } @@ -318,8 +323,13 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, if (ts->output) { fsm_setend(fsm, st, 1); - if (ts->endid != ENDID_NONE) { - if (!fsm_setendidstate(fsm, st, ts->endid)) { + + struct state_iter si; + fsm_state_t state; + state_set_reset(ts->endids, &si); + while (state_set_next(&si, &state)) { + fsm_end_id_t endid = (fsm_end_id_t)state; + if (!fsm_setendidstate(fsm, st, endid)) { return 0; } } diff --git a/tests/re_strings/Makefile b/tests/re_strings/Makefile new file mode 100644 index 000000000..7fc7f2548 --- /dev/null +++ b/tests/re_strings/Makefile @@ -0,0 +1,26 @@ +.include "../../share/mk/top.mk" + +TEST.tests/re_strings != ls -1 tests/re_strings/re_strings*.c +TEST_SRCDIR.tests/re_strings = tests/re_strings +TEST_OUTDIR.tests/re_strings = ${BUILD}/tests/re_strings + +.for n in ${TEST.tests/re_strings:T:R:C/^re_strings//} +test:: ${TEST_OUTDIR.tests/re_strings}/res${n} +SRC += ${TEST_SRCDIR.tests/re_strings}/re_strings${n}.c +CFLAGS.${TEST_SRCDIR.tests/re_strings}/re_strings${n}.c = -UNDEBUG + +${TEST_OUTDIR.tests/re_strings}/run${n}: ${TEST_OUTDIR.tests/re_strings}/re_strings${n}.o ${TEST_OUTDIR.tests/re_strings}/testutil.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/re_strings}/run${n} ${TEST_OUTDIR.tests/re_strings}/re_strings${n}.o ${TEST_OUTDIR.tests/re_strings}/testutil.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + +${TEST_OUTDIR.tests/re_strings}/re_strings${n}.o: tests/re_strings/testutil.h + +${TEST_OUTDIR.tests/re_strings}/res${n}: ${TEST_OUTDIR.tests/re_strings}/run${n} + ( ${TEST_OUTDIR.tests/re_strings}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/re_strings}/res${n} + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${TEST_OUTDIR.tests/re_strings}/run${n}: ${BUILD}/lib/${lib:R}.a +.endfor +.endfor + +${TEST_OUTDIR.tests/re_strings}/testutil.o: tests/re_strings/testutil.c + ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/re_strings}/testutil.o tests/re_strings/testutil.c diff --git a/tests/re_strings/re_strings1.c b/tests/re_strings/re_strings1.c new file mode 100644 index 000000000..44f10d41d --- /dev/null +++ b/tests/re_strings/re_strings1.c @@ -0,0 +1,21 @@ +#include "testutil.h" + +const char *strings[] = { + "aa", + "ab", + "ac", + "ba", + "bb", + "bc", + "ca", + "cb", + "cc", + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/re_strings2.c b/tests/re_strings/re_strings2.c new file mode 100644 index 000000000..6e4c80459 --- /dev/null +++ b/tests/re_strings/re_strings2.c @@ -0,0 +1,17 @@ +#include "testutil.h" + +const char *strings[] = { + "first", + "duplicate", + "duplicate", + "duplicate", + "last", + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c new file mode 100644 index 000000000..078ca4f4c --- /dev/null +++ b/tests/re_strings/testutil.c @@ -0,0 +1,69 @@ +#include "testutil.h" + +#include +#include + +#include "fsm/fsm.h" +#include "fsm/options.h" + +#include "re/re.h" +#include "re/strings.h" + +static struct fsm_options opt; + +#define MAX_INPUTS 100 +static fsm_end_id_t id_buf[MAX_INPUTS]; + +int +run_test(const char **strings) +{ + struct re_strings *s = re_strings_new(); + assert(s != NULL); + + fsm_end_id_t id = 0; + const char **input = strings; + while (*input != NULL) { + if (!re_strings_add_str(s, *input, &id)) { + assert(!"re_strings_add_str"); + } + + input++; + id++; + assert(id < MAX_INPUTS); + } + + const int flags = 0; /* not anchored */ + + struct fsm *fsm = re_strings_build(s, &opt, flags); + assert(fsm != NULL); + + /* Each literal string input should match, and the set of + * matching endids should include the expected one. */ + id = 0; + input = strings; + while (*input != NULL) { + fsm_state_t end; + const char **string = input; + const int res = fsm_exec(fsm, fsm_sgetc, string, &end, NULL); + assert(res > 0); /* match */ + + size_t written; + enum fsm_getendids_res eres = fsm_getendids(fsm, end, + MAX_INPUTS, id_buf, &written); + assert(eres == FSM_GETENDIDS_FOUND); + bool found = false; + for (size_t i = 0; i < written; i++) { + if (id_buf[i] == id) { + found = true; + break; + } + } + assert(found); + + input++; + id++; + } + + re_strings_free(s); + return EXIT_SUCCESS; +} diff --git a/tests/re_strings/testutil.h b/tests/re_strings/testutil.h new file mode 100644 index 000000000..6898200b7 --- /dev/null +++ b/tests/re_strings/testutil.h @@ -0,0 +1,11 @@ +#ifndef TESTUTIL_H +#define TESTUTIL_H + +#include +#include +#include + +int +run_test(const char **strings); + +#endif From b208d59e743e976ca73d8f973b7ccaf361986819 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 29 Apr 2024 11:03:33 -0400 Subject: [PATCH 097/373] ac: free the endid set. --- src/libre/ac.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libre/ac.c b/src/libre/ac.c index 121eed487..bca129c2a 100644 --- a/src/libre/ac.c +++ b/src/libre/ac.c @@ -91,6 +91,10 @@ cleanup_pool(struct trie_graph *g) p = g->pool; g->pool = p->next; + for (size_t i = 0; i < p->n; i++) { + state_set_free(p->states[i].endids); + } + free(p->states); free(p); } From 43439c01b7bc7b54ac398dc56eb6c605cf94a6a8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 29 Apr 2024 11:06:13 -0400 Subject: [PATCH 098/373] tests/re_strings/testutil.c: Free the fsm after the test. --- tests/re_strings/testutil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 078ca4f4c..4f4678b0d 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -65,5 +65,7 @@ run_test(const char **strings) } re_strings_free(s); + fsm_free(fsm); + return EXIT_SUCCESS; } From 8b69338585e18bf43784678bc98ab550d631e549 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 30 Apr 2024 11:38:32 -0400 Subject: [PATCH 099/373] ac: Move field for better struct packing. --- src/libre/ac.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libre/ac.c b/src/libre/ac.c index bca129c2a..efe13df36 100644 --- a/src/libre/ac.c +++ b/src/libre/ac.c @@ -22,13 +22,13 @@ enum { POOL_BLOCK_SIZE = 256 }; struct trie_state { struct trie_state *children[256]; struct trie_state *fail; + /* use a state set as an endid set */ + struct state_set *endids; + fsm_state_t st; unsigned int index; unsigned int output:1; unsigned int have_st:1; - - /* use a state set as an endid set */ - struct state_set *endids; }; struct trie_pool { From 6bc2e5082d6fe1e2ac1aaa4fd64914ccb22f2c77 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 7 May 2024 17:22:16 -0400 Subject: [PATCH 100/373] src/libfsm/print/c.c: Add opaque argument to generated DFA functions. The generated functions for FSM_IO_STR and FSM_IO_PAIR didn't include a `void *opaque` to pass other state, so add it. For example, this could be used to pass info about which endids matched. --- src/libfsm/print/c.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index df1d26f8c..29365c5e6 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -434,7 +434,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, break; case FSM_IO_STR: - fprintf(f, "(const char *s)\n"); + fprintf(f, "(const char *s, void *opaque)\n"); fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); @@ -443,7 +443,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e)\n"); + fprintf(f, "(const char *b, const char *e, void *opaque)\n"); fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); From 7d9b8621048fb71e92a2863b5dc738fcd066b8fc Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 8 May 2024 09:15:56 -0400 Subject: [PATCH 101/373] re_strings: Add tests for ALL duplicates and for the empty set. --- tests/re_strings/re_strings3.c | 15 +++++++++++++++ tests/re_strings/re_strings4.c | 13 +++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/re_strings/re_strings3.c create mode 100644 tests/re_strings/re_strings4.c diff --git a/tests/re_strings/re_strings3.c b/tests/re_strings/re_strings3.c new file mode 100644 index 000000000..3f4f1d052 --- /dev/null +++ b/tests/re_strings/re_strings3.c @@ -0,0 +1,15 @@ +#include "testutil.h" + +const char *strings[] = { + "duplicate", + "duplicate", + "duplicate", + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/re_strings4.c b/tests/re_strings/re_strings4.c new file mode 100644 index 000000000..b1e00e70f --- /dev/null +++ b/tests/re_strings/re_strings4.c @@ -0,0 +1,13 @@ +#include "testutil.h" + +const char *strings[] = { + /* empty */ + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} From 1e07cc9e25c4bfac865c69f0cd8a45dc18dc8332 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 9 May 2024 09:43:15 -0400 Subject: [PATCH 102/373] lx: Add a guard for referencing `lx->end.col`, may not be in the struct. This field doesn't exist when lx is called with `-x pos`, so don't include references to it the generated code. --- src/lx/print/c.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 829f6ff21..6308ba0c1 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -1118,7 +1118,9 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "{\n"); fprintf(f, "\tassert(lx != NULL);\n"); fprintf(f, "\tassert(p != NULL);\n"); - fprintf(f, "\tlx->end.col = 1;\n"); + if (~api_exclude & API_POS) { + fprintf(f, "\tlx->end.col = 1;\n"); + } fprintf(f, "\tlx->p = p;\n"); fprintf(f, "}\n"); } From 1a3e03857e9d839edd610db15eb7981a0fb221c2 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 15 May 2024 15:09:23 -0400 Subject: [PATCH 103/373] queue: Fix a read past the end of the queue. --- src/adt/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adt/queue.c b/src/adt/queue.c index 95f059ce7..1e5d28e08 100644 --- a/src/adt/queue.c +++ b/src/adt/queue.c @@ -51,7 +51,7 @@ queue_push(struct queue *q, fsm_state_t state) * memmove everything after reading 1. */ if (q->rd > 0 && q->rd < q->wr) { memmove(&q->q[0], &q->q[q->rd], - q->rd * sizeof(q->q[0])); + (q->capacity - q->rd) * sizeof(q->q[0])); q->wr -= q->rd; q->rd = 0; } else { From 2c7847833f1373acde8e123aa637fe331963d2e7 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 25 May 2024 11:13:44 +0100 Subject: [PATCH 104/373] First cut at introducing fsm_intersect_charset(), exposed as fsm -U. --- include/fsm/bool.h | 8 ++++ man/fsm.1/fsm.1.xml | 21 ++++++++++ src/fsm/main.c | 33 ++++++++++++++- src/libfsm/Makefile | 1 + src/libfsm/charset.c | 94 ++++++++++++++++++++++++++++++++++++++++++ src/libfsm/libfsm.syms | 1 + 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 src/libfsm/charset.c diff --git a/include/fsm/bool.h b/include/fsm/bool.h index d35d1c425..d92518297 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -55,6 +55,14 @@ fsm_union_array(size_t fsm_count, struct fsm * fsm_intersect(struct fsm *a, struct fsm *b); +/* + * A convenience to intersect against a character set, rather than + * a pre-existing FSM. Unlike fsm_intersect(), the FSM is required + * to be a DFA. + */ +struct fsm * +fsm_intersect_charset(struct fsm *a, size_t n, const char *charset); + /* * Subtract b from a. This is not commutative. */ diff --git a/man/fsm.1/fsm.1.xml b/man/fsm.1/fsm.1.xml index 117e8966f..6df32e186 100644 --- a/man/fsm.1/fsm.1.xml +++ b/man/fsm.1/fsm.1.xml @@ -22,6 +22,7 @@ io"> iterations"> length"> + charset"> -a"> -w"> @@ -32,6 +33,7 @@ -G &length.arg;"> -k &io.arg;"> -i &iterations.arg;"> + -U &charset.arg;"> -X"> -d"> @@ -75,6 +77,7 @@ fsm &x.opt; + &U.opt; &str.arg; @@ -83,6 +86,7 @@ fsm &p.opt; + &U.opt; &l.opt; &a.opt; @@ -103,6 +107,7 @@ &r.opt; &t.opt; + &U.opt; &i.opt; @@ -111,6 +116,7 @@ fsm &q.opt; + &U.opt; &file.arg; @@ -190,6 +196,21 @@ + + &U.opt; + + + Intersect the resulting state machine down to a + given character set. + This is done after any transformations are applied + (e.g. by ). + + The default character set is a byte. + It is not possible to specify a particular character set + that includes a literal \0. + + + &w.opt; diff --git a/src/fsm/main.c b/src/fsm/main.c index 8e668b779..d6cd1e83c 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -368,6 +368,7 @@ main(int argc, char *argv[]) double elapsed; fsm_print *print; enum op op; + const char *charset; struct fsm *fsm; int xfiles; int r; @@ -390,12 +391,13 @@ main(int argc, char *argv[]) iterations = 1; fsm = NULL; + charset = NULL; r = 0; { int c; - while (c = getopt(argc, argv, "h" "aCcgwXe:k:i:" "xpq:l:dG:mrt:EW:"), c != -1) { + while (c = getopt(argc, argv, "h" "aCcgwXe:k:i:" "xpq:l:dG:mrt:EU:W:"), c != -1) { switch (c) { case 'a': opt.anonymous_states = 1; break; case 'c': opt.consolidate_edges = 1; break; @@ -421,6 +423,11 @@ main(int argc, char *argv[]) case 'r': op = op_name("reverse"); break; case 't': op = op_name(optarg); break; case 'E': op = op_name("remove_epsilons"); break; + + case 'U': + charset = optarg; + break; + case 'W': /* print = gen_words; */ /* num_words = strtoul(optarg, NULL, 10); */ @@ -428,6 +435,7 @@ main(int argc, char *argv[]) fprintf(stderr, "not yet implemented.\n"); exit(EXIT_FAILURE); break; + case 'G': generate_bounds = strtoul(optarg, NULL, 10); if (generate_bounds == 0) { @@ -548,6 +556,29 @@ main(int argc, char *argv[]) q = NULL; } + /* + * It might be more efficient to intersect the character set for each + * operand, but that gives a different result for some operations + * (complement especially). So since we'd also need to intersect the + * result here too, I'm just doing it in the one place for simplicity. + * + * Passing a NULL charset is a no-op, so the default charset is a byte. + * We can't include \0 here because optarg is a string, and I don't + * want to invent a syntax for character sets. + */ + if (charset != NULL) { + if (!fsm_determinise(q)) { + perror("fsm_determinise"); + exit(EXIT_FAILURE); + } + + q = fsm_intersect_charset(q, strlen(charset), charset); + if (q == NULL) { + perror("fsm_intersect_charset"); + exit(EXIT_FAILURE); + } + } + fsm_to_cleanup = q; if (-1 == clock_gettime(CLOCK_MONOTONIC, &post)) { diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 9af51a5a4..005f7d4c6 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -1,6 +1,7 @@ .include "../../share/mk/top.mk" SRC += src/libfsm/capture.c +SRC += src/libfsm/charset.c SRC += src/libfsm/collate.c SRC += src/libfsm/complete.c SRC += src/libfsm/consolidate.c diff --git a/src/libfsm/charset.c b/src/libfsm/charset.c new file mode 100644 index 000000000..dd8a679ed --- /dev/null +++ b/src/libfsm/charset.c @@ -0,0 +1,94 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "internal.h" +#include "walk2.h" + +/* ^[abc..]*$ */ +struct fsm * +fsm_intersect_charset(struct fsm *a, size_t n, const char *charset) +{ + struct fsm *b, *q; + + assert(a != NULL); + + if (charset == NULL) { + return a; + } + + /* + * Since intersection is destructive, there's no point in making + * the charset FSM in advance and then fsm_clone()ing it. + * We may as well just make a new one each time. + */ + { + fsm_state_t state; + + // TODO: pass .statealloc explicitly, we know it's 1. the default is overkill + b = fsm_new(a->opt); + if (b == NULL) { + return NULL; + } + + if (!fsm_addstate(b, &state)) { + goto error; + } + + for (size_t i = 0; i < n; i++) { + if (!fsm_addedge_literal(b, state, state, charset[i])) { + goto error; + } + } + + fsm_setend(b, state, true); + fsm_setstart(b, state); + } + + assert(fsm_all(a, fsm_isdfa)); + assert(fsm_all(b, fsm_isdfa)); + + /* + * This is equivalent to fsm_intersect(). fsm_intersect() asserts that + * both operands are DFA at runtime. But we know this empirically from + * our callers. So we call fsm_walk2() directly to avoid needlessly + * running the DFA predicate in DNDEBUG builds. + * + * This is intersection implemented by walking sets of states through + * both FSM simultaneously, as described by Hopcroft, Motwani and Ullman + * (2001, 2nd ed.) 4.2, Closure under Intersection. + * + * The intersection of two FSM consists of only those items which + * are present in _BOTH. + */ + q = fsm_walk2(a, b, FSM_WALK2_BOTH, FSM_WALK2_BOTH); + if (q == NULL) { + return NULL; + } + + fsm_free(a); + fsm_free(b); + + /* walking two DFA produces a DFA */ + assert(fsm_all(q, fsm_isdfa)); + + return q; + +error: + + fsm_free(b); + + return NULL; +} + diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index a2570b8c9..f905e1f04 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -3,6 +3,7 @@ fsm_complement fsm_union fsm_union_array fsm_intersect +fsm_intersect_charset # fsm_cost_legible From ecb9388863390791bee2596e5dfc25275e5255d2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 29 May 2024 13:33:58 +0100 Subject: [PATCH 105/373] Refactoring: No need to carry lexical positions in the ast_count node. These are used for reporting errors, and the only error here is reported from within the action. In fact I think all errors involving lexical positions are reported within actions (i.e. during parse), because they are all syntax errors. And so I think we never need to annotate the AST with positions. --- src/libre/ast.c | 12 +- src/libre/ast.h | 5 +- src/libre/ast_new_from_fsm.c | 5 +- src/libre/dialect/glob/parser.c | 24 +-- src/libre/dialect/glob/parser.h | 2 +- src/libre/dialect/like/parser.c | 24 +-- src/libre/dialect/like/parser.h | 2 +- src/libre/dialect/literal/parser.c | 16 +- src/libre/dialect/literal/parser.h | 2 +- src/libre/dialect/native/parser.c | 302 +++++++++++++-------------- src/libre/dialect/native/parser.h | 2 +- src/libre/dialect/pcre/parser.c | 325 ++++++++++++++--------------- src/libre/dialect/pcre/parser.h | 2 +- src/libre/dialect/sql/parser.c | 156 +++++++------- src/libre/dialect/sql/parser.h | 2 +- src/libre/parser.act | 15 +- 16 files changed, 422 insertions(+), 474 deletions(-) diff --git a/src/libre/ast.c b/src/libre/ast.c index 82c52a790..6086bf1f5 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -153,23 +153,13 @@ ast_free(struct ast *ast) } struct ast_count -ast_make_count(unsigned min, const struct ast_pos *start, - unsigned max, const struct ast_pos *end) +ast_make_count(unsigned min, unsigned max) { struct ast_count res; - memset(&res, 0x00, sizeof res); - res.min = min; res.max = max; - if (start != NULL) { - res.start = *start; - } - if (end != NULL) { - res.end = *end; - } - return res; } diff --git a/src/libre/ast.h b/src/libre/ast.h index c488ee392..824a637bb 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -40,9 +40,7 @@ enum ast_expr_type { #define AST_COUNT_UNBOUNDED ((unsigned)-1) struct ast_count { unsigned min; - struct ast_pos start; unsigned max; - struct ast_pos end; }; enum ast_anchor_type { @@ -265,8 +263,7 @@ void ast_free(struct ast *ast); struct ast_count -ast_make_count(unsigned min, const struct ast_pos *start, - unsigned max, const struct ast_pos *end); +ast_make_count(unsigned min, unsigned max); /* * Expressions diff --git a/src/libre/ast_new_from_fsm.c b/src/libre/ast_new_from_fsm.c index 4b0462205..64322bc02 100644 --- a/src/libre/ast_new_from_fsm.c +++ b/src/libre/ast_new_from_fsm.c @@ -325,7 +325,7 @@ remove_state(struct ast_expr_pool **poolp, struct rese *state) return 0; } rep = ast_make_expr_repeat(poolp, 0, tmp, - ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL)); + ast_make_count(0, AST_COUNT_UNBOUNDED)); if (!rep) { ast_expr_free(*poolp, tmp); ast_expr_free(*poolp, cat); @@ -453,7 +453,8 @@ ast_expr_new_from_fsm(struct ast_expr_pool **poolp, const struct fsm *fsm) } rep = ast_make_expr_repeat( - poolp, 0, restart->expr, ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL)); + poolp, 0, restart->expr, + ast_make_count(0, AST_COUNT_UNBOUNDED)); if (!rep) { ast_expr_free(*poolp, cat); free_restates(*poolp, restates, numstates + 1); diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 2f0e05149..7ae0125e5 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -323,7 +323,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -377,7 +377,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -416,7 +416,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { @@ -447,7 +447,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -461,14 +461,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, { #line 837 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); #line 467 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -509,7 +509,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -542,7 +542,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -556,7 +556,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -577,7 +577,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -594,7 +594,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 120 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -645,7 +645,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1085 "src/libre/parser.act" static int diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 4b44fa3bd..2b07358d3 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1242 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index edda875c8..07e8a66e0 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -323,7 +323,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -377,7 +377,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -416,7 +416,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { @@ -447,7 +447,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -461,14 +461,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, { #line 837 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); #line 467 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -509,7 +509,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -542,7 +542,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -556,7 +556,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -577,7 +577,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -594,7 +594,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 120 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -645,7 +645,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1085 "src/libre/parser.act" static int diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index 2e3c7b1dd..38fd1e3e0 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1242 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index dc00dab39..3a072f4b5 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -323,7 +323,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -368,7 +368,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -382,7 +382,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -403,7 +403,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -420,7 +420,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* END OF INLINE: 118 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -509,7 +509,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -537,7 +537,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -558,7 +558,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1085 "src/libre/parser.act" static int diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index 55fff6039..e1d707b33 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1242 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index aa675da00..808b02377 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -518,7 +518,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 142 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); @@ -576,8 +576,6 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -589,12 +587,9 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI265)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 598 "src/libre/dialect/native/parser.c" +#line 593 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -630,7 +625,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 634 "src/libre/dialect/native/parser.c" +#line 629 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -650,7 +645,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI221; (void) ZIend; -#line 654 "src/libre/dialect/native/parser.c" +#line 649 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -665,15 +660,13 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); -#line 669 "src/libre/dialect/native/parser.c" +#line 664 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -685,12 +678,9 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI265)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 694 "src/libre/dialect/native/parser.c" +#line 684 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -725,13 +715,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 735 "src/libre/dialect/native/parser.c" +#line 725 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -775,7 +765,7 @@ p_155(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI157; (void) ZI158; -#line 779 "src/libre/dialect/native/parser.c" +#line 769 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -796,7 +786,7 @@ ZL1:; } goto ZL2; -#line 800 "src/libre/dialect/native/parser.c" +#line 790 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -824,13 +814,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 834 "src/libre/dialect/native/parser.c" +#line 824 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 225 */ @@ -891,7 +881,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 895 "src/libre/dialect/native/parser.c" +#line 885 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -930,7 +920,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI112; (void) ZI113; -#line 934 "src/libre/dialect/native/parser.c" +#line 924 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -991,7 +981,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 995 "src/libre/dialect/native/parser.c" +#line 985 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1047,7 +1037,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1051 "src/libre/dialect/native/parser.c" +#line 1041 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1060,14 +1050,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 110 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1071 "src/libre/dialect/native/parser.c" +#line 1061 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1106,7 +1096,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI261 = lex_state->buf.a[0]; -#line 1110 "src/libre/dialect/native/parser.c" +#line 1100 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1151,7 +1141,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI250; (void) ZI251; -#line 1155 "src/libre/dialect/native/parser.c" +#line 1145 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1218,7 +1208,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI257 = (char) (unsigned char) u; -#line 1222 "src/libre/dialect/native/parser.c" +#line 1212 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1251,7 +1241,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI246; (void) ZI247; -#line 1255 "src/libre/dialect/native/parser.c" +#line 1245 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1313,7 +1303,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI253 = (char) (unsigned char) u; -#line 1317 "src/libre/dialect/native/parser.c" +#line 1307 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1368,7 +1358,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1372 "src/libre/dialect/native/parser.c" +#line 1362 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1378,12 +1368,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 873 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1387 "src/libre/dialect/native/parser.c" +#line 1377 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1427,20 +1417,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI168; -#line 1431 "src/libre/dialect/native/parser.c" +#line 1421 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1444 "src/libre/dialect/native/parser.c" +#line 1434 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1468,30 +1458,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI187; -#line 1472 "src/libre/dialect/native/parser.c" +#line 1462 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1485 "src/libre/dialect/native/parser.c" +#line 1475 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 914 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1495 "src/libre/dialect/native/parser.c" +#line 1485 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1501,13 +1491,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1511 "src/libre/dialect/native/parser.c" +#line 1501 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1531,26 +1521,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI179; -#line 1535 "src/libre/dialect/native/parser.c" +#line 1525 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1548 "src/libre/dialect/native/parser.c" +#line 1538 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1588,7 +1578,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1592 "src/libre/dialect/native/parser.c" +#line 1582 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ p_181 (flags, lex_state, act_state, err, &ZItmp); @@ -1615,26 +1605,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI194; -#line 1619 "src/libre/dialect/native/parser.c" +#line 1609 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1632 "src/libre/dialect/native/parser.c" +#line 1622 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1672,16 +1662,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1676 "src/libre/dialect/native/parser.c" +#line 1666 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 914 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1685 "src/libre/dialect/native/parser.c" +#line 1675 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1691,13 +1681,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1701 "src/libre/dialect/native/parser.c" +#line 1691 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1732,7 +1722,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI202; (void) ZIend; -#line 1736 "src/libre/dialect/native/parser.c" +#line 1726 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; @@ -1743,7 +1733,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1747 "src/libre/dialect/native/parser.c" +#line 1737 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1766,31 +1756,31 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI204; (void) ZIend; -#line 1770 "src/libre/dialect/native/parser.c" +#line 1760 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 1783 "src/libre/dialect/native/parser.c" +#line 1773 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 1794 "src/libre/dialect/native/parser.c" +#line 1784 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ @@ -1800,7 +1790,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1804 "src/libre/dialect/native/parser.c" +#line 1794 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1820,7 +1810,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } goto ZL1; -#line 1824 "src/libre/dialect/native/parser.c" +#line 1814 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ ZIend = ZIstart; @@ -1845,7 +1835,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 1849 "src/libre/dialect/native/parser.c" +#line 1839 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-expr */ } @@ -1887,18 +1877,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI150; (void) ZIend; -#line 1891 "src/libre/dialect/native/parser.c" +#line 1881 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); -#line 1902 "src/libre/dialect/native/parser.c" +#line 1892 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -1953,7 +1943,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI182; -#line 1957 "src/libre/dialect/native/parser.c" +#line 1947 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1964,14 +1954,14 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 1975 "src/libre/dialect/native/parser.c" +#line 1965 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1987,12 +1977,12 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(ZIc); -#line 1996 "src/libre/dialect/native/parser.c" +#line 1986 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2007,7 +1997,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI185; (void) ZI186; -#line 2011 "src/libre/dialect/native/parser.c" +#line 2001 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2018,7 +2008,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2051,7 +2041,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2055 "src/libre/dialect/native/parser.c" +#line 2045 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2061,13 +2051,13 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 183 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2071 "src/libre/dialect/native/parser.c" +#line 2061 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2103,7 +2093,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2117,7 +2107,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2121 "src/libre/dialect/native/parser.c" +#line 2111 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2141,14 +2131,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2152 "src/libre/dialect/native/parser.c" +#line 2142 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2162,14 +2152,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2173 "src/libre/dialect/native/parser.c" +#line 2163 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2189,19 +2179,19 @@ ZL1:; } goto ZL2; -#line 2193 "src/libre/dialect/native/parser.c" +#line 2183 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2205 "src/libre/dialect/native/parser.c" +#line 2195 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2227,11 +2217,11 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2235 "src/libre/dialect/native/parser.c" +#line 2225 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -2241,14 +2231,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2252 "src/libre/dialect/native/parser.c" +#line 2242 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 233 */ @@ -2274,7 +2264,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } goto ZL1; -#line 2278 "src/libre/dialect/native/parser.c" +#line 2268 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2300,14 +2290,14 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2311 "src/libre/dialect/native/parser.c" +#line 2301 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2324,12 +2314,12 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2333 "src/libre/dialect/native/parser.c" +#line 2323 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2344,7 +2334,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI198; (void) ZI199; -#line 2348 "src/libre/dialect/native/parser.c" +#line 2338 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2355,17 +2345,17 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2364 "src/libre/dialect/native/parser.c" +#line 2354 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2398,7 +2388,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2402 "src/libre/dialect/native/parser.c" +#line 2392 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2479,13 +2469,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2489 "src/libre/dialect/native/parser.c" +#line 2479 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 231 */ @@ -2517,7 +2507,7 @@ ZL1:; } goto ZL4; -#line 2521 "src/libre/dialect/native/parser.c" +#line 2511 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2550,7 +2540,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI265; (void) ZI266; -#line 2554 "src/libre/dialect/native/parser.c" +#line 2544 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2578,7 +2568,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2582 "src/libre/dialect/native/parser.c" +#line 2572 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2600,9 +2590,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 845 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, 1, NULL); + (ZIc) = ast_make_count(0, 1); -#line 2606 "src/libre/dialect/native/parser.c" +#line 2596 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2614,9 +2604,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 841 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2620 "src/libre/dialect/native/parser.c" +#line 2610 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2628,9 +2618,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 837 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2634 "src/libre/dialect/native/parser.c" +#line 2624 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2641,9 +2631,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 849 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 2647 "src/libre/dialect/native/parser.c" +#line 2637 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2663,16 +2653,16 @@ ZL1:; } goto ZL2; -#line 2667 "src/libre/dialect/native/parser.c" +#line 2657 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { #line 849 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 2676 "src/libre/dialect/native/parser.c" +#line 2666 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2702,19 +2692,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2706 "src/libre/dialect/native/parser.c" +#line 2696 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2718 "src/libre/dialect/native/parser.c" +#line 2708 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2724,14 +2714,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 976 "src/libre/parser.act" +#line 971 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 2735 "src/libre/dialect/native/parser.c" +#line 2725 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2744,11 +2734,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2752 "src/libre/dialect/native/parser.c" +#line 2742 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2758,14 +2748,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 2769 "src/libre/dialect/native/parser.c" +#line 2759 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2782,14 +2772,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 969 "src/libre/parser.act" +#line 964 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 2793 "src/libre/dialect/native/parser.c" +#line 2783 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2829,19 +2819,19 @@ ZL1:; } goto ZL2; -#line 2833 "src/libre/dialect/native/parser.c" +#line 2823 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2845 "src/libre/dialect/native/parser.c" +#line 2835 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2863,14 +2853,14 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI245)); if ((ZInode) == NULL) { goto ZL1; } -#line 2874 "src/libre/dialect/native/parser.c" +#line 2864 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2883,12 +2873,12 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 873 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI245); -#line 2892 "src/libre/dialect/native/parser.c" +#line 2882 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_155 (flags, lex_state, act_state, err); @@ -2904,12 +2894,12 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla mark(&act_state->rangestart, &(*ZI246)); mark(&act_state->rangeend, &(ZIend)); -#line 2908 "src/libre/dialect/native/parser.c" +#line 2898 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2942,7 +2932,7 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2946 "src/libre/dialect/native/parser.c" +#line 2936 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2969,14 +2959,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2980 "src/libre/dialect/native/parser.c" +#line 2970 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -3003,14 +2993,14 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI249)); if ((ZInode) == NULL) { goto ZL1; } -#line 3014 "src/libre/dialect/native/parser.c" +#line 3004 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3023,12 +3013,12 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZI249); -#line 3032 "src/libre/dialect/native/parser.c" +#line 3022 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_155 (flags, lex_state, act_state, err); @@ -3044,12 +3034,12 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI250)); mark(&act_state->rangeend, &(ZIend)); -#line 3048 "src/libre/dialect/native/parser.c" +#line 3038 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -3082,7 +3072,7 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3086 "src/libre/dialect/native/parser.c" +#line 3076 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3100,7 +3090,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1085 "src/libre/parser.act" static int @@ -3258,6 +3248,6 @@ ZL0:; return NULL; } -#line 3262 "src/libre/dialect/native/parser.c" +#line 3252 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 959bf3acc..cb9598c50 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1242 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 98afd8ec2..6a7559800 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -740,7 +740,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 156 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); @@ -776,7 +776,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -836,7 +836,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -888,7 +888,7 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI292)); if ((ZInode) == NULL) { @@ -908,7 +908,7 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 873 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI292); @@ -934,7 +934,7 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -1361,7 +1361,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 112 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1659,7 +1659,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1867,7 +1867,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 873 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); @@ -1922,7 +1922,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1963,7 +1963,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1976,7 +1976,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 914 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1990,7 +1990,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -2026,7 +2026,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2039,7 +2039,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -2110,7 +2110,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2123,7 +2123,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -2166,7 +2166,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 914 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -2180,7 +2180,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -2261,7 +2261,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { @@ -2273,7 +2273,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; @@ -2382,7 +2382,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); @@ -2429,7 +2429,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI316)); if ((ZInode) == NULL) { @@ -2449,7 +2449,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZI316); @@ -2475,7 +2475,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2556,14 +2556,14 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, { #line 849 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); #line 2562 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2608,7 +2608,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2641,7 +2641,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2699,8 +2699,6 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -2712,12 +2710,9 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI320)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 2721 "src/libre/dialect/pcre/parser.c" +#line 2716 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2768,7 +2763,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI196; -#line 2772 "src/libre/dialect/pcre/parser.c" +#line 2767 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2779,14 +2774,14 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2790 "src/libre/dialect/pcre/parser.c" +#line 2785 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2802,12 +2797,12 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(ZIc); -#line 2811 "src/libre/dialect/pcre/parser.c" +#line 2806 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2822,7 +2817,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI199; (void) ZI200; -#line 2826 "src/libre/dialect/pcre/parser.c" +#line 2821 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2833,7 +2828,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -2866,7 +2861,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2870 "src/libre/dialect/pcre/parser.c" +#line 2865 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2876,13 +2871,13 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 197 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2886 "src/libre/dialect/pcre/parser.c" +#line 2881 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2920,7 +2915,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI263; (void) ZIend; -#line 2924 "src/libre/dialect/pcre/parser.c" +#line 2919 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -2931,7 +2926,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 2935 "src/libre/dialect/pcre/parser.c" +#line 2930 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ @@ -2940,15 +2935,13 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIn) = AST_COUNT_UNBOUNDED; -#line 2944 "src/libre/dialect/pcre/parser.c" +#line 2939 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -2960,12 +2953,9 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI320)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2969 "src/libre/dialect/pcre/parser.c" +#line 2959 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2998,7 +2988,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 3002 "src/libre/dialect/pcre/parser.c" +#line 2992 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -3014,7 +3004,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI261; (void) ZIend; -#line 3018 "src/libre/dialect/pcre/parser.c" +#line 3008 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -3029,15 +3019,13 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 3033 "src/libre/dialect/pcre/parser.c" +#line 3023 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -3049,12 +3037,9 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI320)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 3058 "src/libre/dialect/pcre/parser.c" +#line 3043 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -3099,7 +3084,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIempty__pos) = RE_FLAGS_NONE; -#line 3103 "src/libre/dialect/pcre/parser.c" +#line 3088 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ @@ -3108,7 +3093,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIempty__neg) = RE_FLAGS_NONE; -#line 3112 "src/libre/dialect/pcre/parser.c" +#line 3097 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF INLINE: 233 */ @@ -3161,7 +3146,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 960 "src/libre/parser.act" +#line 955 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3170,19 +3155,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3174 "src/libre/dialect/pcre/parser.c" +#line 3159 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL5; } -#line 3186 "src/libre/dialect/pcre/parser.c" +#line 3171 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3195,16 +3180,16 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 952 "src/libre/parser.act" +#line 947 "src/libre/parser.act" (ZIflags) = *flags; -#line 3203 "src/libre/dialect/pcre/parser.c" +#line 3188 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 960 "src/libre/parser.act" +#line 955 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3213,7 +3198,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3217 "src/libre/dialect/pcre/parser.c" +#line 3202 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3223,11 +3208,11 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 956 "src/libre/parser.act" +#line 951 "src/libre/parser.act" *flags = (ZIflags); -#line 3231 "src/libre/dialect/pcre/parser.c" +#line 3216 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3255,19 +3240,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } goto ZL1; -#line 3259 "src/libre/dialect/pcre/parser.c" +#line 3244 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 3271 "src/libre/dialect/pcre/parser.c" +#line 3256 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3301,7 +3286,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3315,7 +3300,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3319 "src/libre/dialect/pcre/parser.c" +#line 3304 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ /* BEGINNING OF INLINE: 266 */ @@ -3333,7 +3318,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } goto ZL1; -#line 3337 "src/libre/dialect/pcre/parser.c" +#line 3322 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3350,7 +3335,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } goto ZL1; -#line 3354 "src/libre/dialect/pcre/parser.c" +#line 3339 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3449,7 +3434,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3453 "src/libre/dialect/pcre/parser.c" +#line 3438 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3459,14 +3444,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3470 "src/libre/dialect/pcre/parser.c" +#line 3455 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3490,14 +3475,14 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3501 "src/libre/dialect/pcre/parser.c" +#line 3486 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3514,12 +3499,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 3523 "src/libre/dialect/pcre/parser.c" +#line 3508 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -3534,7 +3519,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI212; (void) ZI213; -#line 3538 "src/libre/dialect/pcre/parser.c" +#line 3523 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3545,17 +3530,17 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 3554 "src/libre/dialect/pcre/parser.c" +#line 3539 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -3588,7 +3573,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3592 "src/libre/dialect/pcre/parser.c" +#line 3577 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3621,13 +3606,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3631 "src/libre/dialect/pcre/parser.c" +#line 3616 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 280 */ @@ -3659,7 +3644,7 @@ ZL1:; } goto ZL4; -#line 3663 "src/libre/dialect/pcre/parser.c" +#line 3648 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3692,7 +3677,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI320; (void) ZI321; -#line 3696 "src/libre/dialect/pcre/parser.c" +#line 3681 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3720,7 +3705,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3724 "src/libre/dialect/pcre/parser.c" +#line 3709 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3742,9 +3727,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 845 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, 1, NULL); + (ZIc) = ast_make_count(0, 1); -#line 3748 "src/libre/dialect/pcre/parser.c" +#line 3733 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3756,9 +3741,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 841 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3762 "src/libre/dialect/pcre/parser.c" +#line 3747 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3770,9 +3755,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 837 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3776 "src/libre/dialect/pcre/parser.c" +#line 3761 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3794,16 +3779,16 @@ ZL1:; } goto ZL2; -#line 3798 "src/libre/dialect/pcre/parser.c" +#line 3783 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { #line 849 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 3807 "src/libre/dialect/pcre/parser.c" +#line 3792 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3829,11 +3814,11 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3837 "src/libre/dialect/pcre/parser.c" +#line 3822 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3843,14 +3828,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3854 "src/libre/dialect/pcre/parser.c" +#line 3839 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 282 */ @@ -3876,7 +3861,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } goto ZL1; -#line 3880 "src/libre/dialect/pcre/parser.c" +#line 3865 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -3910,19 +3895,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3914 "src/libre/dialect/pcre/parser.c" +#line 3899 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3926 "src/libre/dialect/pcre/parser.c" +#line 3911 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3932,14 +3917,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 976 "src/libre/parser.act" +#line 971 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 3943 "src/libre/dialect/pcre/parser.c" +#line 3928 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3949,7 +3934,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 983 "src/libre/parser.act" +#line 978 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3959,7 +3944,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3963 "src/libre/dialect/pcre/parser.c" +#line 3948 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3982,129 +3967,129 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3986 "src/libre/dialect/pcre/parser.c" +#line 3971 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 3998 "src/libre/dialect/pcre/parser.c" +#line 3983 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { goto ZL1; } -#line 4010 "src/libre/dialect/pcre/parser.c" +#line 3995 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIcr) = '\r'; -#line 4019 "src/libre/dialect/pcre/parser.c" +#line 4004 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { goto ZL1; } -#line 4031 "src/libre/dialect/pcre/parser.c" +#line 4016 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 927 "src/libre/parser.act" +#line 922 "src/libre/parser.act" (ZInl) = '\n'; -#line 4040 "src/libre/dialect/pcre/parser.c" +#line 4025 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { goto ZL1; } -#line 4052 "src/libre/dialect/pcre/parser.c" +#line 4037 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4063 "src/libre/dialect/pcre/parser.c" +#line 4048 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4074 "src/libre/dialect/pcre/parser.c" +#line 4059 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 4086 "src/libre/dialect/pcre/parser.c" +#line 4071 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4097 "src/libre/dialect/pcre/parser.c" +#line 4082 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4108 "src/libre/dialect/pcre/parser.c" +#line 4093 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4118,20 +4103,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 952 "src/libre/parser.act" +#line 947 "src/libre/parser.act" (ZIflags) = *flags; -#line 4126 "src/libre/dialect/pcre/parser.c" +#line 4111 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 4135 "src/libre/dialect/pcre/parser.c" +#line 4120 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4141,23 +4126,23 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 956 "src/libre/parser.act" +#line 951 "src/libre/parser.act" *flags = (ZIflags); -#line 4149 "src/libre/dialect/pcre/parser.c" +#line 4134 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 4161 "src/libre/dialect/pcre/parser.c" +#line 4146 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4174,14 +4159,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 969 "src/libre/parser.act" +#line 964 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 4185 "src/libre/dialect/pcre/parser.c" +#line 4170 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4240,19 +4225,19 @@ ZL1:; } goto ZL2; -#line 4244 "src/libre/dialect/pcre/parser.c" +#line 4229 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 4256 "src/libre/dialect/pcre/parser.c" +#line 4241 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4278,14 +4263,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4289 "src/libre/dialect/pcre/parser.c" +#line 4274 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4299,14 +4284,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4310 "src/libre/dialect/pcre/parser.c" +#line 4295 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4342,25 +4327,25 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4353 "src/libre/dialect/pcre/parser.c" +#line 4338 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4364 "src/libre/dialect/pcre/parser.c" +#line 4349 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-expr */ @@ -4380,7 +4365,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t (ZInode)->u.class.end = ast_end; */ -#line 4384 "src/libre/dialect/pcre/parser.c" +#line 4369 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -4394,7 +4379,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1085 "src/libre/parser.act" static int @@ -4552,6 +4537,6 @@ ZL0:; return NULL; } -#line 4556 "src/libre/dialect/pcre/parser.c" +#line 4541 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 20b24942d..b69ef5866 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1242 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index 4592ebd14..e23a44f3e 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -372,7 +372,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -384,7 +384,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -420,7 +420,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -434,7 +434,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -503,7 +503,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; @@ -579,7 +579,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1073 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -707,7 +707,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { @@ -799,7 +799,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { @@ -918,7 +918,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 163 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 993 "src/libre/parser.act" +#line 988 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { @@ -953,7 +953,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -996,7 +996,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1033,7 +1033,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1066,7 +1066,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1114,7 +1114,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1126,7 +1126,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -1137,7 +1137,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1184,7 +1184,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1246,7 +1246,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI206)); if ((ZInode) == NULL) { @@ -1271,7 +1271,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; (ZIa).u.literal.c = (unsigned char)(*ZI206); @@ -1322,7 +1322,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char)(ZIcz); @@ -1342,7 +1342,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" struct ast_pos ast_start, ast_end; unsigned char lower, upper; @@ -1430,8 +1430,6 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -1443,12 +1441,9 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI210)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1452 "src/libre/dialect/sql/parser.c" +#line 1447 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1484,7 +1479,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1488 "src/libre/dialect/sql/parser.c" +#line 1483 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1504,7 +1499,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI180; (void) ZIend; -#line 1508 "src/libre/dialect/sql/parser.c" +#line 1503 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1519,15 +1514,13 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1523 "src/libre/dialect/sql/parser.c" +#line 1518 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { #line 853 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); @@ -1539,12 +1532,9 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI210)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1548 "src/libre/dialect/sql/parser.c" +#line 1538 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1579,13 +1569,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1079 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1589 "src/libre/dialect/sql/parser.c" +#line 1579 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 190 */ @@ -1617,7 +1607,7 @@ ZL1:; } goto ZL4; -#line 1621 "src/libre/dialect/sql/parser.c" +#line 1611 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1650,7 +1640,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI210; (void) ZI211; -#line 1654 "src/libre/dialect/sql/parser.c" +#line 1644 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1678,7 +1668,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1682 "src/libre/dialect/sql/parser.c" +#line 1672 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1700,9 +1690,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 845 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, 1, NULL); + (ZIc) = ast_make_count(0, 1); -#line 1706 "src/libre/dialect/sql/parser.c" +#line 1696 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1714,9 +1704,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 841 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1720 "src/libre/dialect/sql/parser.c" +#line 1710 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1728,9 +1718,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 837 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1734 "src/libre/dialect/sql/parser.c" +#line 1724 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1741,9 +1731,9 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { #line 849 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 1747 "src/libre/dialect/sql/parser.c" +#line 1737 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1763,16 +1753,16 @@ ZL1:; } goto ZL2; -#line 1767 "src/libre/dialect/sql/parser.c" +#line 1757 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { #line 849 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 1776 "src/libre/dialect/sql/parser.c" +#line 1766 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1802,19 +1792,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1806 "src/libre/dialect/sql/parser.c" +#line 1796 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1818 "src/libre/dialect/sql/parser.c" +#line 1808 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1840,20 +1830,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1844 "src/libre/dialect/sql/parser.c" +#line 1834 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 903 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1857 "src/libre/dialect/sql/parser.c" +#line 1847 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1872,33 +1862,33 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1876 "src/libre/dialect/sql/parser.c" +#line 1866 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1888 "src/libre/dialect/sql/parser.c" +#line 1878 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { #line 837 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1897 "src/libre/dialect/sql/parser.c" +#line 1887 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1912,7 +1902,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1916 "src/libre/dialect/sql/parser.c" +#line 1906 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1925,11 +1915,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 910 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 1933 "src/libre/dialect/sql/parser.c" +#line 1923 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1939,14 +1929,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 940 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 1950 "src/libre/dialect/sql/parser.c" +#line 1940 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1984,19 +1974,19 @@ ZL1:; } goto ZL2; -#line 1988 "src/libre/dialect/sql/parser.c" +#line 1978 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2000 "src/libre/dialect/sql/parser.c" +#line 1990 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2039,7 +2029,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI129; (void) ZI130; -#line 2043 "src/libre/dialect/sql/parser.c" +#line 2033 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2049,14 +2039,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1066 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2060 "src/libre/dialect/sql/parser.c" +#line 2050 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2079,14 +2069,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2090 "src/libre/dialect/sql/parser.c" +#line 2080 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2100,14 +2090,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2111 "src/libre/dialect/sql/parser.c" +#line 2101 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2125,7 +2115,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1085 "src/libre/parser.act" static int @@ -2283,6 +2273,6 @@ ZL0:; return NULL; } -#line 2287 "src/libre/dialect/sql/parser.c" +#line 2277 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 63ed8e773..26f0e6b76 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1242 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index b836510ae..761518c05 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -835,24 +835,22 @@ @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + @c = ast_make_count(0, AST_COUNT_UNBOUNDED); @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + @c = ast_make_count(1, AST_COUNT_UNBOUNDED); @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(0, NULL, 1, NULL); + @c = ast_make_count(0, 1); @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(1, NULL, 1, NULL); + @c = ast_make_count(1, 1); @}; : (min :unsigned, start :pos, max :unsigned, end :pos) -> (c :ast_count) = @{ - struct ast_pos ast_start, ast_end; - if (@max < @min) { err->e = RE_ENEGCOUNT; err->m = @min; @@ -864,10 +862,7 @@ @!; } - AST_POS_OF_LX_POS(ast_start, @start); - AST_POS_OF_LX_POS(ast_end, @end); - - @c = ast_make_count(@min, &ast_start, @max, &ast_end); + @c = ast_make_count(@min, @max); @}; : (c :char) -> (e :endpoint) = @{ From 07e8fc1c6d8d086318d6fd8ea09b7277514307a6 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 29 May 2024 13:58:54 +0100 Subject: [PATCH 106/373] Refactoring: No need to carry lexical positions in the ast_expr .range node. --- src/libre/ast.c | 8 +- src/libre/ast.h | 5 +- src/libre/dialect/glob/parser.c | 8 +- src/libre/dialect/glob/parser.h | 2 +- src/libre/dialect/like/parser.c | 8 +- src/libre/dialect/like/parser.h | 2 +- src/libre/dialect/literal/parser.c | 4 +- src/libre/dialect/literal/parser.h | 2 +- src/libre/dialect/native/parser.c | 130 +++++------- src/libre/dialect/native/parser.h | 2 +- src/libre/dialect/pcre/parser.c | 330 ++++++++++++++--------------- src/libre/dialect/pcre/parser.h | 2 +- src/libre/dialect/sql/parser.c | 94 ++++---- src/libre/dialect/sql/parser.h | 2 +- src/libre/parser.act | 6 +- 15 files changed, 279 insertions(+), 326 deletions(-) diff --git a/src/libre/ast.c b/src/libre/ast.c index 6086bf1f5..43da60ea4 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -784,8 +784,7 @@ ast_make_expr_subtract(struct ast_expr_pool **poolp, enum re_flags re_flags, str struct ast_expr * ast_make_expr_range(struct ast_expr_pool **poolp, enum re_flags re_flags, - const struct ast_endpoint *from, struct ast_pos start, - const struct ast_endpoint *to, struct ast_pos end) + const struct ast_endpoint *from, const struct ast_endpoint *to) { struct ast_expr *res; @@ -800,9 +799,7 @@ ast_make_expr_range(struct ast_expr_pool **poolp, enum re_flags re_flags, res->type = AST_EXPR_RANGE; res->re_flags = re_flags; res->u.range.from = *from; - res->u.range.start = start; res->u.range.to = *to; - res->u.range.end = end; return res; } @@ -842,7 +839,6 @@ ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const } } else { struct ast_endpoint from, to; - struct ast_pos pos = { 0, 0, 0 }; /* XXX: pass in pos */ from.type = AST_ENDPOINT_LITERAL; if (class->ranges[i].a <= UCHAR_MAX) { @@ -858,7 +854,7 @@ ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const to.u.codepoint.u = class->ranges[i].b; } - res->u.alt.n[i] = ast_make_expr_range(poolp, re_flags, &from, pos, &to, pos); + res->u.alt.n[i] = ast_make_expr_range(poolp, re_flags, &from, &to); if (res->u.alt.n[i] == NULL) { goto error; } diff --git a/src/libre/ast.h b/src/libre/ast.h index 824a637bb..eca93b037 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -204,9 +204,7 @@ struct ast_expr { struct { struct ast_endpoint from; - struct ast_pos start; struct ast_endpoint to; - struct ast_pos end; } range; struct { @@ -319,8 +317,7 @@ ast_add_expr_concat(struct ast_expr *cat, struct ast_expr *node); struct ast_expr * ast_make_expr_range(struct ast_expr_pool **poolp, enum re_flags re_flags, - const struct ast_endpoint *from, struct ast_pos start, - const struct ast_endpoint *to, struct ast_pos end); + const struct ast_endpoint *from, const struct ast_endpoint *to); struct ast_expr * ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const struct class *class); diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 7ae0125e5..908a86ce2 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -323,7 +323,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -377,7 +377,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -447,7 +447,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -645,7 +645,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1085 "src/libre/parser.act" +#line 1081 "src/libre/parser.act" static int diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 2b07358d3..5fdb546b1 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1242 "src/libre/parser.act" +#line 1238 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index 07e8a66e0..91e603eb9 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -323,7 +323,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -377,7 +377,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -447,7 +447,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -645,7 +645,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1085 "src/libre/parser.act" +#line 1081 "src/libre/parser.act" static int diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index 38fd1e3e0..a56faf2f9 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1242 "src/libre/parser.act" +#line 1238 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 3a072f4b5..126875408 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -323,7 +323,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -558,7 +558,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1085 "src/libre/parser.act" +#line 1081 "src/libre/parser.act" static int diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index e1d707b33..96243a59d 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1242 "src/libre/parser.act" +#line 1238 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 808b02377..f3d2f0f92 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -715,7 +715,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -814,7 +814,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -1491,7 +1491,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -1681,7 +1681,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -1774,7 +1774,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; @@ -2010,12 +2010,8 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (ZIrstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2036,12 +2032,12 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2045 "src/libre/dialect/native/parser.c" +#line 2041 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2051,13 +2047,13 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 183 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2061 "src/libre/dialect/native/parser.c" +#line 2057 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2107,7 +2103,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2111 "src/libre/dialect/native/parser.c" +#line 2107 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2138,7 +2134,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 2142 "src/libre/dialect/native/parser.c" +#line 2138 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2159,7 +2155,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 2163 "src/libre/dialect/native/parser.c" +#line 2159 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2179,7 +2175,7 @@ ZL1:; } goto ZL2; -#line 2183 "src/libre/dialect/native/parser.c" +#line 2179 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2191,7 +2187,7 @@ ZL1:; goto ZL2; } -#line 2195 "src/libre/dialect/native/parser.c" +#line 2191 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2221,7 +2217,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a (ZIid) = act_state->group_id++; -#line 2225 "src/libre/dialect/native/parser.c" +#line 2221 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -2238,7 +2234,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a goto ZL1; } -#line 2242 "src/libre/dialect/native/parser.c" +#line 2238 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 233 */ @@ -2264,7 +2260,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } goto ZL1; -#line 2268 "src/libre/dialect/native/parser.c" +#line 2264 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2297,7 +2293,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2301 "src/libre/dialect/native/parser.c" +#line 2297 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2319,7 +2315,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2323 "src/libre/dialect/native/parser.c" +#line 2319 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2334,7 +2330,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI198; (void) ZI199; -#line 2338 "src/libre/dialect/native/parser.c" +#line 2334 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2350,19 +2346,15 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 2354 "src/libre/dialect/native/parser.c" +#line 2350 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2383,12 +2375,12 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2392 "src/libre/dialect/native/parser.c" +#line 2384 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2469,13 +2461,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2479 "src/libre/dialect/native/parser.c" +#line 2471 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 231 */ @@ -2507,7 +2499,7 @@ ZL1:; } goto ZL4; -#line 2511 "src/libre/dialect/native/parser.c" +#line 2503 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2540,7 +2532,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI265; (void) ZI266; -#line 2544 "src/libre/dialect/native/parser.c" +#line 2536 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2568,7 +2560,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2572 "src/libre/dialect/native/parser.c" +#line 2564 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2592,7 +2584,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, 1); -#line 2596 "src/libre/dialect/native/parser.c" +#line 2588 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2606,7 +2598,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2610 "src/libre/dialect/native/parser.c" +#line 2602 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2620,7 +2612,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2624 "src/libre/dialect/native/parser.c" +#line 2616 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2633,7 +2625,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, 1); -#line 2637 "src/libre/dialect/native/parser.c" +#line 2629 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2653,7 +2645,7 @@ ZL1:; } goto ZL2; -#line 2657 "src/libre/dialect/native/parser.c" +#line 2649 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -2662,7 +2654,7 @@ ZL1:; (ZIc) = ast_make_count(1, 1); -#line 2666 "src/libre/dialect/native/parser.c" +#line 2658 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2692,19 +2684,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2696 "src/libre/dialect/native/parser.c" +#line 2688 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2708 "src/libre/dialect/native/parser.c" +#line 2700 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2721,7 +2713,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2725 "src/libre/dialect/native/parser.c" +#line 2717 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2738,7 +2730,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 2742 "src/libre/dialect/native/parser.c" +#line 2734 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2755,7 +2747,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2759 "src/libre/dialect/native/parser.c" +#line 2751 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2779,7 +2771,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2783 "src/libre/dialect/native/parser.c" +#line 2775 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2819,7 +2811,7 @@ ZL1:; } goto ZL2; -#line 2823 "src/libre/dialect/native/parser.c" +#line 2815 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2831,7 +2823,7 @@ ZL1:; goto ZL2; } -#line 2835 "src/libre/dialect/native/parser.c" +#line 2827 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2853,14 +2845,14 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI245)); if ((ZInode) == NULL) { goto ZL1; } -#line 2864 "src/libre/dialect/native/parser.c" +#line 2856 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2878,7 +2870,7 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI245); -#line 2882 "src/libre/dialect/native/parser.c" +#line 2874 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_155 (flags, lex_state, act_state, err); @@ -2894,19 +2886,15 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla mark(&act_state->rangestart, &(*ZI246)); mark(&act_state->rangeend, &(ZIend)); -#line 2898 "src/libre/dialect/native/parser.c" +#line 2890 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI246)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2927,12 +2915,12 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 2936 "src/libre/dialect/native/parser.c" +#line 2924 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2966,7 +2954,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2970 "src/libre/dialect/native/parser.c" +#line 2958 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -3000,7 +2988,7 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3004 "src/libre/dialect/native/parser.c" +#line 2992 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3018,7 +3006,7 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZI249); -#line 3022 "src/libre/dialect/native/parser.c" +#line 3010 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_155 (flags, lex_state, act_state, err); @@ -3034,19 +3022,15 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI250)); mark(&act_state->rangeend, &(ZIend)); -#line 3038 "src/libre/dialect/native/parser.c" +#line 3026 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI250)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -3067,12 +3051,12 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 3076 "src/libre/dialect/native/parser.c" +#line 3060 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3090,7 +3074,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1085 "src/libre/parser.act" +#line 1081 "src/libre/parser.act" static int @@ -3248,6 +3232,6 @@ ZL0:; return NULL; } -#line 3252 "src/libre/dialect/native/parser.c" +#line 3236 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index cb9598c50..16c706775 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1242 "src/libre/parser.act" +#line 1238 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 6a7559800..9b4e3a0c3 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -776,7 +776,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -836,7 +836,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -888,7 +888,7 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI292)); if ((ZInode) == NULL) { @@ -936,12 +936,8 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI293)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -962,12 +958,12 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 971 "src/libre/dialect/pcre/parser.c" +#line 967 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1008,7 +1004,7 @@ p_169(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI171; (void) ZI172; -#line 1012 "src/libre/dialect/pcre/parser.c" +#line 1008 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1029,7 +1025,7 @@ ZL1:; } goto ZL2; -#line 1033 "src/libre/dialect/pcre/parser.c" +#line 1029 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1074,7 +1070,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 1078 "src/libre/dialect/pcre/parser.c" +#line 1074 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1124,7 +1120,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI124; (void) ZI125; -#line 1128 "src/libre/dialect/pcre/parser.c" +#line 1124 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1163,7 +1159,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI114; (void) ZI115; -#line 1167 "src/libre/dialect/pcre/parser.c" +#line 1163 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1224,7 +1220,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1228 "src/libre/dialect/pcre/parser.c" +#line 1224 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1251,7 +1247,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI116; (void) ZI117; -#line 1255 "src/libre/dialect/pcre/parser.c" +#line 1251 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1307,7 +1303,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1311 "src/libre/dialect/pcre/parser.c" +#line 1307 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1336,7 +1332,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI126; (void) ZI127; -#line 1340 "src/libre/dialect/pcre/parser.c" +#line 1336 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; @@ -1349,7 +1345,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err } goto ZL1; -#line 1353 "src/libre/dialect/pcre/parser.c" +#line 1349 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1368,7 +1364,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } -#line 1372 "src/libre/dialect/pcre/parser.c" +#line 1368 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1407,7 +1403,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI308 = lex_state->buf.a[0]; -#line 1411 "src/libre/dialect/pcre/parser.c" +#line 1407 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1463,7 +1459,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI313; (void) ZI314; -#line 1467 "src/libre/dialect/pcre/parser.c" +#line 1463 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1476,7 +1472,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ } goto ZL1; -#line 1480 "src/libre/dialect/pcre/parser.c" +#line 1476 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_319 (flags, lex_state, act_state, err, &ZI312, &ZI313, &ZInode); @@ -1520,7 +1516,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI297; (void) ZI298; -#line 1524 "src/libre/dialect/pcre/parser.c" +#line 1520 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1587,7 +1583,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI304 = (char) (unsigned char) u; -#line 1591 "src/libre/dialect/pcre/parser.c" +#line 1587 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1620,7 +1616,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI293; (void) ZI294; -#line 1624 "src/libre/dialect/pcre/parser.c" +#line 1620 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1653,7 +1649,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI135; (void) ZI136; -#line 1657 "src/libre/dialect/pcre/parser.c" +#line 1653 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1666,7 +1662,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } -#line 1670 "src/libre/dialect/pcre/parser.c" +#line 1666 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1722,7 +1718,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI300 = (char) (unsigned char) u; -#line 1726 "src/libre/dialect/pcre/parser.c" +#line 1722 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1757,7 +1753,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI317; (void) ZI318; -#line 1761 "src/libre/dialect/pcre/parser.c" +#line 1757 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; @@ -1770,7 +1766,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ } goto ZL1; -#line 1774 "src/libre/dialect/pcre/parser.c" +#line 1770 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_319 (flags, lex_state, act_state, err, &ZI316, &ZI317, &ZInode); @@ -1816,7 +1812,7 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err } goto ZL1; -#line 1820 "src/libre/dialect/pcre/parser.c" +#line 1816 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1857,7 +1853,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1861 "src/libre/dialect/pcre/parser.c" +#line 1857 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1872,7 +1868,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1876 "src/libre/dialect/pcre/parser.c" +#line 1872 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1916,7 +1912,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI182; -#line 1920 "src/libre/dialect/pcre/parser.c" +#line 1916 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; @@ -1929,7 +1925,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1933 "src/libre/dialect/pcre/parser.c" +#line 1929 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1957,7 +1953,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI201; -#line 1961 "src/libre/dialect/pcre/parser.c" +#line 1957 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; @@ -1970,7 +1966,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1974 "src/libre/dialect/pcre/parser.c" +#line 1970 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1980,7 +1976,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZIcbrak) = ']'; -#line 1984 "src/libre/dialect/pcre/parser.c" +#line 1980 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1990,13 +1986,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2000 "src/libre/dialect/pcre/parser.c" +#line 1996 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2020,7 +2016,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI193; -#line 2024 "src/libre/dialect/pcre/parser.c" +#line 2020 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; @@ -2033,7 +2029,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2037 "src/libre/dialect/pcre/parser.c" +#line 2033 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -2077,7 +2073,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2081 "src/libre/dialect/pcre/parser.c" +#line 2077 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ p_195 (flags, lex_state, act_state, err, &ZItmp); @@ -2104,7 +2100,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI208; -#line 2108 "src/libre/dialect/pcre/parser.c" +#line 2104 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; @@ -2117,7 +2113,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2121 "src/libre/dialect/pcre/parser.c" +#line 2117 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -2161,7 +2157,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2165 "src/libre/dialect/pcre/parser.c" +#line 2161 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ @@ -2170,7 +2166,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZIcbrak) = ']'; -#line 2174 "src/libre/dialect/pcre/parser.c" +#line 2170 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -2180,13 +2176,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2190 "src/libre/dialect/pcre/parser.c" +#line 2186 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2221,7 +2217,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI216; (void) ZIend; -#line 2225 "src/libre/dialect/pcre/parser.c" +#line 2221 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; @@ -2232,7 +2228,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2236 "src/libre/dialect/pcre/parser.c" +#line 2232 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2255,7 +2251,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI218; (void) ZIend; -#line 2259 "src/libre/dialect/pcre/parser.c" +#line 2255 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; @@ -2268,18 +2264,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL4; } -#line 2272 "src/libre/dialect/pcre/parser.c" +#line 2268 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 2283 "src/libre/dialect/pcre/parser.c" +#line 2279 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ @@ -2289,7 +2285,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2293 "src/libre/dialect/pcre/parser.c" +#line 2289 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2309,7 +2305,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } goto ZL1; -#line 2313 "src/libre/dialect/pcre/parser.c" +#line 2309 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ ZIend = ZIstart; @@ -2334,7 +2330,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (ZItmp)->u.class.end = ast_end; */ -#line 2338 "src/libre/dialect/pcre/parser.c" +#line 2334 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -2376,7 +2372,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI164; (void) ZIend; -#line 2380 "src/libre/dialect/pcre/parser.c" +#line 2376 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2387,7 +2383,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(ZIc); -#line 2391 "src/libre/dialect/pcre/parser.c" +#line 2387 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2436,7 +2432,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2440 "src/libre/dialect/pcre/parser.c" +#line 2436 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2454,7 +2450,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZI316); -#line 2458 "src/libre/dialect/pcre/parser.c" +#line 2454 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_169 (flags, lex_state, act_state, err); @@ -2470,19 +2466,15 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI317)); mark(&act_state->rangeend, &(ZIend)); -#line 2474 "src/libre/dialect/pcre/parser.c" +#line 2470 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI317)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2503,12 +2495,12 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 2512 "src/libre/dialect/pcre/parser.c" +#line 2504 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2558,7 +2550,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, (ZIc) = ast_make_count(1, 1); -#line 2562 "src/libre/dialect/pcre/parser.c" +#line 2554 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -2577,7 +2569,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2581 "src/libre/dialect/pcre/parser.c" +#line 2573 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2615,7 +2607,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex goto ZL1; } -#line 2619 "src/libre/dialect/pcre/parser.c" +#line 2611 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2636,7 +2628,7 @@ ZL1:; } goto ZL2; -#line 2640 "src/libre/dialect/pcre/parser.c" +#line 2632 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2648,7 +2640,7 @@ ZL1:; goto ZL2; } -#line 2652 "src/libre/dialect/pcre/parser.c" +#line 2644 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2681,7 +2673,7 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI258; (void) ZIend; -#line 2685 "src/libre/dialect/pcre/parser.c" +#line 2677 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -2692,7 +2684,7 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 2696 "src/libre/dialect/pcre/parser.c" +#line 2688 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ @@ -2712,7 +2704,7 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 2716 "src/libre/dialect/pcre/parser.c" +#line 2708 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2763,7 +2755,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI196; -#line 2767 "src/libre/dialect/pcre/parser.c" +#line 2759 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2781,7 +2773,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2785 "src/libre/dialect/pcre/parser.c" +#line 2777 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2802,7 +2794,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(ZIc); -#line 2806 "src/libre/dialect/pcre/parser.c" +#line 2798 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -2817,7 +2809,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI199; (void) ZI200; -#line 2821 "src/libre/dialect/pcre/parser.c" +#line 2813 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2830,12 +2822,8 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (ZIrstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2856,12 +2844,12 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2865 "src/libre/dialect/pcre/parser.c" +#line 2853 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2871,13 +2859,13 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 197 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2881 "src/libre/dialect/pcre/parser.c" +#line 2869 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2915,7 +2903,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI263; (void) ZIend; -#line 2919 "src/libre/dialect/pcre/parser.c" +#line 2907 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -2926,7 +2914,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 2930 "src/libre/dialect/pcre/parser.c" +#line 2918 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ @@ -2935,7 +2923,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIn) = AST_COUNT_UNBOUNDED; -#line 2939 "src/libre/dialect/pcre/parser.c" +#line 2927 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ @@ -2955,7 +2943,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2959 "src/libre/dialect/pcre/parser.c" +#line 2947 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2988,7 +2976,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 2992 "src/libre/dialect/pcre/parser.c" +#line 2980 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -3004,7 +2992,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI261; (void) ZIend; -#line 3008 "src/libre/dialect/pcre/parser.c" +#line 2996 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -3019,7 +3007,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 3023 "src/libre/dialect/pcre/parser.c" +#line 3011 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ @@ -3039,7 +3027,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 3043 "src/libre/dialect/pcre/parser.c" +#line 3031 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -3084,7 +3072,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIempty__pos) = RE_FLAGS_NONE; -#line 3088 "src/libre/dialect/pcre/parser.c" +#line 3076 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ @@ -3093,7 +3081,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIempty__neg) = RE_FLAGS_NONE; -#line 3097 "src/libre/dialect/pcre/parser.c" +#line 3085 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF INLINE: 233 */ @@ -3155,7 +3143,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3159 "src/libre/dialect/pcre/parser.c" +#line 3147 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -3167,7 +3155,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL5; } -#line 3171 "src/libre/dialect/pcre/parser.c" +#line 3159 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3184,7 +3172,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, (ZIflags) = *flags; -#line 3188 "src/libre/dialect/pcre/parser.c" +#line 3176 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ @@ -3198,7 +3186,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3202 "src/libre/dialect/pcre/parser.c" +#line 3190 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3212,7 +3200,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags = (ZIflags); -#line 3216 "src/libre/dialect/pcre/parser.c" +#line 3204 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3240,7 +3228,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } goto ZL1; -#line 3244 "src/libre/dialect/pcre/parser.c" +#line 3232 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -3252,7 +3240,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 3256 "src/libre/dialect/pcre/parser.c" +#line 3244 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3300,7 +3288,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3304 "src/libre/dialect/pcre/parser.c" +#line 3292 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ /* BEGINNING OF INLINE: 266 */ @@ -3318,7 +3306,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } goto ZL1; -#line 3322 "src/libre/dialect/pcre/parser.c" +#line 3310 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3335,7 +3323,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } goto ZL1; -#line 3339 "src/libre/dialect/pcre/parser.c" +#line 3327 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3434,7 +3422,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3438 "src/libre/dialect/pcre/parser.c" +#line 3426 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3444,14 +3432,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3455 "src/libre/dialect/pcre/parser.c" +#line 3443 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3482,7 +3470,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3486 "src/libre/dialect/pcre/parser.c" +#line 3474 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3504,7 +3492,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); -#line 3508 "src/libre/dialect/pcre/parser.c" +#line 3496 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ @@ -3519,7 +3507,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI212; (void) ZI213; -#line 3523 "src/libre/dialect/pcre/parser.c" +#line 3511 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3535,19 +3523,15 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); -#line 3539 "src/libre/dialect/pcre/parser.c" +#line 3527 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -3568,12 +3552,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3577 "src/libre/dialect/pcre/parser.c" +#line 3561 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3606,13 +3590,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3616 "src/libre/dialect/pcre/parser.c" +#line 3600 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 280 */ @@ -3644,7 +3628,7 @@ ZL1:; } goto ZL4; -#line 3648 "src/libre/dialect/pcre/parser.c" +#line 3632 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3677,7 +3661,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI320; (void) ZI321; -#line 3681 "src/libre/dialect/pcre/parser.c" +#line 3665 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3705,7 +3689,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3709 "src/libre/dialect/pcre/parser.c" +#line 3693 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3729,7 +3713,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, 1); -#line 3733 "src/libre/dialect/pcre/parser.c" +#line 3717 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3743,7 +3727,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3747 "src/libre/dialect/pcre/parser.c" +#line 3731 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3757,7 +3741,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3761 "src/libre/dialect/pcre/parser.c" +#line 3745 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3779,7 +3763,7 @@ ZL1:; } goto ZL2; -#line 3783 "src/libre/dialect/pcre/parser.c" +#line 3767 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -3788,7 +3772,7 @@ ZL1:; (ZIc) = ast_make_count(1, 1); -#line 3792 "src/libre/dialect/pcre/parser.c" +#line 3776 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3818,7 +3802,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast (ZIid) = act_state->group_id++; -#line 3822 "src/libre/dialect/pcre/parser.c" +#line 3806 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3835,7 +3819,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast goto ZL1; } -#line 3839 "src/libre/dialect/pcre/parser.c" +#line 3823 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 282 */ @@ -3861,7 +3845,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } goto ZL1; -#line 3865 "src/libre/dialect/pcre/parser.c" +#line 3849 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -3895,19 +3879,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3899 "src/libre/dialect/pcre/parser.c" +#line 3883 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3911 "src/libre/dialect/pcre/parser.c" +#line 3895 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3924,7 +3908,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 3928 "src/libre/dialect/pcre/parser.c" +#line 3912 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3944,7 +3928,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3948 "src/libre/dialect/pcre/parser.c" +#line 3932 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3967,19 +3951,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3971 "src/libre/dialect/pcre/parser.c" +#line 3955 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 3983 "src/libre/dialect/pcre/parser.c" +#line 3967 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ @@ -3991,7 +3975,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 3995 "src/libre/dialect/pcre/parser.c" +#line 3979 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ @@ -4000,7 +3984,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIcr) = '\r'; -#line 4004 "src/libre/dialect/pcre/parser.c" +#line 3988 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ @@ -4012,7 +3996,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4016 "src/libre/dialect/pcre/parser.c" +#line 4000 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ @@ -4021,7 +4005,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZInl) = '\n'; -#line 4025 "src/libre/dialect/pcre/parser.c" +#line 4009 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ @@ -4033,29 +4017,29 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4037 "src/libre/dialect/pcre/parser.c" +#line 4021 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4048 "src/libre/dialect/pcre/parser.c" +#line 4032 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4059 "src/libre/dialect/pcre/parser.c" +#line 4043 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ @@ -4067,29 +4051,29 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4071 "src/libre/dialect/pcre/parser.c" +#line 4055 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4082 "src/libre/dialect/pcre/parser.c" +#line 4066 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4093 "src/libre/dialect/pcre/parser.c" +#line 4077 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4107,7 +4091,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIflags) = *flags; -#line 4111 "src/libre/dialect/pcre/parser.c" +#line 4095 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ @@ -4116,7 +4100,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 4120 "src/libre/dialect/pcre/parser.c" +#line 4104 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4130,7 +4114,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e *flags = (ZIflags); -#line 4134 "src/libre/dialect/pcre/parser.c" +#line 4118 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ @@ -4142,7 +4126,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4146 "src/libre/dialect/pcre/parser.c" +#line 4130 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4166,7 +4150,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 4170 "src/libre/dialect/pcre/parser.c" +#line 4154 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4225,7 +4209,7 @@ ZL1:; } goto ZL2; -#line 4229 "src/libre/dialect/pcre/parser.c" +#line 4213 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -4237,7 +4221,7 @@ ZL1:; goto ZL2; } -#line 4241 "src/libre/dialect/pcre/parser.c" +#line 4225 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4270,7 +4254,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 4274 "src/libre/dialect/pcre/parser.c" +#line 4258 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4291,7 +4275,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 4295 "src/libre/dialect/pcre/parser.c" +#line 4279 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4334,18 +4318,18 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t goto ZL1; } -#line 4338 "src/libre/dialect/pcre/parser.c" +#line 4322 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4349 "src/libre/dialect/pcre/parser.c" +#line 4333 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-expr */ @@ -4365,7 +4349,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t (ZInode)->u.class.end = ast_end; */ -#line 4369 "src/libre/dialect/pcre/parser.c" +#line 4353 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -4379,7 +4363,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1085 "src/libre/parser.act" +#line 1081 "src/libre/parser.act" static int @@ -4537,6 +4521,6 @@ ZL0:; return NULL; } -#line 4541 "src/libre/dialect/pcre/parser.c" +#line 4525 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index b69ef5866..8346c0641 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1242 "src/libre/parser.act" +#line 1238 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index e23a44f3e..c96e30107 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -384,7 +384,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -503,7 +503,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; @@ -579,7 +579,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1073 "src/libre/parser.act" +#line 1069 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -1126,7 +1126,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -1344,12 +1344,8 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { #line 1033 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI207)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIa).type != AST_ENDPOINT_LITERAL || (ZIz).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -1370,12 +1366,12 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIa), ast_start, &(ZIz), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIa), &(ZIz)); if ((ZInode) == NULL) { goto ZL1; } -#line 1379 "src/libre/dialect/sql/parser.c" +#line 1375 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1412,7 +1408,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI177; (void) ZIend; -#line 1416 "src/libre/dialect/sql/parser.c" +#line 1412 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; @@ -1423,7 +1419,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1427 "src/libre/dialect/sql/parser.c" +#line 1423 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ @@ -1443,7 +1439,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1447 "src/libre/dialect/sql/parser.c" +#line 1443 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1479,7 +1475,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1483 "src/libre/dialect/sql/parser.c" +#line 1479 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1499,7 +1495,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI180; (void) ZIend; -#line 1503 "src/libre/dialect/sql/parser.c" +#line 1499 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1514,7 +1510,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1518 "src/libre/dialect/sql/parser.c" +#line 1514 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ @@ -1534,7 +1530,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1538 "src/libre/dialect/sql/parser.c" +#line 1534 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1569,13 +1565,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1079 "src/libre/parser.act" +#line 1075 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1579 "src/libre/dialect/sql/parser.c" +#line 1575 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 190 */ @@ -1607,7 +1603,7 @@ ZL1:; } goto ZL4; -#line 1611 "src/libre/dialect/sql/parser.c" +#line 1607 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1640,7 +1636,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI210; (void) ZI211; -#line 1644 "src/libre/dialect/sql/parser.c" +#line 1640 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1668,7 +1664,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1672 "src/libre/dialect/sql/parser.c" +#line 1668 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1692,7 +1688,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, 1); -#line 1696 "src/libre/dialect/sql/parser.c" +#line 1692 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1706,7 +1702,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1710 "src/libre/dialect/sql/parser.c" +#line 1706 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1720,7 +1716,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1724 "src/libre/dialect/sql/parser.c" +#line 1720 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1733,7 +1729,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (ZIc) = ast_make_count(1, 1); -#line 1737 "src/libre/dialect/sql/parser.c" +#line 1733 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1753,7 +1749,7 @@ ZL1:; } goto ZL2; -#line 1757 "src/libre/dialect/sql/parser.c" +#line 1753 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ @@ -1762,7 +1758,7 @@ ZL1:; (ZIc) = ast_make_count(1, 1); -#line 1766 "src/libre/dialect/sql/parser.c" +#line 1762 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1792,19 +1788,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1796 "src/libre/dialect/sql/parser.c" +#line 1792 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1808 "src/libre/dialect/sql/parser.c" +#line 1804 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1830,7 +1826,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1834 "src/libre/dialect/sql/parser.c" +#line 1830 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1843,7 +1839,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1847 "src/libre/dialect/sql/parser.c" +#line 1843 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1862,19 +1858,19 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1866 "src/libre/dialect/sql/parser.c" +#line 1862 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1878 "src/libre/dialect/sql/parser.c" +#line 1874 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ @@ -1883,7 +1879,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1887 "src/libre/dialect/sql/parser.c" +#line 1883 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ @@ -1902,7 +1898,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1906 "src/libre/dialect/sql/parser.c" +#line 1902 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1919,7 +1915,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 1923 "src/libre/dialect/sql/parser.c" +#line 1919 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1936,7 +1932,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1940 "src/libre/dialect/sql/parser.c" +#line 1936 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1974,7 +1970,7 @@ ZL1:; } goto ZL2; -#line 1978 "src/libre/dialect/sql/parser.c" +#line 1974 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -1986,7 +1982,7 @@ ZL1:; goto ZL2; } -#line 1990 "src/libre/dialect/sql/parser.c" +#line 1986 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2029,7 +2025,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI129; (void) ZI130; -#line 2033 "src/libre/dialect/sql/parser.c" +#line 2029 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2039,14 +2035,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1066 "src/libre/parser.act" +#line 1062 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2050 "src/libre/dialect/sql/parser.c" +#line 2046 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2076,7 +2072,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2080 "src/libre/dialect/sql/parser.c" +#line 2076 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2097,7 +2093,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2101 "src/libre/dialect/sql/parser.c" +#line 2097 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2115,7 +2111,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1085 "src/libre/parser.act" +#line 1081 "src/libre/parser.act" static int @@ -2273,6 +2269,6 @@ ZL0:; return NULL; } -#line 2277 "src/libre/dialect/sql/parser.c" +#line 2273 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 26f0e6b76..5cc78ad47 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1242 "src/libre/parser.act" +#line 1238 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index 761518c05..9977f1ad8 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -1031,12 +1031,8 @@ @}; : (from :endpoint, start :pos, to :endpoint, end :pos) -> (node :ast_expr) = @{ - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, @start); - AST_POS_OF_LX_POS(ast_end, @end); - if (@from.type != AST_ENDPOINT_LITERAL || @to.type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -1057,7 +1053,7 @@ @!; } - @node = ast_make_expr_range(act_state->poolp, *flags, &@from, ast_start, &@to, ast_end); + @node = ast_make_expr_range(act_state->poolp, *flags, &@from, &@to); if (@node == NULL) { @!; } From bf46a2c342474843520fddfb05f6120b28d26a0b Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 29 May 2024 14:20:37 +0100 Subject: [PATCH 107/373] Whitespace. --- src/libre/parser.act | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libre/parser.act b/src/libre/parser.act index 9977f1ad8..195a61dbf 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -181,6 +181,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -465,16 +466,21 @@ assert(lex_state->buf.a[3] == '\0'); @c = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)@c; + unsigned char cc = (unsigned char) @c; if (cc > 126 || cc < 32) { @!; } @@ -867,7 +873,7 @@ : (c :char) -> (e :endpoint) = @{ @e.type = AST_ENDPOINT_LITERAL; - @e.u.literal.c = (unsigned char)@c; + @e.u.literal.c = (unsigned char) @c; @}; : (id :ast_class_id) -> (e :endpoint) = @{ From 51c42fad189e14e96b8e7ee9f9918c699b398d14 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 29 May 2024 15:17:41 +0100 Subject: [PATCH 108/373] Refactoring: No need to mark group start/end for expr in general. --- src/libre/dialect/glob/parser.c | 71 +-- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/like/parser.c | 71 +-- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/literal/parser.c | 47 +- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/native/parser.c | 420 +++++++++-------- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/pcre/parser.c | 712 +++++++++++++++-------------- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/sql/parser.c | 338 +++++++------- src/libre/dialect/sql/parser.h | 4 +- src/libre/parser.act | 9 +- 13 files changed, 839 insertions(+), 853 deletions(-) diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 908a86ce2..89af72030 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -172,6 +172,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -288,7 +289,7 @@ return s; } -#line 292 "src/libre/dialect/glob/parser.c" +#line 293 "src/libre/dialect/glob/parser.c" #ifndef ERROR_TERMINAL @@ -323,13 +324,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 333 "src/libre/dialect/glob/parser.c" +#line 334 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 116 */ @@ -367,24 +368,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 376 "src/libre/dialect/glob/parser.c" +#line 377 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 388 "src/libre/dialect/glob/parser.c" +#line 389 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -397,7 +398,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -410,20 +411,20 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 414 "src/libre/dialect/glob/parser.c" +#line 415 "src/libre/dialect/glob/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 427 "src/libre/dialect/glob/parser.c" +#line 428 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -437,38 +438,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 446 "src/libre/dialect/glob/parser.c" +#line 447 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 458 "src/libre/dialect/glob/parser.c" +#line 459 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 838 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 467 "src/libre/dialect/glob/parser.c" +#line 468 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -482,7 +483,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 486 "src/libre/dialect/glob/parser.c" +#line 487 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -497,26 +498,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 724 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 508 "src/libre/dialect/glob/parser.c" +#line 509 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 520 "src/libre/dialect/glob/parser.c" +#line 521 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -542,11 +543,11 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 550 "src/libre/dialect/glob/parser.c" +#line 551 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 120 */ @@ -556,14 +557,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 567 "src/libre/dialect/glob/parser.c" +#line 568 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -577,14 +578,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 588 "src/libre/dialect/glob/parser.c" +#line 589 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -594,14 +595,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 120 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 605 "src/libre/dialect/glob/parser.c" +#line 606 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 121 */ @@ -620,14 +621,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 773 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 631 "src/libre/dialect/glob/parser.c" +#line 632 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -645,7 +646,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1081 "src/libre/parser.act" +#line 1082 "src/libre/parser.act" static int @@ -803,6 +804,6 @@ ZL0:; return NULL; } -#line 807 "src/libre/dialect/glob/parser.c" +#line 808 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 5fdb546b1..8e69b6c57 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 301 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1238 "src/libre/parser.act" +#line 1239 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index 91e603eb9..a0b69bec6 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -172,6 +172,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -288,7 +289,7 @@ return s; } -#line 292 "src/libre/dialect/like/parser.c" +#line 293 "src/libre/dialect/like/parser.c" #ifndef ERROR_TERMINAL @@ -323,13 +324,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 333 "src/libre/dialect/like/parser.c" +#line 334 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 116 */ @@ -367,24 +368,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 376 "src/libre/dialect/like/parser.c" +#line 377 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 388 "src/libre/dialect/like/parser.c" +#line 389 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -397,7 +398,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -410,20 +411,20 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 414 "src/libre/dialect/like/parser.c" +#line 415 "src/libre/dialect/like/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 427 "src/libre/dialect/like/parser.c" +#line 428 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -437,38 +438,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 446 "src/libre/dialect/like/parser.c" +#line 447 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 458 "src/libre/dialect/like/parser.c" +#line 459 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 838 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 467 "src/libre/dialect/like/parser.c" +#line 468 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -482,7 +483,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 486 "src/libre/dialect/like/parser.c" +#line 487 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -497,26 +498,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 724 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 508 "src/libre/dialect/like/parser.c" +#line 509 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 520 "src/libre/dialect/like/parser.c" +#line 521 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -542,11 +543,11 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 550 "src/libre/dialect/like/parser.c" +#line 551 "src/libre/dialect/like/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 120 */ @@ -556,14 +557,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 567 "src/libre/dialect/like/parser.c" +#line 568 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -577,14 +578,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 588 "src/libre/dialect/like/parser.c" +#line 589 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -594,14 +595,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 120 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 605 "src/libre/dialect/like/parser.c" +#line 606 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 121 */ @@ -620,14 +621,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 773 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 631 "src/libre/dialect/like/parser.c" +#line 632 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -645,7 +646,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1081 "src/libre/parser.act" +#line 1082 "src/libre/parser.act" static int @@ -803,6 +804,6 @@ ZL0:; return NULL; } -#line 807 "src/libre/dialect/like/parser.c" +#line 808 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index a56faf2f9..fc11ce83c 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 301 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1238 "src/libre/parser.act" +#line 1239 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 126875408..067fab0ef 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -172,6 +172,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -288,7 +289,7 @@ return s; } -#line 292 "src/libre/dialect/literal/parser.c" +#line 293 "src/libre/dialect/literal/parser.c" #ifndef ERROR_TERMINAL @@ -323,13 +324,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 333 "src/libre/dialect/literal/parser.c" +#line 334 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 115 */ @@ -368,11 +369,11 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 376 "src/libre/dialect/literal/parser.c" +#line 377 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 118 */ @@ -382,14 +383,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 393 "src/libre/dialect/literal/parser.c" +#line 394 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -403,14 +404,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 414 "src/libre/dialect/literal/parser.c" +#line 415 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -420,14 +421,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* END OF INLINE: 118 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 431 "src/libre/dialect/literal/parser.c" +#line 432 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ @@ -446,14 +447,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 773 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 457 "src/libre/dialect/literal/parser.c" +#line 458 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -486,7 +487,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -499,7 +500,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 503 "src/libre/dialect/literal/parser.c" +#line 504 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -509,14 +510,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 520 "src/libre/dialect/literal/parser.c" +#line 521 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -525,26 +526,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 724 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 536 "src/libre/dialect/literal/parser.c" +#line 537 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 548 "src/libre/dialect/literal/parser.c" +#line 549 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -558,7 +559,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1081 "src/libre/parser.act" +#line 1082 "src/libre/parser.act" static int @@ -716,6 +717,6 @@ ZL0:; return NULL; } -#line 720 "src/libre/dialect/literal/parser.c" +#line 721 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index 96243a59d..f9eecbaa5 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 301 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1238 "src/libre/parser.act" +#line 1239 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index f3d2f0f92..4efd632db 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -172,6 +172,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -288,7 +289,7 @@ return s; } -#line 292 "src/libre/dialect/native/parser.c" +#line 293 "src/libre/dialect/native/parser.c" #ifndef ERROR_TERMINAL @@ -345,7 +346,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -358,7 +359,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 362 "src/libre/dialect/native/parser.c" +#line 363 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -368,7 +369,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 406 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -394,7 +395,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 398 "src/libre/dialect/native/parser.c" +#line 399 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -404,7 +405,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 547 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -452,7 +453,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 456 "src/libre/dialect/native/parser.c" +#line 457 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -462,7 +463,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 504 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -505,7 +506,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 509 "src/libre/dialect/native/parser.c" +#line 510 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -518,12 +519,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 142 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 527 "src/libre/dialect/native/parser.c" +#line 528 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -550,7 +551,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI218 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -558,23 +559,23 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI218; (void) ZIend; -#line 562 "src/libre/dialect/native/parser.c" +#line 563 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); -#line 573 "src/libre/dialect/native/parser.c" +#line 574 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -589,7 +590,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 593 "src/libre/dialect/native/parser.c" +#line 594 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -605,7 +606,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 647 "src/libre/parser.act" unsigned long u; char *e; @@ -625,7 +626,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 629 "src/libre/dialect/native/parser.c" +#line 630 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -637,7 +638,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI221 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -645,7 +646,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI221; (void) ZIend; -#line 649 "src/libre/dialect/native/parser.c" +#line 650 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -655,17 +656,17 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI265)); mark(&act_state->countend, &(ZIend)); -#line 664 "src/libre/dialect/native/parser.c" +#line 665 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -680,7 +681,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 684 "src/libre/dialect/native/parser.c" +#line 685 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -715,13 +716,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 725 "src/libre/dialect/native/parser.c" +#line 726 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -755,7 +756,7 @@ p_155(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI156 = '-'; ZI157 = lex_state->lx.start; @@ -765,7 +766,7 @@ p_155(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI157; (void) ZI158; -#line 769 "src/libre/dialect/native/parser.c" +#line 770 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -779,14 +780,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 732 "src/libre/parser.act" +#line 738 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 790 "src/libre/dialect/native/parser.c" +#line 791 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -814,13 +815,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 824 "src/libre/dialect/native/parser.c" +#line 825 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 225 */ @@ -868,7 +869,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -881,7 +882,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 885 "src/libre/dialect/native/parser.c" +#line 886 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -894,7 +895,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 406 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -920,7 +921,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI112; (void) ZI113; -#line 924 "src/libre/dialect/native/parser.c" +#line 925 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -933,7 +934,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 547 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -981,7 +982,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 985 "src/libre/dialect/native/parser.c" +#line 986 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -994,7 +995,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 504 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1037,7 +1038,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1041 "src/libre/dialect/native/parser.c" +#line 1042 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1050,14 +1051,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 110 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1061 "src/libre/dialect/native/parser.c" +#line 1062 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1083,7 +1084,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1096,7 +1097,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI261 = lex_state->buf.a[0]; -#line 1100 "src/libre/dialect/native/parser.c" +#line 1101 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1115,7 +1116,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 406 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1141,7 +1142,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI250; (void) ZI251; -#line 1145 "src/libre/dialect/native/parser.c" +#line 1146 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1160,7 +1161,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 547 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1208,7 +1209,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI257 = (char) (unsigned char) u; -#line 1212 "src/libre/dialect/native/parser.c" +#line 1213 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1227,7 +1228,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 667 "src/libre/parser.act" ZI245 = DIALECT_CLASS(lex_state->buf.a); if (ZI245 == NULL) { @@ -1241,7 +1242,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI246; (void) ZI247; -#line 1245 "src/libre/dialect/native/parser.c" +#line 1246 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1260,7 +1261,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 504 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1303,7 +1304,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI253 = (char) (unsigned char) u; -#line 1307 "src/libre/dialect/native/parser.c" +#line 1308 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1344,7 +1345,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 667 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1358,7 +1359,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1362 "src/libre/dialect/native/parser.c" +#line 1363 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1368,12 +1369,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 873 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1377 "src/libre/dialect/native/parser.c" +#line 1378 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1409,7 +1410,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 337 "src/libre/parser.act" +#line 338 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI168 = lex_state->lx.end; @@ -1417,20 +1418,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI168; -#line 1421 "src/libre/dialect/native/parser.c" +#line 1422 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1434 "src/libre/dialect/native/parser.c" +#line 1435 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1450,7 +1451,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 353 "src/libre/parser.act" +#line 354 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI187 = lex_state->lx.end; @@ -1458,30 +1459,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI187; -#line 1462 "src/libre/dialect/native/parser.c" +#line 1463 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1475 "src/libre/dialect/native/parser.c" +#line 1476 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 914 "src/libre/parser.act" +#line 915 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1485 "src/libre/dialect/native/parser.c" +#line 1486 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1491,13 +1492,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1501 "src/libre/dialect/native/parser.c" +#line 1502 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1513,7 +1514,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 345 "src/libre/parser.act" +#line 346 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI179 = lex_state->lx.end; @@ -1521,26 +1522,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI179; -#line 1525 "src/libre/dialect/native/parser.c" +#line 1526 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1538 "src/libre/dialect/native/parser.c" +#line 1539 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 996 "src/libre/parser.act" struct ast_expr *any; @@ -1578,7 +1579,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1582 "src/libre/dialect/native/parser.c" +#line 1583 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ p_181 (flags, lex_state, act_state, err, &ZItmp); @@ -1597,7 +1598,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 361 "src/libre/parser.act" +#line 362 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI194 = lex_state->lx.end; @@ -1605,26 +1606,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI194; -#line 1609 "src/libre/dialect/native/parser.c" +#line 1610 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1622 "src/libre/dialect/native/parser.c" +#line 1623 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 996 "src/libre/parser.act" struct ast_expr *any; @@ -1662,16 +1663,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1666 "src/libre/dialect/native/parser.c" +#line 1667 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 914 "src/libre/parser.act" +#line 915 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1675 "src/libre/dialect/native/parser.c" +#line 1676 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1681,13 +1682,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1691 "src/libre/dialect/native/parser.c" +#line 1692 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1712,7 +1713,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI201 = ']'; ZI202 = lex_state->lx.start; @@ -1722,18 +1723,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI202; (void) ZIend; -#line 1726 "src/libre/dialect/native/parser.c" +#line 1727 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 787 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1737 "src/libre/dialect/native/parser.c" +#line 1738 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1746,7 +1747,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 379 "src/libre/parser.act" +#line 380 "src/libre/parser.act" ZIcrange = '-'; ZI204 = lex_state->lx.start; @@ -1756,41 +1757,41 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI204; (void) ZIend; -#line 1760 "src/libre/dialect/native/parser.c" +#line 1761 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 1773 "src/libre/dialect/native/parser.c" +#line 1774 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 1784 "src/libre/dialect/native/parser.c" +#line 1785 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 787 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1794 "src/libre/dialect/native/parser.c" +#line 1795 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1803,14 +1804,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 745 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 1814 "src/libre/dialect/native/parser.c" +#line 1815 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ ZIend = ZIstart; @@ -1820,22 +1821,17 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 200 */ /* BEGINNING OF ACTION: mark-expr */ { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); +#line 802 "src/libre/parser.act" - mark(&act_state->groupstart, &(ZIstart)); - mark(&act_state->groupend, &(ZIend)); + (void) (ZIstart); + (void) (ZIend); /* TODO: reinstate this, applies to an expr node in general (ZItmp)->u.class.start = ast_start; (ZItmp)->u.class.end = ast_end; */ -#line 1839 "src/libre/dialect/native/parser.c" +#line 1835 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-expr */ } @@ -1867,7 +1863,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZIc = '-'; ZI150 = lex_state->lx.start; @@ -1877,18 +1873,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI150; (void) ZIend; -#line 1881 "src/libre/dialect/native/parser.c" +#line 1877 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 1892 "src/libre/dialect/native/parser.c" +#line 1888 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -1933,7 +1929,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -1943,7 +1939,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI182; -#line 1947 "src/libre/dialect/native/parser.c" +#line 1943 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1954,14 +1950,14 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 1965 "src/libre/dialect/native/parser.c" +#line 1961 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1977,17 +1973,17 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(ZIc); + (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 1986 "src/libre/dialect/native/parser.c" +#line 1982 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI184 = '-'; ZI185 = lex_state->lx.start; @@ -1997,7 +1993,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI185; (void) ZI186; -#line 2001 "src/libre/dialect/native/parser.c" +#line 1997 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2008,7 +2004,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -2037,7 +2033,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2041 "src/libre/dialect/native/parser.c" +#line 2037 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2047,13 +2043,13 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 183 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2057 "src/libre/dialect/native/parser.c" +#line 2053 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2089,7 +2085,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2103,7 +2099,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2107 "src/libre/dialect/native/parser.c" +#line 2103 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2127,14 +2123,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2138 "src/libre/dialect/native/parser.c" +#line 2134 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2148,14 +2144,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2159 "src/libre/dialect/native/parser.c" +#line 2155 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2168,26 +2164,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 731 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2179 "src/libre/dialect/native/parser.c" +#line 2175 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2191 "src/libre/dialect/native/parser.c" +#line 2187 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2213,11 +2209,11 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2221 "src/libre/dialect/native/parser.c" +#line 2217 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -2227,14 +2223,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2238 "src/libre/dialect/native/parser.c" +#line 2234 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 233 */ @@ -2253,14 +2249,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 773 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 2264 "src/libre/dialect/native/parser.c" +#line 2260 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2286,14 +2282,14 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2297 "src/libre/dialect/native/parser.c" +#line 2293 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2310,17 +2306,17 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2319 "src/libre/dialect/native/parser.c" +#line 2315 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI197 = '-'; ZI198 = lex_state->lx.start; @@ -2330,7 +2326,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI198; (void) ZI199; -#line 2334 "src/libre/dialect/native/parser.c" +#line 2330 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2341,17 +2337,17 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2350 "src/libre/dialect/native/parser.c" +#line 2346 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -2380,7 +2376,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2384 "src/libre/dialect/native/parser.c" +#line 2380 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2461,13 +2457,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2471 "src/libre/dialect/native/parser.c" +#line 2467 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 231 */ @@ -2492,14 +2488,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 731 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 2503 "src/libre/dialect/native/parser.c" +#line 2499 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2524,7 +2520,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 389 "src/libre/parser.act" +#line 390 "src/libre/parser.act" ZI265 = lex_state->lx.start; ZI266 = lex_state->lx.end; @@ -2532,7 +2528,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI265; (void) ZI266; -#line 2536 "src/libre/dialect/native/parser.c" +#line 2532 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2540,7 +2536,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 647 "src/libre/parser.act" unsigned long u; char *e; @@ -2560,7 +2556,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2564 "src/libre/dialect/native/parser.c" +#line 2560 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2580,11 +2576,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 845 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 2588 "src/libre/dialect/native/parser.c" +#line 2584 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2594,11 +2590,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 841 "src/libre/parser.act" +#line 842 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2602 "src/libre/dialect/native/parser.c" +#line 2598 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2608,11 +2604,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 838 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2616 "src/libre/dialect/native/parser.c" +#line 2612 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2621,11 +2617,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 850 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2629 "src/libre/dialect/native/parser.c" +#line 2625 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2638,23 +2634,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 711 "src/libre/parser.act" +#line 717 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 2649 "src/libre/dialect/native/parser.c" +#line 2645 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 850 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2658 "src/libre/dialect/native/parser.c" +#line 2654 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2679,24 +2675,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2688 "src/libre/dialect/native/parser.c" +#line 2684 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2700 "src/libre/dialect/native/parser.c" +#line 2696 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2706,14 +2702,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 971 "src/libre/parser.act" +#line 972 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 2717 "src/libre/dialect/native/parser.c" +#line 2713 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2726,11 +2722,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2734 "src/libre/dialect/native/parser.c" +#line 2730 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2740,14 +2736,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 2751 "src/libre/dialect/native/parser.c" +#line 2747 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2764,14 +2760,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 964 "src/libre/parser.act" +#line 965 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 2775 "src/libre/dialect/native/parser.c" +#line 2771 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2804,26 +2800,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 724 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 2815 "src/libre/dialect/native/parser.c" +#line 2811 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2827 "src/libre/dialect/native/parser.c" +#line 2823 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2845,14 +2841,14 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI245)); if ((ZInode) == NULL) { goto ZL1; } -#line 2856 "src/libre/dialect/native/parser.c" +#line 2852 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2865,12 +2861,12 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 873 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI245); -#line 2874 "src/libre/dialect/native/parser.c" +#line 2870 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_155 (flags, lex_state, act_state, err); @@ -2881,17 +2877,17 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 792 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI246)); mark(&act_state->rangeend, &(ZIend)); -#line 2890 "src/libre/dialect/native/parser.c" +#line 2886 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -2920,7 +2916,7 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2924 "src/libre/dialect/native/parser.c" +#line 2920 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2947,14 +2943,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2958 "src/libre/dialect/native/parser.c" +#line 2954 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2981,14 +2977,14 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI249)); if ((ZInode) == NULL) { goto ZL1; } -#line 2992 "src/libre/dialect/native/parser.c" +#line 2988 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3001,12 +2997,12 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZI249); + (ZIlower).u.literal.c = (unsigned char) (*ZI249); -#line 3010 "src/libre/dialect/native/parser.c" +#line 3006 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_155 (flags, lex_state, act_state, err); @@ -3017,17 +3013,17 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 792 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI250)); mark(&act_state->rangeend, &(ZIend)); -#line 3026 "src/libre/dialect/native/parser.c" +#line 3022 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -3056,7 +3052,7 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3060 "src/libre/dialect/native/parser.c" +#line 3056 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3074,7 +3070,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1081 "src/libre/parser.act" +#line 1082 "src/libre/parser.act" static int @@ -3232,6 +3228,6 @@ ZL0:; return NULL; } -#line 3236 "src/libre/dialect/native/parser.c" +#line 3232 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 16c706775..8591ec200 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 301 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1238 "src/libre/parser.act" +#line 1239 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 9b4e3a0c3..c1cbc2bd4 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -172,6 +172,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -288,7 +289,7 @@ return s; } -#line 292 "src/libre/dialect/pcre/parser.c" +#line 293 "src/libre/dialect/pcre/parser.c" #ifndef ERROR_TERMINAL @@ -344,21 +345,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 679 "src/libre/parser.act" +#line 685 "src/libre/parser.act" ZIc = RE_EXTENDED; -#line 352 "src/libre/dialect/pcre/parser.c" +#line 353 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_EXTENDED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 829 "src/libre/parser.act" +#line 830 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 362 "src/libre/dialect/pcre/parser.c" +#line 363 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -375,21 +376,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 675 "src/libre/parser.act" +#line 681 "src/libre/parser.act" ZIc = RE_ICASE; -#line 383 "src/libre/dialect/pcre/parser.c" +#line 384 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_INSENSITIVE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 829 "src/libre/parser.act" +#line 830 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 393 "src/libre/dialect/pcre/parser.c" +#line 394 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -400,21 +401,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 683 "src/libre/parser.act" +#line 689 "src/libre/parser.act" ZIc = RE_SINGLE; -#line 408 "src/libre/dialect/pcre/parser.c" +#line 409 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_SINGLE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 829 "src/libre/parser.act" +#line 830 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 418 "src/libre/dialect/pcre/parser.c" +#line 419 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -425,14 +426,14 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 753 "src/libre/parser.act" +#line 759 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; } goto ZL1; -#line 436 "src/libre/dialect/pcre/parser.c" +#line 437 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unknown-flag */ } @@ -470,7 +471,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -483,7 +484,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 487 "src/libre/dialect/pcre/parser.c" +#line 488 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -493,7 +494,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 461 "src/libre/parser.act" +#line 462 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -501,16 +502,21 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags assert(lex_state->buf.a[3] == '\0'); ZIc = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)ZIc; + unsigned char cc = (unsigned char) ZIc; if (cc > 126 || cc < 32) { goto ZL1; } @@ -530,20 +536,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 534 "src/libre/dialect/pcre/parser.c" +#line 540 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 547 "src/libre/dialect/pcre/parser.c" +#line 553 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -552,7 +558,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 406 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -578,7 +584,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 582 "src/libre/dialect/pcre/parser.c" +#line 588 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -588,7 +594,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 547 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -636,7 +642,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 640 "src/libre/dialect/pcre/parser.c" +#line 646 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -646,7 +652,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 504 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -689,7 +695,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 693 "src/libre/dialect/pcre/parser.c" +#line 699 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -699,7 +705,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 445 "src/libre/parser.act" +#line 446 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -715,20 +721,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 719 "src/libre/dialect/pcre/parser.c" +#line 725 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 732 "src/libre/dialect/pcre/parser.c" +#line 738 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -740,12 +746,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 156 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 749 "src/libre/dialect/pcre/parser.c" +#line 755 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -776,13 +782,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 786 "src/libre/dialect/pcre/parser.c" +#line 792 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -836,13 +842,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 846 "src/libre/dialect/pcre/parser.c" +#line 852 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ } @@ -888,14 +894,14 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI292)); if ((ZInode) == NULL) { goto ZL1; } -#line 899 "src/libre/dialect/pcre/parser.c" +#line 905 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -908,12 +914,12 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 873 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI292); -#line 917 "src/libre/dialect/pcre/parser.c" +#line 923 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_169 (flags, lex_state, act_state, err); @@ -924,17 +930,17 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 792 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI293)); mark(&act_state->rangeend, &(ZIend)); -#line 933 "src/libre/dialect/pcre/parser.c" +#line 939 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -963,7 +969,7 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 967 "src/libre/dialect/pcre/parser.c" +#line 973 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -994,7 +1000,7 @@ p_169(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI170 = '-'; ZI171 = lex_state->lx.start; @@ -1004,7 +1010,7 @@ p_169(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI171; (void) ZI172; -#line 1008 "src/libre/dialect/pcre/parser.c" +#line 1014 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1018,14 +1024,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 732 "src/libre/parser.act" +#line 738 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 1029 "src/libre/dialect/pcre/parser.c" +#line 1035 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1057,7 +1063,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1070,7 +1076,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 1074 "src/libre/dialect/pcre/parser.c" +#line 1080 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1083,7 +1089,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CONTROL */ { -#line 461 "src/libre/parser.act" +#line 462 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1091,16 +1097,21 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(lex_state->buf.a[3] == '\0'); ZIc = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)ZIc; + unsigned char cc = (unsigned char) ZIc; if (cc > 126 || cc < 32) { goto ZL1; } @@ -1120,7 +1131,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI124; (void) ZI125; -#line 1124 "src/libre/dialect/pcre/parser.c" +#line 1135 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1133,7 +1144,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 406 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1159,7 +1170,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI114; (void) ZI115; -#line 1163 "src/libre/dialect/pcre/parser.c" +#line 1174 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1172,7 +1183,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 547 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1220,7 +1231,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1224 "src/libre/dialect/pcre/parser.c" +#line 1235 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1233,7 +1244,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: NOESC */ { -#line 431 "src/libre/parser.act" +#line 432 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1247,7 +1258,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI116; (void) ZI117; -#line 1251 "src/libre/dialect/pcre/parser.c" +#line 1262 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1260,7 +1271,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 504 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1303,7 +1314,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1307 "src/libre/dialect/pcre/parser.c" +#line 1318 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1316,7 +1327,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 445 "src/libre/parser.act" +#line 446 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1332,20 +1343,20 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI126; (void) ZI127; -#line 1336 "src/libre/dialect/pcre/parser.c" +#line 1347 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1349 "src/libre/dialect/pcre/parser.c" +#line 1360 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1357,14 +1368,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 112 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1368 "src/libre/dialect/pcre/parser.c" +#line 1379 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1390,7 +1401,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1403,7 +1414,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI308 = lex_state->buf.a[0]; -#line 1407 "src/libre/dialect/pcre/parser.c" +#line 1418 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1422,7 +1433,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CONTROL */ { -#line 461 "src/libre/parser.act" +#line 462 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1430,16 +1441,21 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(lex_state->buf.a[3] == '\0'); ZI312 = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)ZI312; + unsigned char cc = (unsigned char) ZI312; if (cc > 126 || cc < 32) { goto ZL1; } @@ -1459,20 +1475,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI313; (void) ZI314; -#line 1463 "src/libre/dialect/pcre/parser.c" +#line 1479 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1476 "src/libre/dialect/pcre/parser.c" +#line 1492 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_319 (flags, lex_state, act_state, err, &ZI312, &ZI313, &ZInode); @@ -1490,7 +1506,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 406 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1516,7 +1532,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI297; (void) ZI298; -#line 1520 "src/libre/dialect/pcre/parser.c" +#line 1536 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1535,7 +1551,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 547 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1583,7 +1599,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI304 = (char) (unsigned char) u; -#line 1587 "src/libre/dialect/pcre/parser.c" +#line 1603 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1602,7 +1618,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 667 "src/libre/parser.act" ZI292 = DIALECT_CLASS(lex_state->buf.a); if (ZI292 == NULL) { @@ -1616,7 +1632,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI293; (void) ZI294; -#line 1620 "src/libre/dialect/pcre/parser.c" +#line 1636 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1635,7 +1651,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NOESC */ { -#line 431 "src/libre/parser.act" +#line 432 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1649,20 +1665,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI135; (void) ZI136; -#line 1653 "src/libre/dialect/pcre/parser.c" +#line 1669 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1666 "src/libre/dialect/pcre/parser.c" +#line 1682 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1675,7 +1691,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 504 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1718,7 +1734,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI300 = (char) (unsigned char) u; -#line 1722 "src/libre/dialect/pcre/parser.c" +#line 1738 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1737,7 +1753,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 445 "src/libre/parser.act" +#line 446 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1753,20 +1769,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI317; (void) ZI318; -#line 1757 "src/libre/dialect/pcre/parser.c" +#line 1773 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1770 "src/libre/dialect/pcre/parser.c" +#line 1786 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_319 (flags, lex_state, act_state, err, &ZI316, &ZI317, &ZInode); @@ -1805,14 +1821,14 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 697 "src/libre/parser.act" +#line 703 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; } goto ZL1; -#line 1816 "src/libre/dialect/pcre/parser.c" +#line 1832 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1839,7 +1855,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 667 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1853,7 +1869,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1857 "src/libre/dialect/pcre/parser.c" +#line 1873 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1863,12 +1879,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 873 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1872 "src/libre/dialect/pcre/parser.c" +#line 1888 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1904,7 +1920,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 337 "src/libre/parser.act" +#line 338 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI182 = lex_state->lx.end; @@ -1912,20 +1928,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI182; -#line 1916 "src/libre/dialect/pcre/parser.c" +#line 1932 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1929 "src/libre/dialect/pcre/parser.c" +#line 1945 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1945,7 +1961,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 353 "src/libre/parser.act" +#line 354 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI201 = lex_state->lx.end; @@ -1953,30 +1969,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI201; -#line 1957 "src/libre/dialect/pcre/parser.c" +#line 1973 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1970 "src/libre/dialect/pcre/parser.c" +#line 1986 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 914 "src/libre/parser.act" +#line 915 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1980 "src/libre/dialect/pcre/parser.c" +#line 1996 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1986,13 +2002,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1996 "src/libre/dialect/pcre/parser.c" +#line 2012 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2008,7 +2024,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 345 "src/libre/parser.act" +#line 346 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI193 = lex_state->lx.end; @@ -2016,26 +2032,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI193; -#line 2020 "src/libre/dialect/pcre/parser.c" +#line 2036 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2033 "src/libre/dialect/pcre/parser.c" +#line 2049 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 996 "src/libre/parser.act" struct ast_expr *any; @@ -2073,7 +2089,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2077 "src/libre/dialect/pcre/parser.c" +#line 2093 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ p_195 (flags, lex_state, act_state, err, &ZItmp); @@ -2092,7 +2108,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 361 "src/libre/parser.act" +#line 362 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI208 = lex_state->lx.end; @@ -2100,26 +2116,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI208; -#line 2104 "src/libre/dialect/pcre/parser.c" +#line 2120 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2117 "src/libre/dialect/pcre/parser.c" +#line 2133 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 996 "src/libre/parser.act" struct ast_expr *any; @@ -2157,16 +2173,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2161 "src/libre/dialect/pcre/parser.c" +#line 2177 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 914 "src/libre/parser.act" +#line 915 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 2170 "src/libre/dialect/pcre/parser.c" +#line 2186 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -2176,13 +2192,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2186 "src/libre/dialect/pcre/parser.c" +#line 2202 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2207,7 +2223,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI215 = ']'; ZI216 = lex_state->lx.start; @@ -2217,18 +2233,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI216; (void) ZIend; -#line 2221 "src/libre/dialect/pcre/parser.c" +#line 2237 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 787 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2232 "src/libre/dialect/pcre/parser.c" +#line 2248 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2241,7 +2257,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 379 "src/libre/parser.act" +#line 380 "src/libre/parser.act" ZIcrange = '-'; ZI218 = lex_state->lx.start; @@ -2251,41 +2267,41 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI218; (void) ZIend; -#line 2255 "src/libre/dialect/pcre/parser.c" +#line 2271 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 2268 "src/libre/dialect/pcre/parser.c" +#line 2284 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 2279 "src/libre/dialect/pcre/parser.c" +#line 2295 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 787 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2289 "src/libre/dialect/pcre/parser.c" +#line 2305 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2298,14 +2314,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 745 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 2309 "src/libre/dialect/pcre/parser.c" +#line 2325 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ ZIend = ZIstart; @@ -2315,22 +2331,17 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 214 */ /* BEGINNING OF ACTION: mark-expr */ { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 802 "src/libre/parser.act" - AST_POS_OF_LX_POS(ast_start, (ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - mark(&act_state->groupstart, &(ZIstart)); - mark(&act_state->groupend, &(ZIend)); + (void) (ZIstart); + (void) (ZIend); /* TODO: reinstate this, applies to an expr node in general (ZItmp)->u.class.start = ast_start; (ZItmp)->u.class.end = ast_end; */ -#line 2334 "src/libre/dialect/pcre/parser.c" +#line 2345 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -2362,7 +2373,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZIc = '-'; ZI164 = lex_state->lx.start; @@ -2372,18 +2383,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI164; (void) ZIend; -#line 2376 "src/libre/dialect/pcre/parser.c" +#line 2387 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2387 "src/libre/dialect/pcre/parser.c" +#line 2398 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2425,14 +2436,14 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI316)); if ((ZInode) == NULL) { goto ZL1; } -#line 2436 "src/libre/dialect/pcre/parser.c" +#line 2447 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2445,12 +2456,12 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZI316); + (ZIlower).u.literal.c = (unsigned char) (*ZI316); -#line 2454 "src/libre/dialect/pcre/parser.c" +#line 2465 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_169 (flags, lex_state, act_state, err); @@ -2461,17 +2472,17 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 792 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI317)); mark(&act_state->rangeend, &(ZIend)); -#line 2470 "src/libre/dialect/pcre/parser.c" +#line 2481 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -2500,7 +2511,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2504 "src/libre/dialect/pcre/parser.c" +#line 2515 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2546,16 +2557,16 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 850 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2554 "src/libre/dialect/pcre/parser.c" +#line 2565 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2569,7 +2580,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2573 "src/libre/dialect/pcre/parser.c" +#line 2584 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2600,14 +2611,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2611 "src/libre/dialect/pcre/parser.c" +#line 2622 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2621,26 +2632,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 731 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2632 "src/libre/dialect/pcre/parser.c" +#line 2643 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2644 "src/libre/dialect/pcre/parser.c" +#line 2655 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2665,7 +2676,7 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI258 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2673,23 +2684,23 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI258; (void) ZIend; -#line 2677 "src/libre/dialect/pcre/parser.c" +#line 2688 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 2688 "src/libre/dialect/pcre/parser.c" +#line 2699 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2704,7 +2715,7 @@ p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 2708 "src/libre/dialect/pcre/parser.c" +#line 2719 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2745,7 +2756,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -2755,7 +2766,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI196; -#line 2759 "src/libre/dialect/pcre/parser.c" +#line 2770 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2766,14 +2777,14 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2777 "src/libre/dialect/pcre/parser.c" +#line 2788 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2789,17 +2800,17 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(ZIc); + (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 2798 "src/libre/dialect/pcre/parser.c" +#line 2809 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI198 = '-'; ZI199 = lex_state->lx.start; @@ -2809,7 +2820,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI199; (void) ZI200; -#line 2813 "src/libre/dialect/pcre/parser.c" +#line 2824 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2820,7 +2831,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -2849,7 +2860,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2853 "src/libre/dialect/pcre/parser.c" +#line 2864 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2859,13 +2870,13 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 197 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2869 "src/libre/dialect/pcre/parser.c" +#line 2880 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2895,7 +2906,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI263 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2903,32 +2914,32 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI263; (void) ZIend; -#line 2907 "src/libre/dialect/pcre/parser.c" +#line 2918 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 2918 "src/libre/dialect/pcre/parser.c" +#line 2929 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ { -#line 833 "src/libre/parser.act" +#line 834 "src/libre/parser.act" (ZIn) = AST_COUNT_UNBOUNDED; -#line 2927 "src/libre/dialect/pcre/parser.c" +#line 2938 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2943,7 +2954,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2947 "src/libre/dialect/pcre/parser.c" +#line 2958 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2956,7 +2967,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 647 "src/libre/parser.act" unsigned long u; char *e; @@ -2976,7 +2987,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 2980 "src/libre/dialect/pcre/parser.c" +#line 2991 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -2984,7 +2995,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2992,7 +3003,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI261; (void) ZIend; -#line 2996 "src/libre/dialect/pcre/parser.c" +#line 3007 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -3002,17 +3013,17 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI320)); mark(&act_state->countend, &(ZIend)); -#line 3011 "src/libre/dialect/pcre/parser.c" +#line 3022 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -3027,7 +3038,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 3031 "src/libre/dialect/pcre/parser.c" +#line 3042 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -3068,20 +3079,20 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 825 "src/libre/parser.act" +#line 826 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; -#line 3076 "src/libre/dialect/pcre/parser.c" +#line 3087 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 825 "src/libre/parser.act" +#line 826 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; -#line 3085 "src/libre/dialect/pcre/parser.c" +#line 3096 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF INLINE: 233 */ @@ -3134,7 +3145,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 955 "src/libre/parser.act" +#line 956 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3143,19 +3154,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3147 "src/libre/dialect/pcre/parser.c" +#line 3158 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL5; } -#line 3159 "src/libre/dialect/pcre/parser.c" +#line 3170 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3168,16 +3179,16 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 947 "src/libre/parser.act" +#line 948 "src/libre/parser.act" (ZIflags) = *flags; -#line 3176 "src/libre/dialect/pcre/parser.c" +#line 3187 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 955 "src/libre/parser.act" +#line 956 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3186,7 +3197,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3190 "src/libre/dialect/pcre/parser.c" +#line 3201 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3196,11 +3207,11 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 951 "src/libre/parser.act" +#line 952 "src/libre/parser.act" *flags = (ZIflags); -#line 3204 "src/libre/dialect/pcre/parser.c" +#line 3215 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3221,26 +3232,26 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 760 "src/libre/parser.act" +#line 766 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; } goto ZL1; -#line 3232 "src/libre/dialect/pcre/parser.c" +#line 3243 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 3244 "src/libre/dialect/pcre/parser.c" +#line 3255 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3274,7 +3285,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3288,7 +3299,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3292 "src/libre/dialect/pcre/parser.c" +#line 3303 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ /* BEGINNING OF INLINE: 266 */ @@ -3299,14 +3310,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3310 "src/libre/dialect/pcre/parser.c" +#line 3321 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3316,14 +3327,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 780 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3327 "src/libre/dialect/pcre/parser.c" +#line 3338 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3408,7 +3419,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 667 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3422,7 +3433,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3426 "src/libre/dialect/pcre/parser.c" +#line 3437 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3432,14 +3443,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3443 "src/libre/dialect/pcre/parser.c" +#line 3454 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3463,14 +3474,14 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3474 "src/libre/dialect/pcre/parser.c" +#line 3485 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3487,17 +3498,17 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3496 "src/libre/dialect/pcre/parser.c" +#line 3507 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI211 = '-'; ZI212 = lex_state->lx.start; @@ -3507,7 +3518,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI212; (void) ZI213; -#line 3511 "src/libre/dialect/pcre/parser.c" +#line 3522 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3518,17 +3529,17 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3527 "src/libre/dialect/pcre/parser.c" +#line 3538 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -3557,7 +3568,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3561 "src/libre/dialect/pcre/parser.c" +#line 3572 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3590,13 +3601,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3600 "src/libre/dialect/pcre/parser.c" +#line 3611 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 280 */ @@ -3621,14 +3632,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 731 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 3632 "src/libre/dialect/pcre/parser.c" +#line 3643 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3653,7 +3664,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 389 "src/libre/parser.act" +#line 390 "src/libre/parser.act" ZI320 = lex_state->lx.start; ZI321 = lex_state->lx.end; @@ -3661,7 +3672,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI320; (void) ZI321; -#line 3665 "src/libre/dialect/pcre/parser.c" +#line 3676 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3669,7 +3680,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 647 "src/libre/parser.act" unsigned long u; char *e; @@ -3689,7 +3700,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3693 "src/libre/dialect/pcre/parser.c" +#line 3704 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3709,11 +3720,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 845 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 3717 "src/libre/dialect/pcre/parser.c" +#line 3728 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3723,11 +3734,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 841 "src/libre/parser.act" +#line 842 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3731 "src/libre/dialect/pcre/parser.c" +#line 3742 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3737,11 +3748,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 838 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3745 "src/libre/dialect/pcre/parser.c" +#line 3756 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3756,23 +3767,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 711 "src/libre/parser.act" +#line 717 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 3767 "src/libre/dialect/pcre/parser.c" +#line 3778 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 850 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 3776 "src/libre/dialect/pcre/parser.c" +#line 3787 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3798,11 +3809,11 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3806 "src/libre/dialect/pcre/parser.c" +#line 3817 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3812,14 +3823,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3823 "src/libre/dialect/pcre/parser.c" +#line 3834 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 282 */ @@ -3838,14 +3849,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 773 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 3849 "src/libre/dialect/pcre/parser.c" +#line 3860 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -3874,24 +3885,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3883 "src/libre/dialect/pcre/parser.c" +#line 3894 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3895 "src/libre/dialect/pcre/parser.c" +#line 3906 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3901,14 +3912,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 971 "src/libre/parser.act" +#line 972 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 3912 "src/libre/dialect/pcre/parser.c" +#line 3923 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3918,7 +3929,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 978 "src/libre/parser.act" +#line 979 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3928,7 +3939,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3932 "src/libre/dialect/pcre/parser.c" +#line 3943 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3946,134 +3957,134 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 816 "src/libre/parser.act" +#line 817 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3955 "src/libre/dialect/pcre/parser.c" +#line 3966 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 3967 "src/libre/dialect/pcre/parser.c" +#line 3978 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { goto ZL1; } -#line 3979 "src/libre/dialect/pcre/parser.c" +#line 3990 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 918 "src/libre/parser.act" +#line 919 "src/libre/parser.act" (ZIcr) = '\r'; -#line 3988 "src/libre/dialect/pcre/parser.c" +#line 3999 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { goto ZL1; } -#line 4000 "src/libre/dialect/pcre/parser.c" +#line 4011 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 922 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZInl) = '\n'; -#line 4009 "src/libre/dialect/pcre/parser.c" +#line 4020 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { goto ZL1; } -#line 4021 "src/libre/dialect/pcre/parser.c" +#line 4032 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4032 "src/libre/dialect/pcre/parser.c" +#line 4043 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4043 "src/libre/dialect/pcre/parser.c" +#line 4054 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 4055 "src/libre/dialect/pcre/parser.c" +#line 4066 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4066 "src/libre/dialect/pcre/parser.c" +#line 4077 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4077 "src/libre/dialect/pcre/parser.c" +#line 4088 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4087,20 +4098,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 947 "src/libre/parser.act" +#line 948 "src/libre/parser.act" (ZIflags) = *flags; -#line 4095 "src/libre/dialect/pcre/parser.c" +#line 4106 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 4104 "src/libre/dialect/pcre/parser.c" +#line 4115 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4110,23 +4121,23 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 951 "src/libre/parser.act" +#line 952 "src/libre/parser.act" *flags = (ZIflags); -#line 4118 "src/libre/dialect/pcre/parser.c" +#line 4129 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 4130 "src/libre/dialect/pcre/parser.c" +#line 4141 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4143,14 +4154,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 964 "src/libre/parser.act" +#line 965 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 4154 "src/libre/dialect/pcre/parser.c" +#line 4165 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4202,26 +4213,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 724 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 4213 "src/libre/dialect/pcre/parser.c" +#line 4224 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 4225 "src/libre/dialect/pcre/parser.c" +#line 4236 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4247,14 +4258,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4258 "src/libre/dialect/pcre/parser.c" +#line 4269 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4268,14 +4279,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4279 "src/libre/dialect/pcre/parser.c" +#line 4290 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4311,45 +4322,40 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4322 "src/libre/dialect/pcre/parser.c" +#line 4333 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4333 "src/libre/dialect/pcre/parser.c" +#line 4344 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-expr */ { -#line 796 "src/libre/parser.act" +#line 802 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - mark(&act_state->groupstart, &(ZIstart)); - mark(&act_state->groupend, &(ZIend)); + (void) (ZIstart); + (void) (ZIend); /* TODO: reinstate this, applies to an expr node in general (ZInode)->u.class.start = ast_start; (ZInode)->u.class.end = ast_end; */ -#line 4353 "src/libre/dialect/pcre/parser.c" +#line 4359 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-expr */ } @@ -4363,7 +4369,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1081 "src/libre/parser.act" +#line 1082 "src/libre/parser.act" static int @@ -4521,6 +4527,6 @@ ZL0:; return NULL; } -#line 4525 "src/libre/dialect/pcre/parser.c" +#line 4531 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 8346c0641..733754335 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 301 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1238 "src/libre/parser.act" +#line 1239 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index c96e30107..4d30cdf44 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -172,6 +172,7 @@ } restart: + tok = act_state->lex_tok; if (lex_state->extended_mode_comment) { switch (tok) { @@ -288,7 +289,7 @@ return s; } -#line 292 "src/libre/dialect/sql/parser.c" +#line 293 "src/libre/dialect/sql/parser.c" #ifndef ERROR_TERMINAL @@ -330,13 +331,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: INVERT */ { -#line 321 "src/libre/parser.act" +#line 322 "src/libre/parser.act" ZI204 = '^'; (void) ZI204; -#line 340 "src/libre/dialect/sql/parser.c" +#line 341 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; @@ -356,7 +357,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZIc = '-'; ZI115 = lex_state->lx.start; @@ -366,31 +367,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ (void) ZI115; (void) ZI116; -#line 370 "src/libre/dialect/sql/parser.c" +#line 371 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 383 "src/libre/dialect/sql/parser.c" +#line 384 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 394 "src/libre/dialect/sql/parser.c" +#line 395 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -420,11 +421,11 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 428 "src/libre/dialect/sql/parser.c" +#line 429 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -434,14 +435,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 445 "src/libre/dialect/sql/parser.c" +#line 446 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 192 */ @@ -460,14 +461,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 773 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 471 "src/libre/dialect/sql/parser.c" +#line 472 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -503,13 +504,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; } -#line 513 "src/libre/dialect/sql/parser.c" +#line 514 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -518,14 +519,14 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 704 "src/libre/parser.act" +#line 710 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; } goto ZL1; -#line 529 "src/libre/dialect/sql/parser.c" +#line 530 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } @@ -579,13 +580,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1069 "src/libre/parser.act" +#line 1070 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 589 "src/libre/dialect/sql/parser.c" +#line 590 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 184 */ @@ -625,7 +626,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -638,7 +639,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI206 = lex_state->buf.a[0]; -#line 642 "src/libre/dialect/sql/parser.c" +#line 643 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -689,7 +690,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 337 "src/libre/parser.act" +#line 338 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -697,7 +698,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 701 "src/libre/dialect/sql/parser.c" +#line 702 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -707,14 +708,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { goto ZL1; } -#line 718 "src/libre/dialect/sql/parser.c" +#line 719 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; @@ -732,7 +733,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI155 = ']'; ZIclose__start = lex_state->lx.start; @@ -742,38 +743,33 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 746 "src/libre/dialect/sql/parser.c" +#line 747 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 787 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIopen__end)); -#line 757 "src/libre/dialect/sql/parser.c" +#line 758 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ /* BEGINNING OF ACTION: mark-expr */ { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIopen__start)); - AST_POS_OF_LX_POS(ast_end, (ZIclose__end)); +#line 802 "src/libre/parser.act" - mark(&act_state->groupstart, &(ZIopen__start)); - mark(&act_state->groupend, &(ZIclose__end)); + (void) (ZIopen__start); + (void) (ZIclose__end); /* TODO: reinstate this, applies to an expr node in general (ZItmp)->u.class.start = ast_start; (ZItmp)->u.class.end = ast_end; */ -#line 777 "src/libre/dialect/sql/parser.c" +#line 773 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ ZInode = ZIclass; @@ -787,26 +783,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: INVERT */ { -#line 321 "src/libre/parser.act" +#line 322 "src/libre/parser.act" ZI159 = '^'; (void) ZI159; -#line 797 "src/libre/dialect/sql/parser.c" +#line 793 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { goto ZL3; } -#line 810 "src/libre/dialect/sql/parser.c" +#line 806 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; @@ -828,7 +824,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI164 = ']'; ZIclose__start = lex_state->lx.start; @@ -838,7 +834,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 842 "src/libre/dialect/sql/parser.c" +#line 838 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -848,52 +844,42 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 787 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIclose__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 857 "src/libre/dialect/sql/parser.c" +#line 853 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ /* BEGINNING OF ACTION: mark-expr */ { -#line 796 "src/libre/parser.act" +#line 802 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIopen__start)); - AST_POS_OF_LX_POS(ast_end, (ZIclose__end)); - - mark(&act_state->groupstart, &(ZIopen__start)); - mark(&act_state->groupend, &(ZIclose__end)); + (void) (ZIopen__start); + (void) (ZIclose__end); /* TODO: reinstate this, applies to an expr node in general (ZItmp)->u.class.start = ast_start; (ZItmp)->u.class.end = ast_end; */ -#line 877 "src/libre/dialect/sql/parser.c" +#line 868 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ /* BEGINNING OF ACTION: mark-expr */ { -#line 796 "src/libre/parser.act" +#line 802 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIopen__start)); - AST_POS_OF_LX_POS(ast_end, (ZIclose__end)); - - mark(&act_state->groupstart, &(ZIopen__start)); - mark(&act_state->groupend, &(ZIclose__end)); + (void) (ZIopen__start); + (void) (ZIclose__end); /* TODO: reinstate this, applies to an expr node in general (ZImask__tmp)->u.class.start = ast_start; (ZImask__tmp)->u.class.end = ast_end; */ -#line 897 "src/libre/dialect/sql/parser.c" +#line 883 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-expr */ } @@ -902,14 +888,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 745 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL3; -#line 913 "src/libre/dialect/sql/parser.c" +#line 899 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -918,14 +904,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 163 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 988 "src/libre/parser.act" +#line 989 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { goto ZL3; } -#line 929 "src/libre/dialect/sql/parser.c" +#line 915 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -941,26 +927,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 745 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 952 "src/libre/dialect/sql/parser.c" +#line 938 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 964 "src/libre/dialect/sql/parser.c" +#line 950 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -996,7 +982,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1010,7 +996,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 1014 "src/libre/dialect/sql/parser.c" +#line 1000 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1033,14 +1019,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 896 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1044 "src/libre/dialect/sql/parser.c" +#line 1030 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -1054,26 +1040,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 731 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 1065 "src/libre/dialect/sql/parser.c" +#line 1051 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 1077 "src/libre/dialect/sql/parser.c" +#line 1063 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1098,7 +1084,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZIc = '-'; ZI118 = lex_state->lx.start; @@ -1108,36 +1094,36 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI118; (void) ZI119; -#line 1112 "src/libre/dialect/sql/parser.c" +#line 1098 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1125 "src/libre/dialect/sql/parser.c" +#line 1111 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 1136 "src/libre/dialect/sql/parser.c" +#line 1122 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 996 "src/libre/parser.act" struct ast_expr *any; @@ -1175,7 +1161,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1179 "src/libre/dialect/sql/parser.c" +#line 1165 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1184,7 +1170,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 996 "src/libre/parser.act" struct ast_expr *any; @@ -1222,7 +1208,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1226 "src/libre/dialect/sql/parser.c" +#line 1212 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1246,14 +1232,14 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI206)); if ((ZInode) == NULL) { goto ZL1; } -#line 1257 "src/libre/dialect/sql/parser.c" +#line 1243 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1271,17 +1257,17 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; - (ZIa).u.literal.c = (unsigned char)(*ZI206); + (ZIa).u.literal.c = (unsigned char) (*ZI206); -#line 1280 "src/libre/dialect/sql/parser.c" +#line 1266 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 328 "src/libre/parser.act" ZI137 = '-'; ZI138 = lex_state->lx.start; @@ -1291,7 +1277,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (void) ZI138; (void) ZI139; -#line 1295 "src/libre/dialect/sql/parser.c" +#line 1281 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1299,7 +1285,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1312,7 +1298,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ZIcz = lex_state->buf.a[0]; -#line 1316 "src/libre/dialect/sql/parser.c" +#line 1302 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1322,27 +1308,27 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 868 "src/libre/parser.act" +#line 869 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; - (ZIz).u.literal.c = (unsigned char)(ZIcz); + (ZIz).u.literal.c = (unsigned char) (ZIcz); -#line 1331 "src/libre/dialect/sql/parser.c" +#line 1317 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 792 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI207)); mark(&act_state->rangeend, &(ZIend)); -#line 1341 "src/libre/dialect/sql/parser.c" +#line 1327 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" unsigned char lower, upper; @@ -1371,7 +1357,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1375 "src/libre/dialect/sql/parser.c" +#line 1361 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1400,7 +1386,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI177 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1408,23 +1394,23 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI177; (void) ZIend; -#line 1412 "src/libre/dialect/sql/parser.c" +#line 1398 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1423 "src/libre/dialect/sql/parser.c" +#line 1409 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1439,7 +1425,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1443 "src/libre/dialect/sql/parser.c" +#line 1429 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1455,7 +1441,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 647 "src/libre/parser.act" unsigned long u; char *e; @@ -1475,7 +1461,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1479 "src/libre/dialect/sql/parser.c" +#line 1465 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1487,7 +1473,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 398 "src/libre/parser.act" ZI180 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1495,7 +1481,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI180; (void) ZIend; -#line 1499 "src/libre/dialect/sql/parser.c" +#line 1485 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1505,17 +1491,17 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 797 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI210)); mark(&act_state->countend, &(ZIend)); -#line 1514 "src/libre/dialect/sql/parser.c" +#line 1500 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1530,7 +1516,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1534 "src/libre/dialect/sql/parser.c" +#line 1520 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1565,13 +1551,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1075 "src/libre/parser.act" +#line 1076 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1575 "src/libre/dialect/sql/parser.c" +#line 1561 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 190 */ @@ -1596,14 +1582,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 731 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 1607 "src/libre/dialect/sql/parser.c" +#line 1593 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1628,7 +1614,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 389 "src/libre/parser.act" +#line 390 "src/libre/parser.act" ZI210 = lex_state->lx.start; ZI211 = lex_state->lx.end; @@ -1636,7 +1622,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI210; (void) ZI211; -#line 1640 "src/libre/dialect/sql/parser.c" +#line 1626 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1644,7 +1630,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 647 "src/libre/parser.act" unsigned long u; char *e; @@ -1664,7 +1650,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1668 "src/libre/dialect/sql/parser.c" +#line 1654 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1684,11 +1670,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 845 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 1692 "src/libre/dialect/sql/parser.c" +#line 1678 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1698,11 +1684,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 841 "src/libre/parser.act" +#line 842 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1706 "src/libre/dialect/sql/parser.c" +#line 1692 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1712,11 +1698,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 838 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1720 "src/libre/dialect/sql/parser.c" +#line 1706 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1725,11 +1711,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 850 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1733 "src/libre/dialect/sql/parser.c" +#line 1719 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1742,23 +1728,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 711 "src/libre/parser.act" +#line 717 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 1753 "src/libre/dialect/sql/parser.c" +#line 1739 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 850 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1762 "src/libre/dialect/sql/parser.c" +#line 1748 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1783,24 +1769,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1792 "src/libre/dialect/sql/parser.c" +#line 1778 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1804 "src/libre/dialect/sql/parser.c" +#line 1790 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1813,7 +1799,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 595 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1826,20 +1812,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1830 "src/libre/dialect/sql/parser.c" +#line 1816 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 903 "src/libre/parser.act" +#line 904 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1843 "src/libre/dialect/sql/parser.c" +#line 1829 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1853,38 +1839,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 812 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1862 "src/libre/dialect/sql/parser.c" +#line 1848 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1874 "src/libre/dialect/sql/parser.c" +#line 1860 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 838 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1883 "src/libre/dialect/sql/parser.c" +#line 1869 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 926 "src/libre/parser.act" +#line 927 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1898,7 +1884,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1902 "src/libre/dialect/sql/parser.c" +#line 1888 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1911,11 +1897,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 910 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 1919 "src/libre/dialect/sql/parser.c" +#line 1905 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1925,14 +1911,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 940 "src/libre/parser.act" +#line 941 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 1936 "src/libre/dialect/sql/parser.c" +#line 1922 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1963,26 +1949,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 724 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 1974 "src/libre/dialect/sql/parser.c" +#line 1960 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 1986 "src/libre/dialect/sql/parser.c" +#line 1972 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2011,7 +1997,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 667 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -2025,7 +2011,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI129; (void) ZI130; -#line 2029 "src/libre/dialect/sql/parser.c" +#line 2015 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2035,14 +2021,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1062 "src/libre/parser.act" +#line 1063 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2046 "src/libre/dialect/sql/parser.c" +#line 2032 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2065,14 +2051,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2076 "src/libre/dialect/sql/parser.c" +#line 2062 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2086,14 +2072,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 882 "src/libre/parser.act" +#line 883 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2097 "src/libre/dialect/sql/parser.c" +#line 2083 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2111,7 +2097,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1081 "src/libre/parser.act" +#line 1082 "src/libre/parser.act" static int @@ -2269,6 +2255,6 @@ ZL0:; return NULL; } -#line 2273 "src/libre/dialect/sql/parser.c" +#line 2259 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 5cc78ad47..1ca760dd3 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 301 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1238 "src/libre/parser.act" +#line 1239 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index 195a61dbf..c22197dcd 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -800,13 +800,8 @@ @}; : (node :ast_expr, start :pos, end :pos) -> () = @{ - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, @start); - AST_POS_OF_LX_POS(ast_end, @end); - - mark(&act_state->groupstart, &@start); - mark(&act_state->groupend, &@end); + (void) @start; + (void) @end; /* TODO: reinstate this, applies to an expr node in general @node->u.class.start = ast_start; From f0da88d10280dc72730d41224b3d918e1e656e9c Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 29 May 2024 16:25:25 +0100 Subject: [PATCH 109/373] Refactoring: No need for . This also removes struct ast_pos, with no remaining uses. --- src/libre/ast.h | 12 - src/libre/dialect/glob/parser.c | 98 +- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/glob/parser.sid | 1 - src/libre/dialect/like/parser.c | 98 +- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/like/parser.sid | 1 - src/libre/dialect/literal/parser.c | 74 +- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/literal/parser.sid | 1 - src/libre/dialect/native/parser.c | 1041 +++++++++---------- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/native/parser.sid | 4 - src/libre/dialect/pcre/parser.c | 1422 +++++++++++++------------- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/pcre/parser.sid | 6 - src/libre/dialect/sql/parser.c | 517 +++++----- src/libre/dialect/sql/parser.h | 4 +- src/libre/dialect/sql/parser.sid | 10 +- src/libre/parser.act | 18 - 20 files changed, 1570 insertions(+), 1757 deletions(-) diff --git a/src/libre/ast.h b/src/libre/ast.h index eca93b037..d443db9af 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -7,18 +7,6 @@ #ifndef RE_AST_H #define RE_AST_H -/* - * This is a duplicate of struct lx_pos, but since we're linking to - * code with several distinct lexers, there isn't a clear lexer.h - * to include here. The parser sees both definitions, and will - * build a `struct ast_pos` in the appropriate places. - */ -struct ast_pos { - unsigned byte; - unsigned line; - unsigned col; -}; - enum ast_expr_type { /* Reserve one value (0) indicating a freed expression. This value is * intentionally unnamed: code that switches on n->type should be able diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 89af72030..d9d30b552 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -289,7 +281,7 @@ return s; } -#line 293 "src/libre/dialect/glob/parser.c" +#line 285 "src/libre/dialect/glob/parser.c" #ifndef ERROR_TERMINAL @@ -324,16 +316,16 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 334 "src/libre/dialect/glob/parser.c" +#line 326 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 116 */ + /* BEGINNING OF INLINE: 115 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): @@ -347,7 +339,7 @@ ZL2_list_Hof_Hatoms:; break; } } - /* END OF INLINE: 116 */ + /* END OF INLINE: 115 */ } return; ZL1:; @@ -368,24 +360,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 377 "src/libre/dialect/glob/parser.c" +#line 369 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 389 "src/libre/dialect/glob/parser.c" +#line 381 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -393,38 +385,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): { t_char ZIc; + t_pos ZI111; t_pos ZI112; - t_pos ZI113; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI112 = lex_state->lx.start; - ZI113 = lex_state->lx.end; + ZI111 = lex_state->lx.start; + ZI112 = lex_state->lx.end; + (void) ZI111; (void) ZI112; - (void) ZI113; ZIc = lex_state->buf.a[0]; -#line 415 "src/libre/dialect/glob/parser.c" +#line 407 "src/libre/dialect/glob/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 428 "src/libre/dialect/glob/parser.c" +#line 420 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -438,38 +430,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 447 "src/libre/dialect/glob/parser.c" +#line 439 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 459 "src/libre/dialect/glob/parser.c" +#line 451 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 838 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 468 "src/libre/dialect/glob/parser.c" +#line 460 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -483,7 +475,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 487 "src/libre/dialect/glob/parser.c" +#line 479 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -498,26 +490,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 724 "src/libre/parser.act" +#line 716 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 509 "src/libre/dialect/glob/parser.c" +#line 501 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 521 "src/libre/dialect/glob/parser.c" +#line 513 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -543,28 +535,28 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 551 "src/libre/dialect/glob/parser.c" +#line 543 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: make-group-id */ - /* BEGINNING OF INLINE: 120 */ + /* BEGINNING OF INLINE: 119 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 568 "src/libre/dialect/glob/parser.c" +#line 560 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -578,34 +570,34 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 589 "src/libre/dialect/glob/parser.c" +#line 581 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } break; } } - /* END OF INLINE: 120 */ + /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 606 "src/libre/dialect/glob/parser.c" +#line 598 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-group */ - /* BEGINNING OF INLINE: 121 */ + /* BEGINNING OF INLINE: 120 */ { { switch (CURRENT_TERMINAL) { @@ -621,20 +613,20 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 773 "src/libre/parser.act" +#line 765 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 632 "src/libre/dialect/glob/parser.c" +#line 624 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 121 */ + /* END OF INLINE: 120 */ } goto ZL0; ZL1:; @@ -646,7 +638,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1082 "src/libre/parser.act" +#line 1064 "src/libre/parser.act" static int @@ -804,6 +796,6 @@ ZL0:; return NULL; } -#line 808 "src/libre/dialect/glob/parser.c" +#line 800 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 8e69b6c57..074b0a37c 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 301 "src/libre/parser.act" +#line 293 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1239 "src/libre/parser.act" +#line 1221 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/glob/parser.sid b/src/libre/dialect/glob/parser.sid index d60c415ee..735827f37 100644 --- a/src/libre/dialect/glob/parser.sid +++ b/src/libre/dialect/glob/parser.sid @@ -128,7 +128,6 @@ !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); - !: (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index a0b69bec6..c9f1f7769 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -289,7 +281,7 @@ return s; } -#line 293 "src/libre/dialect/like/parser.c" +#line 285 "src/libre/dialect/like/parser.c" #ifndef ERROR_TERMINAL @@ -324,16 +316,16 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 334 "src/libre/dialect/like/parser.c" +#line 326 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 116 */ + /* BEGINNING OF INLINE: 115 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): @@ -347,7 +339,7 @@ ZL2_list_Hof_Hatoms:; break; } } - /* END OF INLINE: 116 */ + /* END OF INLINE: 115 */ } return; ZL1:; @@ -368,24 +360,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 377 "src/libre/dialect/like/parser.c" +#line 369 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 389 "src/libre/dialect/like/parser.c" +#line 381 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -393,38 +385,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): { t_char ZIc; + t_pos ZI113; t_pos ZI114; - t_pos ZI115; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI114 = lex_state->lx.start; - ZI115 = lex_state->lx.end; + ZI113 = lex_state->lx.start; + ZI114 = lex_state->lx.end; + (void) ZI113; (void) ZI114; - (void) ZI115; ZIc = lex_state->buf.a[0]; -#line 415 "src/libre/dialect/like/parser.c" +#line 407 "src/libre/dialect/like/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 428 "src/libre/dialect/like/parser.c" +#line 420 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -438,38 +430,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 447 "src/libre/dialect/like/parser.c" +#line 439 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 459 "src/libre/dialect/like/parser.c" +#line 451 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 838 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 468 "src/libre/dialect/like/parser.c" +#line 460 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -483,7 +475,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 487 "src/libre/dialect/like/parser.c" +#line 479 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -498,26 +490,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 724 "src/libre/parser.act" +#line 716 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 509 "src/libre/dialect/like/parser.c" +#line 501 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 521 "src/libre/dialect/like/parser.c" +#line 513 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -543,28 +535,28 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 551 "src/libre/dialect/like/parser.c" +#line 543 "src/libre/dialect/like/parser.c" } /* END OF ACTION: make-group-id */ - /* BEGINNING OF INLINE: 120 */ + /* BEGINNING OF INLINE: 119 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 568 "src/libre/dialect/like/parser.c" +#line 560 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -578,34 +570,34 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 589 "src/libre/dialect/like/parser.c" +#line 581 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } break; } } - /* END OF INLINE: 120 */ + /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 606 "src/libre/dialect/like/parser.c" +#line 598 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-group */ - /* BEGINNING OF INLINE: 121 */ + /* BEGINNING OF INLINE: 120 */ { { switch (CURRENT_TERMINAL) { @@ -621,20 +613,20 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 773 "src/libre/parser.act" +#line 765 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 632 "src/libre/dialect/like/parser.c" +#line 624 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 121 */ + /* END OF INLINE: 120 */ } goto ZL0; ZL1:; @@ -646,7 +638,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1082 "src/libre/parser.act" +#line 1064 "src/libre/parser.act" static int @@ -804,6 +796,6 @@ ZL0:; return NULL; } -#line 808 "src/libre/dialect/like/parser.c" +#line 800 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index fc11ce83c..c7e00987d 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 301 "src/libre/parser.act" +#line 293 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1239 "src/libre/parser.act" +#line 1221 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/like/parser.sid b/src/libre/dialect/like/parser.sid index 56d14a23a..309fea3b8 100644 --- a/src/libre/dialect/like/parser.sid +++ b/src/libre/dialect/like/parser.sid @@ -128,7 +128,6 @@ !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); - !: (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 067fab0ef..118aa0393 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -289,7 +281,7 @@ return s; } -#line 293 "src/libre/dialect/literal/parser.c" +#line 285 "src/libre/dialect/literal/parser.c" #ifndef ERROR_TERMINAL @@ -324,16 +316,16 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 334 "src/libre/dialect/literal/parser.c" +#line 326 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 115 */ + /* BEGINNING OF INLINE: 114 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): @@ -347,7 +339,7 @@ ZL2_list_Hof_Hatoms:; break; } } - /* END OF INLINE: 115 */ + /* END OF INLINE: 114 */ } return; ZL1:; @@ -369,28 +361,28 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 377 "src/libre/dialect/literal/parser.c" +#line 369 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: make-group-id */ - /* BEGINNING OF INLINE: 118 */ + /* BEGINNING OF INLINE: 117 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 394 "src/libre/dialect/literal/parser.c" +#line 386 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -404,34 +396,34 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 415 "src/libre/dialect/literal/parser.c" +#line 407 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } break; } } - /* END OF INLINE: 118 */ + /* END OF INLINE: 117 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 432 "src/libre/dialect/literal/parser.c" +#line 424 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-group */ - /* BEGINNING OF INLINE: 120 */ + /* BEGINNING OF INLINE: 119 */ { { switch (CURRENT_TERMINAL) { @@ -447,20 +439,20 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 773 "src/libre/parser.act" +#line 765 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 458 "src/libre/dialect/literal/parser.c" +#line 450 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 120 */ + /* END OF INLINE: 119 */ } goto ZL0; ZL1:; @@ -480,27 +472,27 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, } { t_char ZIc; + t_pos ZI111; t_pos ZI112; - t_pos ZI113; switch (CURRENT_TERMINAL) { case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI112 = lex_state->lx.start; - ZI113 = lex_state->lx.end; + ZI111 = lex_state->lx.start; + ZI112 = lex_state->lx.end; + (void) ZI111; (void) ZI112; - (void) ZI113; ZIc = lex_state->buf.a[0]; -#line 504 "src/libre/dialect/literal/parser.c" +#line 496 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -510,14 +502,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 521 "src/libre/dialect/literal/parser.c" +#line 513 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -526,26 +518,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 724 "src/libre/parser.act" +#line 716 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 537 "src/libre/dialect/literal/parser.c" +#line 529 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 549 "src/libre/dialect/literal/parser.c" +#line 541 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -559,7 +551,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1082 "src/libre/parser.act" +#line 1064 "src/libre/parser.act" static int @@ -717,6 +709,6 @@ ZL0:; return NULL; } -#line 721 "src/libre/dialect/literal/parser.c" +#line 713 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index f9eecbaa5..2b6ef4b73 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 301 "src/libre/parser.act" +#line 293 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1239 "src/libre/parser.act" +#line 1221 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/literal/parser.sid b/src/libre/dialect/literal/parser.sid index 99f8f69d9..3e4d6c388 100644 --- a/src/libre/dialect/literal/parser.sid +++ b/src/libre/dialect/literal/parser.sid @@ -128,7 +128,6 @@ !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); - !: (:ast_expr, :pos, :pos) -> (); !: () -> (:ast_class_id); !: () -> (:ast_class_id); diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 4efd632db..01f4a4a8a 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -289,7 +281,7 @@ return s; } -#line 293 "src/libre/dialect/native/parser.c" +#line 285 "src/libre/dialect/native/parser.c" #ifndef ERROR_TERMINAL @@ -299,27 +291,27 @@ /* BEGINNING OF FUNCTION DECLARATIONS */ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); -static void p_267(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_265(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags, lex_state, act_state, err, t_ast__expr); -static void p_155(flags, lex_state, act_state, err); +static void p_154(flags, lex_state, act_state, err); static void p_expr_C_Clist_Hof_Hpieces(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cliteral(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); +static void p_180(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags, lex_state, act_state, err, t_endpoint *, t_pos *); -static void p_181(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); +static void p_195(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); -static void p_196(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); static void p_expr_C_Cpiece_C_Catom(flags, lex_state, act_state, err, t_ast__expr *); -static void p_248(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); +static void p_246(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); static void p_expr_C_Calt(flags, lex_state, act_state, err, t_ast__expr *); -static void p_252(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_250(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); /* BEGINNING OF STATIC VARIABLES */ @@ -339,14 +331,14 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { t_char ZIc; - /* BEGINNING OF INLINE: 142 */ + /* BEGINNING OF INLINE: 141 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -359,7 +351,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 363 "src/libre/dialect/native/parser.c" +#line 355 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -369,7 +361,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 406 "src/libre/parser.act" +#line 398 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -395,7 +387,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 399 "src/libre/dialect/native/parser.c" +#line 391 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -405,7 +397,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 547 "src/libre/parser.act" +#line 539 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -453,7 +445,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 457 "src/libre/dialect/native/parser.c" +#line 449 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -463,7 +455,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 504 "src/libre/parser.act" +#line 496 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -506,7 +498,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 510 "src/libre/dialect/native/parser.c" +#line 502 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -516,15 +508,15 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags goto ZL1; } } - /* END OF INLINE: 142 */ + /* END OF INLINE: 141 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 528 "src/libre/dialect/native/parser.c" +#line 520 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -539,50 +531,50 @@ ZL0:; } static void -p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI265, t_unsigned *ZIm, t_ast__count *ZOc) +p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI263, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI218; + t_pos ZI216; t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 398 "src/libre/parser.act" +#line 390 "src/libre/parser.act" - ZI218 = lex_state->lx.start; + ZI216 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI218; + (void) ZI216; (void) ZIend; -#line 563 "src/libre/dialect/native/parser.c" +#line 555 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 797 "src/libre/parser.act" +#line 789 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI265)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 574 "src/libre/dialect/native/parser.c" +#line 566 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 854 "src/libre/parser.act" +#line 836 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI265)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); goto ZL1; @@ -590,7 +582,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 594 "src/libre/dialect/native/parser.c" +#line 586 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -598,7 +590,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_SEP): { t_unsigned ZIn; - t_pos ZI221; + t_pos ZI219; t_pos ZIend; ADVANCE_LEXER; @@ -606,7 +598,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 647 "src/libre/parser.act" +#line 639 "src/libre/parser.act" unsigned long u; char *e; @@ -626,7 +618,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 630 "src/libre/dialect/native/parser.c" +#line 622 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -638,15 +630,15 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 398 "src/libre/parser.act" +#line 390 "src/libre/parser.act" - ZI221 = lex_state->lx.start; + ZI219 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI221; + (void) ZI219; (void) ZIend; -#line 650 "src/libre/dialect/native/parser.c" +#line 642 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -656,24 +648,24 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 797 "src/libre/parser.act" +#line 789 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI265)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 665 "src/libre/dialect/native/parser.c" +#line 657 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 854 "src/libre/parser.act" +#line 836 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI265)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); goto ZL1; @@ -681,7 +673,7 @@ p_267(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 685 "src/libre/dialect/native/parser.c" +#line 677 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -716,13 +708,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 726 "src/libre/dialect/native/parser.c" +#line 718 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -742,31 +734,31 @@ ZL1:; } static void -p_155(flags flags, lex_state lex_state, act_state act_state, err err) +p_154(flags flags, lex_state lex_state, act_state act_state, err err) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } { - t_char ZI156; + t_char ZI155; + t_pos ZI156; t_pos ZI157; - t_pos ZI158; switch (CURRENT_TERMINAL) { case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI156 = '-'; - ZI157 = lex_state->lx.start; - ZI158 = lex_state->lx.end; + ZI155 = '-'; + ZI156 = lex_state->lx.start; + ZI157 = lex_state->lx.end; + (void) ZI155; (void) ZI156; (void) ZI157; - (void) ZI158; -#line 770 "src/libre/dialect/native/parser.c" +#line 762 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -780,14 +772,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 738 "src/libre/parser.act" +#line 730 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 791 "src/libre/dialect/native/parser.c" +#line 783 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -815,16 +807,16 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 825 "src/libre/dialect/native/parser.c" +#line 817 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 225 */ + /* BEGINNING OF INLINE: 223 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_START): case (TOK_END): case (TOK_OPENSUB): @@ -840,7 +832,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; break; } } - /* END OF INLINE: 225 */ + /* END OF INLINE: 223 */ } return; ZL1:; @@ -859,30 +851,30 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err { t_char ZIc; - /* BEGINNING OF INLINE: 110 */ + /* BEGINNING OF INLINE: 109 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { + t_pos ZI117; t_pos ZI118; - t_pos ZI119; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI118 = lex_state->lx.start; - ZI119 = lex_state->lx.end; + ZI117 = lex_state->lx.start; + ZI118 = lex_state->lx.end; + (void) ZI117; (void) ZI118; - (void) ZI119; ZIc = lex_state->buf.a[0]; -#line 886 "src/libre/dialect/native/parser.c" +#line 878 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -890,12 +882,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_ESC): { + t_pos ZI111; t_pos ZI112; - t_pos ZI113; /* BEGINNING OF EXTRACT: ESC */ { -#line 406 "src/libre/parser.act" +#line 398 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -915,13 +907,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err default: break; } - ZI112 = lex_state->lx.start; - ZI113 = lex_state->lx.end; + ZI111 = lex_state->lx.start; + ZI112 = lex_state->lx.end; + (void) ZI111; (void) ZI112; - (void) ZI113; -#line 925 "src/libre/dialect/native/parser.c" +#line 917 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -929,12 +921,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_HEX): { + t_pos ZI115; t_pos ZI116; - t_pos ZI117; /* BEGINNING OF EXTRACT: HEX */ { -#line 547 "src/libre/parser.act" +#line 539 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -943,11 +935,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI116 = lex_state->lx.start; - ZI117 = lex_state->lx.end; + ZI115 = lex_state->lx.start; + ZI116 = lex_state->lx.end; + (void) ZI115; (void) ZI116; - (void) ZI117; errno = 0; @@ -982,7 +974,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 986 "src/libre/dialect/native/parser.c" +#line 978 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -990,12 +982,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_OCT): { + t_pos ZI113; t_pos ZI114; - t_pos ZI115; /* BEGINNING OF EXTRACT: OCT */ { -#line 504 "src/libre/parser.act" +#line 496 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1004,11 +996,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI114 = lex_state->lx.start; - ZI115 = lex_state->lx.end; + ZI113 = lex_state->lx.start; + ZI114 = lex_state->lx.end; + (void) ZI113; (void) ZI114; - (void) ZI115; errno = 0; @@ -1038,7 +1030,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1042 "src/libre/dialect/native/parser.c" +#line 1034 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1048,17 +1040,17 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } } - /* END OF INLINE: 110 */ + /* END OF INLINE: 109 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1062 "src/libre/dialect/native/parser.c" +#line 1054 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1078,30 +1070,30 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI261; - t_pos ZI262; - t_pos ZI263; + t_char ZI259; + t_pos ZI260; + t_pos ZI261; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI262 = lex_state->lx.start; - ZI263 = lex_state->lx.end; + ZI260 = lex_state->lx.start; + ZI261 = lex_state->lx.end; - (void) ZI262; - (void) ZI263; + (void) ZI260; + (void) ZI261; - ZI261 = lex_state->buf.a[0]; + ZI259 = lex_state->buf.a[0]; -#line 1101 "src/libre/dialect/native/parser.c" +#line 1093 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_252 (flags, lex_state, act_state, err, &ZI261, &ZI262, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI259, &ZI260, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1110,43 +1102,43 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_ESC): { - t_char ZI249; - t_pos ZI250; - t_pos ZI251; + t_char ZI247; + t_pos ZI248; + t_pos ZI249; /* BEGINNING OF EXTRACT: ESC */ { -#line 406 "src/libre/parser.act" +#line 398 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); assert(lex_state->buf.a[2] == '\0'); - ZI249 = lex_state->buf.a[1]; - - switch (ZI249) { - case 'a': ZI249 = '\a'; break; - case 'b': ZI249 = '\b'; break; - case 'e': ZI249 = '\033'; break; - case 'f': ZI249 = '\f'; break; - case 'n': ZI249 = '\n'; break; - case 'r': ZI249 = '\r'; break; - case 't': ZI249 = '\t'; break; - case 'v': ZI249 = '\v'; break; + ZI247 = lex_state->buf.a[1]; + + switch (ZI247) { + case 'a': ZI247 = '\a'; break; + case 'b': ZI247 = '\b'; break; + case 'e': ZI247 = '\033'; break; + case 'f': ZI247 = '\f'; break; + case 'n': ZI247 = '\n'; break; + case 'r': ZI247 = '\r'; break; + case 't': ZI247 = '\t'; break; + case 'v': ZI247 = '\v'; break; default: break; } - ZI250 = lex_state->lx.start; - ZI251 = lex_state->lx.end; + ZI248 = lex_state->lx.start; + ZI249 = lex_state->lx.end; - (void) ZI250; - (void) ZI251; + (void) ZI248; + (void) ZI249; -#line 1146 "src/libre/dialect/native/parser.c" +#line 1138 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; - p_252 (flags, lex_state, act_state, err, &ZI249, &ZI250, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI247, &ZI248, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1155,13 +1147,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_HEX): { - t_char ZI257; - t_pos ZI258; - t_pos ZI259; + t_char ZI255; + t_pos ZI256; + t_pos ZI257; /* BEGINNING OF EXTRACT: HEX */ { -#line 547 "src/libre/parser.act" +#line 539 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1170,11 +1162,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI258 = lex_state->lx.start; - ZI259 = lex_state->lx.end; + ZI256 = lex_state->lx.start; + ZI257 = lex_state->lx.end; - (void) ZI258; - (void) ZI259; + (void) ZI256; + (void) ZI257; errno = 0; @@ -1207,13 +1199,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI257 = (char) (unsigned char) u; + ZI255 = (char) (unsigned char) u; -#line 1213 "src/libre/dialect/native/parser.c" +#line 1205 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; - p_252 (flags, lex_state, act_state, err, &ZI257, &ZI258, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI255, &ZI256, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1222,31 +1214,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_NAMED__CLASS): { - t_ast__class__id ZI245; - t_pos ZI246; - t_pos ZI247; + t_ast__class__id ZI243; + t_pos ZI244; + t_pos ZI245; /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 667 "src/libre/parser.act" +#line 659 "src/libre/parser.act" - ZI245 = DIALECT_CLASS(lex_state->buf.a); - if (ZI245 == NULL) { + ZI243 = DIALECT_CLASS(lex_state->buf.a); + if (ZI243 == NULL) { /* syntax error -- unrecognized class */ goto ZL1; } - ZI246 = lex_state->lx.start; - ZI247 = lex_state->lx.end; + ZI244 = lex_state->lx.start; + ZI245 = lex_state->lx.end; - (void) ZI246; - (void) ZI247; + (void) ZI244; + (void) ZI245; -#line 1246 "src/libre/dialect/native/parser.c" +#line 1238 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; - p_248 (flags, lex_state, act_state, err, &ZI245, &ZI246, &ZInode); + p_246 (flags, lex_state, act_state, err, &ZI243, &ZI244, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1255,13 +1247,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_OCT): { - t_char ZI253; - t_pos ZI254; - t_pos ZI255; + t_char ZI251; + t_pos ZI252; + t_pos ZI253; /* BEGINNING OF EXTRACT: OCT */ { -#line 504 "src/libre/parser.act" +#line 496 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1270,11 +1262,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI254 = lex_state->lx.start; - ZI255 = lex_state->lx.end; + ZI252 = lex_state->lx.start; + ZI253 = lex_state->lx.end; - (void) ZI254; - (void) ZI255; + (void) ZI252; + (void) ZI253; errno = 0; @@ -1302,13 +1294,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI253 = (char) (unsigned char) u; + ZI251 = (char) (unsigned char) u; -#line 1308 "src/libre/dialect/native/parser.c" +#line 1300 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; - p_252 (flags, lex_state, act_state, err, &ZI253, &ZI254, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI251, &ZI252, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1345,7 +1337,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 667 "src/libre/parser.act" +#line 659 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1359,7 +1351,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1363 "src/libre/dialect/native/parser.c" +#line 1355 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1369,12 +1361,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 874 "src/libre/parser.act" +#line 856 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1378 "src/libre/dialect/native/parser.c" +#line 1370 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1399,43 +1391,42 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { t_pos ZIstart; t_ast__expr ZItmp; - t_pos ZIend; - /* BEGINNING OF INLINE: 167 */ + /* BEGINNING OF INLINE: 166 */ { switch (CURRENT_TERMINAL) { case (TOK_OPENGROUP): { - t_pos ZI168; + t_pos ZI167; /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 338 "src/libre/parser.act" +#line 330 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI168 = lex_state->lx.end; + ZI167 = lex_state->lx.end; (void) ZIstart; - (void) ZI168; + (void) ZI167; -#line 1422 "src/libre/dialect/native/parser.c" +#line 1413 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1435 "src/libre/dialect/native/parser.c" +#line 1426 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; - p_181 (flags, lex_state, act_state, err, &ZItmp); + p_180 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1445,60 +1436,60 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPCB): { - t_pos ZI187; + t_pos ZI186; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 354 "src/libre/parser.act" +#line 346 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI187 = lex_state->lx.end; + ZI186 = lex_state->lx.end; (void) ZIstart; - (void) ZI187; + (void) ZI186; -#line 1463 "src/libre/dialect/native/parser.c" +#line 1454 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1476 "src/libre/dialect/native/parser.c" +#line 1467 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 915 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1486 "src/libre/dialect/native/parser.c" +#line 1477 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1502 "src/libre/dialect/native/parser.c" +#line 1493 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1510,38 +1501,38 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINV): { - t_pos ZI179; + t_pos ZI178; /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 346 "src/libre/parser.act" +#line 338 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI179 = lex_state->lx.end; + ZI178 = lex_state->lx.end; (void) ZIstart; - (void) ZI179; + (void) ZI178; -#line 1526 "src/libre/dialect/native/parser.c" +#line 1517 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1539 "src/libre/dialect/native/parser.c" +#line 1530 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 996 "src/libre/parser.act" +#line 978 "src/libre/parser.act" struct ast_expr *any; @@ -1579,10 +1570,10 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1583 "src/libre/dialect/native/parser.c" +#line 1574 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ - p_181 (flags, lex_state, act_state, err, &ZItmp); + p_180 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1592,40 +1583,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINVCB): { - t_pos ZI194; + t_pos ZI193; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 362 "src/libre/parser.act" +#line 354 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI194 = lex_state->lx.end; + ZI193 = lex_state->lx.end; (void) ZIstart; - (void) ZI194; + (void) ZI193; -#line 1610 "src/libre/dialect/native/parser.c" +#line 1601 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1623 "src/libre/dialect/native/parser.c" +#line 1614 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 996 "src/libre/parser.act" +#line 978 "src/libre/parser.act" struct ast_expr *any; @@ -1663,32 +1654,32 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1667 "src/libre/dialect/native/parser.c" +#line 1658 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 915 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1676 "src/libre/dialect/native/parser.c" +#line 1667 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1692 "src/libre/dialect/native/parser.c" +#line 1683 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1702,39 +1693,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } } - /* END OF INLINE: 167 */ - /* BEGINNING OF INLINE: 200 */ + /* END OF INLINE: 166 */ + /* BEGINNING OF INLINE: 199 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSEGROUP): { - t_char ZI201; - t_pos ZI202; + t_char ZI200; + t_pos ZI201; + t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 370 "src/libre/parser.act" +#line 362 "src/libre/parser.act" - ZI201 = ']'; - ZI202 = lex_state->lx.start; + ZI200 = ']'; + ZI201 = lex_state->lx.start; ZIend = lex_state->lx.end; + (void) ZI200; (void) ZI201; - (void) ZI202; (void) ZIend; -#line 1727 "src/libre/dialect/native/parser.c" +#line 1719 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 787 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1738 "src/libre/dialect/native/parser.c" +#line 1730 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1742,56 +1734,57 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUPRANGE): { t_char ZIcrange; - t_pos ZI204; + t_pos ZI203; + t_pos ZIend; t_ast__expr ZIrange; /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 380 "src/libre/parser.act" +#line 372 "src/libre/parser.act" ZIcrange = '-'; - ZI204 = lex_state->lx.start; + ZI203 = lex_state->lx.start; ZIend = lex_state->lx.end; (void) ZIcrange; - (void) ZI204; + (void) ZI203; (void) ZIend; -#line 1761 "src/libre/dialect/native/parser.c" +#line 1754 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 1774 "src/libre/dialect/native/parser.c" +#line 1767 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 1785 "src/libre/dialect/native/parser.c" +#line 1778 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 787 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1795 "src/libre/dialect/native/parser.c" +#line 1788 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1804,36 +1797,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 745 "src/libre/parser.act" +#line 737 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 1815 "src/libre/dialect/native/parser.c" +#line 1808 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ - ZIend = ZIstart; } ZL3:; } - /* END OF INLINE: 200 */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 802 "src/libre/parser.act" - - (void) (ZIstart); - (void) (ZIend); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 1835 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: mark-expr */ + /* END OF INLINE: 199 */ } goto ZL0; ZL1:; @@ -1844,120 +1821,47 @@ ZL0:; } static void -p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_state, act_state act_state, err err, t_endpoint *ZOr, t_pos *ZOend) -{ - t_endpoint ZIr; - t_pos ZIend; - - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - /* BEGINNING OF INLINE: 148 */ - { - switch (CURRENT_TERMINAL) { - case (TOK_RANGE): - { - t_char ZIc; - t_pos ZI150; - - /* BEGINNING OF EXTRACT: RANGE */ - { -#line 328 "src/libre/parser.act" - - ZIc = '-'; - ZI150 = lex_state->lx.start; - ZIend = lex_state->lx.end; - - (void) ZIc; - (void) ZI150; - (void) ZIend; - -#line 1877 "src/libre/dialect/native/parser.c" - } - /* END OF EXTRACT: RANGE */ - ADVANCE_LEXER; - /* BEGINNING OF ACTION: ast-range-endpoint-literal */ - { -#line 869 "src/libre/parser.act" - - (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char) (ZIc); - -#line 1888 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: ast-range-endpoint-literal */ - } - break; - case (TOK_NAMED__CLASS): case (TOK_ESC): case (TOK_OCT): case (TOK_HEX): - case (TOK_CHAR): - { - t_pos ZI149; - - p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI149, &ZIend); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - break; - default: - goto ZL1; - } - } - /* END OF INLINE: 148 */ - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOr = ZIr; - *ZOend = ZIend; -} - -static void -p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) +p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; t_pos ZIrstart; - t_pos ZI182; + t_pos ZI181; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; - ZI182 = lex_state->lx.end; + ZI181 = lex_state->lx.end; (void) ZIc; (void) ZIrstart; - (void) ZI182; + (void) ZI181; -#line 1943 "src/libre/dialect/native/parser.c" +#line 1847 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; - /* BEGINNING OF INLINE: 183 */ + /* BEGINNING OF INLINE: 182 */ { switch (CURRENT_TERMINAL) { default: { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 1961 "src/libre/dialect/native/parser.c" +#line 1865 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1965,35 +1869,35 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp case (TOK_RANGE): { t_endpoint ZIlower; - t_char ZI184; + t_char ZI183; + t_pos ZI184; t_pos ZI185; - t_pos ZI186; t_endpoint ZIupper; t_pos ZIend; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 1982 "src/libre/dialect/native/parser.c" +#line 1886 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI184 = '-'; - ZI185 = lex_state->lx.start; - ZI186 = lex_state->lx.end; + ZI183 = '-'; + ZI184 = lex_state->lx.start; + ZI185 = lex_state->lx.end; + (void) ZI183; (void) ZI184; (void) ZI185; - (void) ZI186; -#line 1997 "src/libre/dialect/native/parser.c" +#line 1901 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2004,7 +1908,7 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -2033,23 +1937,23 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2037 "src/libre/dialect/native/parser.c" +#line 1941 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } break; } } - /* END OF INLINE: 183 */ + /* END OF INLINE: 182 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2053 "src/libre/dialect/native/parser.c" +#line 1957 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2065,6 +1969,79 @@ ZL1:; return; } +static void +p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_state, act_state act_state, err err, t_endpoint *ZOr, t_pos *ZOend) +{ + t_endpoint ZIr; + t_pos ZIend; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + /* BEGINNING OF INLINE: 147 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_RANGE): + { + t_char ZIc; + t_pos ZI149; + + /* BEGINNING OF EXTRACT: RANGE */ + { +#line 320 "src/libre/parser.act" + + ZIc = '-'; + ZI149 = lex_state->lx.start; + ZIend = lex_state->lx.end; + + (void) ZIc; + (void) ZI149; + (void) ZIend; + +#line 2003 "src/libre/dialect/native/parser.c" + } + /* END OF EXTRACT: RANGE */ + ADVANCE_LEXER; + /* BEGINNING OF ACTION: ast-range-endpoint-literal */ + { +#line 851 "src/libre/parser.act" + + (ZIr).type = AST_ENDPOINT_LITERAL; + (ZIr).u.literal.c = (unsigned char) (ZIc); + +#line 2014 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: ast-range-endpoint-literal */ + } + break; + case (TOK_NAMED__CLASS): case (TOK_ESC): case (TOK_OCT): case (TOK_HEX): + case (TOK_CHAR): + { + t_pos ZI148; + + p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI148, &ZIend); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + default: + goto ZL1; + } + } + /* END OF INLINE: 147 */ + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOr = ZIr; + *ZOend = ZIend; +} + static void p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { @@ -2085,7 +2062,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2099,7 +2076,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2103 "src/libre/dialect/native/parser.c" +#line 2080 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2123,14 +2100,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2134 "src/libre/dialect/native/parser.c" +#line 2111 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2144,14 +2121,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2155 "src/libre/dialect/native/parser.c" +#line 2132 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2164,26 +2141,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 731 "src/libre/parser.act" +#line 723 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2175 "src/libre/dialect/native/parser.c" +#line 2152 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2187 "src/libre/dialect/native/parser.c" +#line 2164 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2195,85 +2172,8 @@ ZL0:; *ZOnode = ZInode; } -void -p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) -{ - t_ast__expr ZInode; - - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - t_group__id ZIid; - t_ast__expr ZIe; - - /* BEGINNING OF ACTION: make-group-id */ - { -#line 911 "src/libre/parser.act" - - (ZIid) = act_state->group_id++; - -#line 2217 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: make-group-id */ - p_expr (flags, lex_state, act_state, err, &ZIe); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - /* BEGINNING OF ACTION: ast-make-group */ - { -#line 941 "src/libre/parser.act" - - (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); - if ((ZInode) == NULL) { - goto ZL1; - } - -#line 2234 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: ast-make-group */ - /* BEGINNING OF INLINE: 233 */ - { - { - switch (CURRENT_TERMINAL) { - case (TOK_EOF): - break; - default: - goto ZL3; - } - ADVANCE_LEXER; - } - goto ZL2; - ZL3:; - { - /* BEGINNING OF ACTION: err-expected-eof */ - { -#line 773 "src/libre/parser.act" - - if (err->e == RE_ESUCCESS) { - err->e = RE_EXEOF; - } - goto ZL1; - -#line 2260 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: err-expected-eof */ - } - ZL2:; - } - /* END OF INLINE: 233 */ - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOnode = ZInode; -} - static void -p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) +p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) { t_ast__expr ZInode1; @@ -2282,14 +2182,14 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2293 "src/libre/dialect/native/parser.c" +#line 2193 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2297,36 +2197,36 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs case (TOK_RANGE): { t_endpoint ZIr; - t_char ZI197; + t_char ZI196; + t_pos ZI197; t_pos ZI198; - t_pos ZI199; t_endpoint ZIupper; t_pos ZIend; t_endpoint ZIlower; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2315 "src/libre/dialect/native/parser.c" +#line 2215 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI197 = '-'; - ZI198 = lex_state->lx.start; - ZI199 = lex_state->lx.end; + ZI196 = '-'; + ZI197 = lex_state->lx.start; + ZI198 = lex_state->lx.end; + (void) ZI196; (void) ZI197; (void) ZI198; - (void) ZI199; -#line 2330 "src/libre/dialect/native/parser.c" +#line 2230 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2337,17 +2237,17 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2346 "src/libre/dialect/native/parser.c" +#line 2246 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -2376,7 +2276,7 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2380 "src/libre/dialect/native/parser.c" +#line 2280 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2392,6 +2292,83 @@ ZL0:; *ZOnode1 = ZInode1; } +void +p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) +{ + t_ast__expr ZInode; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 893 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 2314 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 923 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 2331 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 231 */ + { + { + switch (CURRENT_TERMINAL) { + case (TOK_EOF): + break; + default: + goto ZL3; + } + ADVANCE_LEXER; + } + goto ZL2; + ZL3:; + { + /* BEGINNING OF ACTION: err-expected-eof */ + { +#line 765 "src/libre/parser.act" + + if (err->e == RE_ESUCCESS) { + err->e = RE_EXEOF; + } + goto ZL1; + +#line 2357 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: err-expected-eof */ + } + ZL2:; + } + /* END OF INLINE: 231 */ + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOnode = ZInode; +} + static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, act_state act_state, err err, t_endpoint *ZOr, t_pos *ZOstart, t_pos *ZOend) { @@ -2403,7 +2380,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, return; } { - /* BEGINNING OF INLINE: 145 */ + /* BEGINNING OF INLINE: 144 */ { switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): @@ -2428,7 +2405,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, goto ZL1; } } - /* END OF INLINE: 145 */ + /* END OF INLINE: 144 */ } goto ZL0; ZL1:; @@ -2457,16 +2434,16 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2467 "src/libre/dialect/native/parser.c" +#line 2444 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF INLINE: 231 */ + /* BEGINNING OF INLINE: 229 */ { switch (CURRENT_TERMINAL) { case (TOK_ALT): @@ -2481,21 +2458,21 @@ ZL2_expr_C_Clist_Hof_Halts:; break; } } - /* END OF INLINE: 231 */ + /* END OF INLINE: 229 */ } return; ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 731 "src/libre/parser.act" +#line 723 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 2499 "src/libre/dialect/native/parser.c" +#line 2476 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2514,21 +2491,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { - t_pos ZI265; - t_pos ZI266; + t_pos ZI263; + t_pos ZI264; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 390 "src/libre/parser.act" +#line 382 "src/libre/parser.act" - ZI265 = lex_state->lx.start; - ZI266 = lex_state->lx.end; + ZI263 = lex_state->lx.start; + ZI264 = lex_state->lx.end; - (void) ZI265; - (void) ZI266; + (void) ZI263; + (void) ZI264; -#line 2532 "src/libre/dialect/native/parser.c" +#line 2509 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2536,7 +2513,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 647 "src/libre/parser.act" +#line 639 "src/libre/parser.act" unsigned long u; char *e; @@ -2556,7 +2533,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2560 "src/libre/dialect/native/parser.c" +#line 2537 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2564,7 +2541,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_267 (flags, lex_state, act_state, err, &ZI265, &ZIm, &ZIc); + p_265 (flags, lex_state, act_state, err, &ZI263, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2576,11 +2553,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 846 "src/libre/parser.act" +#line 828 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 2584 "src/libre/dialect/native/parser.c" +#line 2561 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2590,11 +2567,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 842 "src/libre/parser.act" +#line 824 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2598 "src/libre/dialect/native/parser.c" +#line 2575 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2604,11 +2581,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 838 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2612 "src/libre/dialect/native/parser.c" +#line 2589 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2617,11 +2594,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 850 "src/libre/parser.act" +#line 832 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2625 "src/libre/dialect/native/parser.c" +#line 2602 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2634,23 +2611,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 717 "src/libre/parser.act" +#line 709 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 2645 "src/libre/dialect/native/parser.c" +#line 2622 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 850 "src/libre/parser.act" +#line 832 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2654 "src/libre/dialect/native/parser.c" +#line 2631 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2675,24 +2652,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2684 "src/libre/dialect/native/parser.c" +#line 2661 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2696 "src/libre/dialect/native/parser.c" +#line 2673 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2702,14 +2679,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 972 "src/libre/parser.act" +#line 954 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 2713 "src/libre/dialect/native/parser.c" +#line 2690 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2722,11 +2699,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2730 "src/libre/dialect/native/parser.c" +#line 2707 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2736,14 +2713,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 2747 "src/libre/dialect/native/parser.c" +#line 2724 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2760,14 +2737,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 965 "src/libre/parser.act" +#line 947 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 2771 "src/libre/dialect/native/parser.c" +#line 2748 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2800,26 +2777,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 724 "src/libre/parser.act" +#line 716 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 2811 "src/libre/dialect/native/parser.c" +#line 2788 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2823 "src/libre/dialect/native/parser.c" +#line 2800 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2832,7 +2809,7 @@ ZL0:; } static void -p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI245, t_pos *ZI246, t_ast__expr *ZOnode) +p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI243, t_pos *ZI244, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2841,14 +2818,14 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" - (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI245)); + (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI243)); if ((ZInode) == NULL) { goto ZL1; } -#line 2852 "src/libre/dialect/native/parser.c" +#line 2829 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2861,15 +2838,15 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 874 "src/libre/parser.act" +#line 856 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; - (ZIlower).u.named.class = (*ZI245); + (ZIlower).u.named.class = (*ZI243); -#line 2870 "src/libre/dialect/native/parser.c" +#line 2847 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ - p_155 (flags, lex_state, act_state, err); + p_154 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -2877,17 +2854,17 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 792 "src/libre/parser.act" +#line 784 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI246)); + mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); -#line 2886 "src/libre/dialect/native/parser.c" +#line 2863 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -2916,7 +2893,7 @@ p_248(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2920 "src/libre/dialect/native/parser.c" +#line 2897 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2943,14 +2920,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2954 "src/libre/dialect/native/parser.c" +#line 2931 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2968,7 +2945,7 @@ ZL0:; } static void -p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI249, t_pos *ZI250, t_ast__expr *ZOnode) +p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI247, t_pos *ZI248, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2977,14 +2954,14 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI249)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI247)); if ((ZInode) == NULL) { goto ZL1; } -#line 2988 "src/libre/dialect/native/parser.c" +#line 2965 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2997,15 +2974,15 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char) (*ZI249); + (ZIlower).u.literal.c = (unsigned char) (*ZI247); -#line 3006 "src/libre/dialect/native/parser.c" +#line 2983 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ - p_155 (flags, lex_state, act_state, err); + p_154 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -3013,17 +2990,17 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 792 "src/libre/parser.act" +#line 784 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI250)); + mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); -#line 3022 "src/libre/dialect/native/parser.c" +#line 2999 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -3052,7 +3029,7 @@ p_252(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3056 "src/libre/dialect/native/parser.c" +#line 3033 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3070,7 +3047,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1082 "src/libre/parser.act" +#line 1064 "src/libre/parser.act" static int @@ -3228,6 +3205,6 @@ ZL0:; return NULL; } -#line 3232 "src/libre/dialect/native/parser.c" +#line 3209 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 8591ec200..6aaf2a576 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 301 "src/libre/parser.act" +#line 293 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1239 "src/libre/parser.act" +#line 1221 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/native/parser.sid b/src/libre/dialect/native/parser.sid index 20f0912a3..aeb894c64 100644 --- a/src/libre/dialect/native/parser.sid +++ b/src/libre/dialect/native/parser.sid @@ -134,7 +134,6 @@ : (:pos, :pos) -> (); : (:pos, :pos) -> (); : (:pos, :pos) -> (); - : (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -350,10 +349,7 @@ (start, end); ## ; - end = start; /* appease sid */ }; - - (tmp, start, end); }; piece: () -> (node :ast_expr) [ diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index c1cbc2bd4..8536441e3 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -289,7 +281,7 @@ return s; } -#line 293 "src/libre/dialect/pcre/parser.c" +#line 285 "src/libre/dialect/pcre/parser.c" #ifndef ERROR_TERMINAL @@ -302,25 +294,25 @@ static void p_expr_C_Cflags_C_Cflag__set(flags, lex_state, act_state, err, t_re_ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Clist_Hof_Hpieces(flags, lex_state, act_state, err, t_ast__expr); -static void p_295(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); -static void p_169(flags, lex_state, act_state, err); +static void p_293(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); +static void p_168(flags, lex_state, act_state, err); static void p_expr_C_Cliteral(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccomment(flags, lex_state, act_state, err); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags, lex_state, act_state, err, t_endpoint *, t_pos *); -static void p_319(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_317(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); +static void p_320(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); -static void p_322(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); -static void p_195(flags, lex_state, act_state, err, t_ast__expr *); -static void p_323(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_321(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_194(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cflags(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags, lex_state, act_state, err, t_ast__expr, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_class_Hnamed(flags, lex_state, act_state, err, t_ast__expr *, t_pos *, t_pos *); -static void p_210(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); +static void p_209(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); @@ -345,21 +337,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 685 "src/libre/parser.act" +#line 677 "src/libre/parser.act" ZIc = RE_EXTENDED; -#line 353 "src/libre/dialect/pcre/parser.c" +#line 345 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_EXTENDED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 830 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 363 "src/libre/dialect/pcre/parser.c" +#line 355 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -376,21 +368,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 681 "src/libre/parser.act" +#line 673 "src/libre/parser.act" ZIc = RE_ICASE; -#line 384 "src/libre/dialect/pcre/parser.c" +#line 376 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_INSENSITIVE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 830 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 394 "src/libre/dialect/pcre/parser.c" +#line 386 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -401,21 +393,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 689 "src/libre/parser.act" +#line 681 "src/libre/parser.act" ZIc = RE_SINGLE; -#line 409 "src/libre/dialect/pcre/parser.c" +#line 401 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_SINGLE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 830 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 419 "src/libre/dialect/pcre/parser.c" +#line 411 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -426,14 +418,14 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 759 "src/libre/parser.act" +#line 751 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; } goto ZL1; -#line 437 "src/libre/dialect/pcre/parser.c" +#line 429 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unknown-flag */ } @@ -464,14 +456,14 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { t_char ZIc; - /* BEGINNING OF INLINE: 156 */ + /* BEGINNING OF INLINE: 155 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -484,7 +476,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 488 "src/libre/dialect/pcre/parser.c" +#line 480 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -494,7 +486,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 462 "src/libre/parser.act" +#line 454 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -536,20 +528,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 540 "src/libre/dialect/pcre/parser.c" +#line 532 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 553 "src/libre/dialect/pcre/parser.c" +#line 545 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -558,7 +550,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 406 "src/libre/parser.act" +#line 398 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -584,7 +576,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 588 "src/libre/dialect/pcre/parser.c" +#line 580 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -594,7 +586,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 547 "src/libre/parser.act" +#line 539 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -642,7 +634,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 646 "src/libre/dialect/pcre/parser.c" +#line 638 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -652,7 +644,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 504 "src/libre/parser.act" +#line 496 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -695,7 +687,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 699 "src/libre/dialect/pcre/parser.c" +#line 691 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -705,7 +697,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 446 "src/libre/parser.act" +#line 438 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -721,20 +713,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 725 "src/libre/dialect/pcre/parser.c" +#line 717 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 738 "src/libre/dialect/pcre/parser.c" +#line 730 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -743,15 +735,15 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags goto ZL1; } } - /* END OF INLINE: 156 */ + /* END OF INLINE: 155 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 755 "src/libre/dialect/pcre/parser.c" +#line 747 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -782,13 +774,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 792 "src/libre/dialect/pcre/parser.c" +#line 784 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -815,7 +807,7 @@ p_expr_C_Clist_Hof_Hpieces(flags flags, lex_state lex_state, act_state act_state } ZL2_expr_C_Clist_Hof_Hpieces:; { - /* BEGINNING OF INLINE: 273 */ + /* BEGINNING OF INLINE: 271 */ { switch (CURRENT_TERMINAL) { case (TOK_INVALID__COMMENT): @@ -842,13 +834,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 852 "src/libre/dialect/pcre/parser.c" +#line 844 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ } @@ -857,8 +849,8 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL1; } } - /* END OF INLINE: 273 */ - /* BEGINNING OF INLINE: 274 */ + /* END OF INLINE: 271 */ + /* BEGINNING OF INLINE: 272 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_EOL): case (TOK_START): case (TOK_END): @@ -876,7 +868,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; break; } } - /* END OF INLINE: 274 */ + /* END OF INLINE: 272 */ } return; ZL1:; @@ -885,7 +877,7 @@ ZL1:; } static void -p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI292, t_pos *ZI293, t_ast__expr *ZOnode) +p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI290, t_pos *ZI291, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -894,14 +886,14 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" - (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI292)); + (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI290)); if ((ZInode) == NULL) { goto ZL1; } -#line 905 "src/libre/dialect/pcre/parser.c" +#line 897 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -914,15 +906,15 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 874 "src/libre/parser.act" +#line 856 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; - (ZIlower).u.named.class = (*ZI292); + (ZIlower).u.named.class = (*ZI290); -#line 923 "src/libre/dialect/pcre/parser.c" +#line 915 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ - p_169 (flags, lex_state, act_state, err); + p_168 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -930,17 +922,17 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 792 "src/libre/parser.act" +#line 784 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI293)); + mark(&act_state->rangestart, &(*ZI291)); mark(&act_state->rangeend, &(ZIend)); -#line 939 "src/libre/dialect/pcre/parser.c" +#line 931 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -969,7 +961,7 @@ p_295(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 973 "src/libre/dialect/pcre/parser.c" +#line 965 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -986,31 +978,31 @@ ZL0:; } static void -p_169(flags flags, lex_state lex_state, act_state act_state, err err) +p_168(flags flags, lex_state lex_state, act_state act_state, err err) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } { - t_char ZI170; + t_char ZI169; + t_pos ZI170; t_pos ZI171; - t_pos ZI172; switch (CURRENT_TERMINAL) { case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI170 = '-'; - ZI171 = lex_state->lx.start; - ZI172 = lex_state->lx.end; + ZI169 = '-'; + ZI170 = lex_state->lx.start; + ZI171 = lex_state->lx.end; + (void) ZI169; (void) ZI170; (void) ZI171; - (void) ZI172; -#line 1014 "src/libre/dialect/pcre/parser.c" +#line 1006 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1024,14 +1016,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 738 "src/libre/parser.act" +#line 730 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 1035 "src/libre/dialect/pcre/parser.c" +#line 1027 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1053,30 +1045,30 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err { t_char ZIc; - /* BEGINNING OF INLINE: 112 */ + /* BEGINNING OF INLINE: 111 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { + t_pos ZI121; t_pos ZI122; - t_pos ZI123; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI122 = lex_state->lx.start; - ZI123 = lex_state->lx.end; + ZI121 = lex_state->lx.start; + ZI122 = lex_state->lx.end; + (void) ZI121; (void) ZI122; - (void) ZI123; ZIc = lex_state->buf.a[0]; -#line 1080 "src/libre/dialect/pcre/parser.c" +#line 1072 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1084,12 +1076,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_CONTROL): { + t_pos ZI123; t_pos ZI124; - t_pos ZI125; /* BEGINNING OF EXTRACT: CONTROL */ { -#line 462 "src/libre/parser.act" +#line 454 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1125,13 +1117,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = cc; } - ZI124 = lex_state->lx.start; - ZI125 = lex_state->lx.end; + ZI123 = lex_state->lx.start; + ZI124 = lex_state->lx.end; + (void) ZI123; (void) ZI124; - (void) ZI125; -#line 1135 "src/libre/dialect/pcre/parser.c" +#line 1127 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1139,12 +1131,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_ESC): { + t_pos ZI113; t_pos ZI114; - t_pos ZI115; /* BEGINNING OF EXTRACT: ESC */ { -#line 406 "src/libre/parser.act" +#line 398 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1164,13 +1156,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err default: break; } - ZI114 = lex_state->lx.start; - ZI115 = lex_state->lx.end; + ZI113 = lex_state->lx.start; + ZI114 = lex_state->lx.end; + (void) ZI113; (void) ZI114; - (void) ZI115; -#line 1174 "src/libre/dialect/pcre/parser.c" +#line 1166 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1178,12 +1170,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_HEX): { + t_pos ZI119; t_pos ZI120; - t_pos ZI121; /* BEGINNING OF EXTRACT: HEX */ { -#line 547 "src/libre/parser.act" +#line 539 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1192,11 +1184,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI120 = lex_state->lx.start; - ZI121 = lex_state->lx.end; + ZI119 = lex_state->lx.start; + ZI120 = lex_state->lx.end; + (void) ZI119; (void) ZI120; - (void) ZI121; errno = 0; @@ -1231,7 +1223,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1235 "src/libre/dialect/pcre/parser.c" +#line 1227 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1239,12 +1231,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_NOESC): { + t_pos ZI115; t_pos ZI116; - t_pos ZI117; /* BEGINNING OF EXTRACT: NOESC */ { -#line 432 "src/libre/parser.act" +#line 424 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1252,13 +1244,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[1]; - ZI116 = lex_state->lx.start; - ZI117 = lex_state->lx.end; + ZI115 = lex_state->lx.start; + ZI116 = lex_state->lx.end; + (void) ZI115; (void) ZI116; - (void) ZI117; -#line 1262 "src/libre/dialect/pcre/parser.c" +#line 1254 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1266,12 +1258,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_OCT): { + t_pos ZI117; t_pos ZI118; - t_pos ZI119; /* BEGINNING OF EXTRACT: OCT */ { -#line 504 "src/libre/parser.act" +#line 496 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1280,11 +1272,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI118 = lex_state->lx.start; - ZI119 = lex_state->lx.end; + ZI117 = lex_state->lx.start; + ZI118 = lex_state->lx.end; + (void) ZI117; (void) ZI118; - (void) ZI119; errno = 0; @@ -1314,7 +1306,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1318 "src/libre/dialect/pcre/parser.c" +#line 1310 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1322,12 +1314,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_UNSUPPORTED): { + t_pos ZI125; t_pos ZI126; - t_pos ZI127; /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 446 "src/libre/parser.act" +#line 438 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1337,26 +1329,26 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; } - ZI126 = lex_state->lx.start; - ZI127 = lex_state->lx.end; + ZI125 = lex_state->lx.start; + ZI126 = lex_state->lx.end; + (void) ZI125; (void) ZI126; - (void) ZI127; -#line 1347 "src/libre/dialect/pcre/parser.c" +#line 1339 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1360 "src/libre/dialect/pcre/parser.c" +#line 1352 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1365,17 +1357,17 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } } - /* END OF INLINE: 112 */ + /* END OF INLINE: 111 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1379 "src/libre/dialect/pcre/parser.c" +#line 1371 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1395,30 +1387,30 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI308; - t_pos ZI309; - t_pos ZI310; + t_char ZI306; + t_pos ZI307; + t_pos ZI308; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI309 = lex_state->lx.start; - ZI310 = lex_state->lx.end; + ZI307 = lex_state->lx.start; + ZI308 = lex_state->lx.end; - (void) ZI309; - (void) ZI310; + (void) ZI307; + (void) ZI308; - ZI308 = lex_state->buf.a[0]; + ZI306 = lex_state->buf.a[0]; -#line 1418 "src/libre/dialect/pcre/parser.c" +#line 1410 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_319 (flags, lex_state, act_state, err, &ZI308, &ZI309, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI306, &ZI307, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1427,20 +1419,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_CONTROL): { - t_char ZI312; - t_pos ZI313; - t_pos ZI314; + t_char ZI310; + t_pos ZI311; + t_pos ZI312; /* BEGINNING OF EXTRACT: CONTROL */ { -#line 462 "src/libre/parser.act" +#line 454 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); assert(lex_state->buf.a[2] != '\0'); assert(lex_state->buf.a[3] == '\0'); - ZI312 = lex_state->buf.a[2]; + ZI310 = lex_state->buf.a[2]; /* * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 @@ -1455,7 +1447,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ */ { - unsigned char cc = (unsigned char) ZI312; + unsigned char cc = (unsigned char) ZI310; if (cc > 126 || cc < 32) { goto ZL1; } @@ -1466,32 +1458,32 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ cc ^= 0x40; - ZI312 = cc; + ZI310 = cc; } - ZI313 = lex_state->lx.start; - ZI314 = lex_state->lx.end; + ZI311 = lex_state->lx.start; + ZI312 = lex_state->lx.end; - (void) ZI313; - (void) ZI314; + (void) ZI311; + (void) ZI312; -#line 1479 "src/libre/dialect/pcre/parser.c" +#line 1471 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1492 "src/libre/dialect/pcre/parser.c" +#line 1484 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ - p_319 (flags, lex_state, act_state, err, &ZI312, &ZI313, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI310, &ZI311, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1500,43 +1492,43 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_ESC): { - t_char ZI296; - t_pos ZI297; - t_pos ZI298; + t_char ZI294; + t_pos ZI295; + t_pos ZI296; /* BEGINNING OF EXTRACT: ESC */ { -#line 406 "src/libre/parser.act" +#line 398 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); assert(lex_state->buf.a[2] == '\0'); - ZI296 = lex_state->buf.a[1]; - - switch (ZI296) { - case 'a': ZI296 = '\a'; break; - case 'b': ZI296 = '\b'; break; - case 'e': ZI296 = '\033'; break; - case 'f': ZI296 = '\f'; break; - case 'n': ZI296 = '\n'; break; - case 'r': ZI296 = '\r'; break; - case 't': ZI296 = '\t'; break; - case 'v': ZI296 = '\v'; break; + ZI294 = lex_state->buf.a[1]; + + switch (ZI294) { + case 'a': ZI294 = '\a'; break; + case 'b': ZI294 = '\b'; break; + case 'e': ZI294 = '\033'; break; + case 'f': ZI294 = '\f'; break; + case 'n': ZI294 = '\n'; break; + case 'r': ZI294 = '\r'; break; + case 't': ZI294 = '\t'; break; + case 'v': ZI294 = '\v'; break; default: break; } - ZI297 = lex_state->lx.start; - ZI298 = lex_state->lx.end; + ZI295 = lex_state->lx.start; + ZI296 = lex_state->lx.end; - (void) ZI297; - (void) ZI298; + (void) ZI295; + (void) ZI296; -#line 1536 "src/libre/dialect/pcre/parser.c" +#line 1528 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; - p_319 (flags, lex_state, act_state, err, &ZI296, &ZI297, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI294, &ZI295, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1545,13 +1537,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_HEX): { - t_char ZI304; - t_pos ZI305; - t_pos ZI306; + t_char ZI302; + t_pos ZI303; + t_pos ZI304; /* BEGINNING OF EXTRACT: HEX */ { -#line 547 "src/libre/parser.act" +#line 539 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1560,11 +1552,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI305 = lex_state->lx.start; - ZI306 = lex_state->lx.end; + ZI303 = lex_state->lx.start; + ZI304 = lex_state->lx.end; - (void) ZI305; - (void) ZI306; + (void) ZI303; + (void) ZI304; errno = 0; @@ -1597,13 +1589,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI304 = (char) (unsigned char) u; + ZI302 = (char) (unsigned char) u; -#line 1603 "src/libre/dialect/pcre/parser.c" +#line 1595 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; - p_319 (flags, lex_state, act_state, err, &ZI304, &ZI305, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI302, &ZI303, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1612,31 +1604,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_NAMED__CLASS): { - t_ast__class__id ZI292; - t_pos ZI293; - t_pos ZI294; + t_ast__class__id ZI290; + t_pos ZI291; + t_pos ZI292; /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 667 "src/libre/parser.act" +#line 659 "src/libre/parser.act" - ZI292 = DIALECT_CLASS(lex_state->buf.a); - if (ZI292 == NULL) { + ZI290 = DIALECT_CLASS(lex_state->buf.a); + if (ZI290 == NULL) { /* syntax error -- unrecognized class */ goto ZL1; } - ZI293 = lex_state->lx.start; - ZI294 = lex_state->lx.end; + ZI291 = lex_state->lx.start; + ZI292 = lex_state->lx.end; - (void) ZI293; - (void) ZI294; + (void) ZI291; + (void) ZI292; -#line 1636 "src/libre/dialect/pcre/parser.c" +#line 1628 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; - p_295 (flags, lex_state, act_state, err, &ZI292, &ZI293, &ZInode); + p_293 (flags, lex_state, act_state, err, &ZI290, &ZI291, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1646,12 +1638,12 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ case (TOK_NOESC): { t_char ZIc; + t_pos ZI134; t_pos ZI135; - t_pos ZI136; /* BEGINNING OF EXTRACT: NOESC */ { -#line 432 "src/libre/parser.act" +#line 424 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1659,39 +1651,39 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZIc = lex_state->buf.a[1]; - ZI135 = lex_state->lx.start; - ZI136 = lex_state->lx.end; + ZI134 = lex_state->lx.start; + ZI135 = lex_state->lx.end; + (void) ZI134; (void) ZI135; - (void) ZI136; -#line 1669 "src/libre/dialect/pcre/parser.c" +#line 1661 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1682 "src/libre/dialect/pcre/parser.c" +#line 1674 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } break; case (TOK_OCT): { - t_char ZI300; - t_pos ZI301; - t_pos ZI302; + t_char ZI298; + t_pos ZI299; + t_pos ZI300; /* BEGINNING OF EXTRACT: OCT */ { -#line 504 "src/libre/parser.act" +#line 496 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1700,11 +1692,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI301 = lex_state->lx.start; - ZI302 = lex_state->lx.end; + ZI299 = lex_state->lx.start; + ZI300 = lex_state->lx.end; - (void) ZI301; - (void) ZI302; + (void) ZI299; + (void) ZI300; errno = 0; @@ -1732,13 +1724,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI300 = (char) (unsigned char) u; + ZI298 = (char) (unsigned char) u; -#line 1738 "src/libre/dialect/pcre/parser.c" +#line 1730 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; - p_319 (flags, lex_state, act_state, err, &ZI300, &ZI301, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI298, &ZI299, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1747,45 +1739,45 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_UNSUPPORTED): { - t_char ZI316; - t_pos ZI317; - t_pos ZI318; + t_char ZI314; + t_pos ZI315; + t_pos ZI316; /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 446 "src/libre/parser.act" +#line 438 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { - ZI316 = lex_state->buf.a[1]; + ZI314 = lex_state->buf.a[1]; } else { - ZI316 = lex_state->buf.a[0]; + ZI314 = lex_state->buf.a[0]; } - ZI317 = lex_state->lx.start; - ZI318 = lex_state->lx.end; + ZI315 = lex_state->lx.start; + ZI316 = lex_state->lx.end; - (void) ZI317; - (void) ZI318; + (void) ZI315; + (void) ZI316; -#line 1773 "src/libre/dialect/pcre/parser.c" +#line 1765 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1786 "src/libre/dialect/pcre/parser.c" +#line 1778 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ - p_319 (flags, lex_state, act_state, err, &ZI316, &ZI317, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI314, &ZI315, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1821,14 +1813,14 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 703 "src/libre/parser.act" +#line 695 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; } goto ZL1; -#line 1832 "src/libre/dialect/pcre/parser.c" +#line 1824 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1855,7 +1847,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 667 "src/libre/parser.act" +#line 659 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1869,7 +1861,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1873 "src/libre/dialect/pcre/parser.c" +#line 1865 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1879,12 +1871,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 874 "src/libre/parser.act" +#line 856 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1888 "src/libre/dialect/pcre/parser.c" +#line 1880 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1909,43 +1901,42 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { t_pos ZIstart; t_ast__expr ZItmp; - t_pos ZIend; - /* BEGINNING OF INLINE: 181 */ + /* BEGINNING OF INLINE: 180 */ { switch (CURRENT_TERMINAL) { case (TOK_OPENGROUP): { - t_pos ZI182; + t_pos ZI181; /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 338 "src/libre/parser.act" +#line 330 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI182 = lex_state->lx.end; + ZI181 = lex_state->lx.end; (void) ZIstart; - (void) ZI182; + (void) ZI181; -#line 1932 "src/libre/dialect/pcre/parser.c" +#line 1923 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1945 "src/libre/dialect/pcre/parser.c" +#line 1936 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; - p_195 (flags, lex_state, act_state, err, &ZItmp); + p_194 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1955,60 +1946,60 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPCB): { - t_pos ZI201; + t_pos ZI200; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 354 "src/libre/parser.act" +#line 346 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI201 = lex_state->lx.end; + ZI200 = lex_state->lx.end; (void) ZIstart; - (void) ZI201; + (void) ZI200; -#line 1973 "src/libre/dialect/pcre/parser.c" +#line 1964 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1986 "src/libre/dialect/pcre/parser.c" +#line 1977 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 915 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1996 "src/libre/dialect/pcre/parser.c" +#line 1987 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2012 "src/libre/dialect/pcre/parser.c" +#line 2003 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2020,38 +2011,38 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINV): { - t_pos ZI193; + t_pos ZI192; /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 346 "src/libre/parser.act" +#line 338 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI193 = lex_state->lx.end; + ZI192 = lex_state->lx.end; (void) ZIstart; - (void) ZI193; + (void) ZI192; -#line 2036 "src/libre/dialect/pcre/parser.c" +#line 2027 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2049 "src/libre/dialect/pcre/parser.c" +#line 2040 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 996 "src/libre/parser.act" +#line 978 "src/libre/parser.act" struct ast_expr *any; @@ -2089,10 +2080,10 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2093 "src/libre/dialect/pcre/parser.c" +#line 2084 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ - p_195 (flags, lex_state, act_state, err, &ZItmp); + p_194 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -2102,40 +2093,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINVCB): { - t_pos ZI208; + t_pos ZI207; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 362 "src/libre/parser.act" +#line 354 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI208 = lex_state->lx.end; + ZI207 = lex_state->lx.end; (void) ZIstart; - (void) ZI208; + (void) ZI207; -#line 2120 "src/libre/dialect/pcre/parser.c" +#line 2111 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2133 "src/libre/dialect/pcre/parser.c" +#line 2124 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 996 "src/libre/parser.act" +#line 978 "src/libre/parser.act" struct ast_expr *any; @@ -2173,32 +2164,32 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2177 "src/libre/dialect/pcre/parser.c" +#line 2168 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 915 "src/libre/parser.act" +#line 897 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 2186 "src/libre/dialect/pcre/parser.c" +#line 2177 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2202 "src/libre/dialect/pcre/parser.c" +#line 2193 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2212,39 +2203,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } } - /* END OF INLINE: 181 */ - /* BEGINNING OF INLINE: 214 */ + /* END OF INLINE: 180 */ + /* BEGINNING OF INLINE: 213 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSEGROUP): { - t_char ZI215; - t_pos ZI216; + t_char ZI214; + t_pos ZI215; + t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 370 "src/libre/parser.act" +#line 362 "src/libre/parser.act" - ZI215 = ']'; - ZI216 = lex_state->lx.start; + ZI214 = ']'; + ZI215 = lex_state->lx.start; ZIend = lex_state->lx.end; + (void) ZI214; (void) ZI215; - (void) ZI216; (void) ZIend; -#line 2237 "src/libre/dialect/pcre/parser.c" +#line 2229 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 787 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2248 "src/libre/dialect/pcre/parser.c" +#line 2240 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2252,56 +2244,57 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUPRANGE): { t_char ZIcrange; - t_pos ZI218; + t_pos ZI217; + t_pos ZIend; t_ast__expr ZIrange; /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 380 "src/libre/parser.act" +#line 372 "src/libre/parser.act" ZIcrange = '-'; - ZI218 = lex_state->lx.start; + ZI217 = lex_state->lx.start; ZIend = lex_state->lx.end; (void) ZIcrange; - (void) ZI218; + (void) ZI217; (void) ZIend; -#line 2271 "src/libre/dialect/pcre/parser.c" +#line 2264 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 2284 "src/libre/dialect/pcre/parser.c" +#line 2277 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 2295 "src/libre/dialect/pcre/parser.c" +#line 2288 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 787 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2305 "src/libre/dialect/pcre/parser.c" +#line 2298 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2314,36 +2307,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 745 "src/libre/parser.act" +#line 737 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 2325 "src/libre/dialect/pcre/parser.c" +#line 2318 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ - ZIend = ZIstart; } ZL3:; } - /* END OF INLINE: 214 */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 802 "src/libre/parser.act" - - (void) (ZIstart); - (void) (ZIend); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 2345 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-expr */ + /* END OF INLINE: 213 */ } goto ZL0; ZL1:; @@ -2363,38 +2340,38 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st return; } { - /* BEGINNING OF INLINE: 162 */ + /* BEGINNING OF INLINE: 161 */ { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; - t_pos ZI164; + t_pos ZI163; /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" ZIc = '-'; - ZI164 = lex_state->lx.start; + ZI163 = lex_state->lx.start; ZIend = lex_state->lx.end; (void) ZIc; - (void) ZI164; + (void) ZI163; (void) ZIend; -#line 2387 "src/libre/dialect/pcre/parser.c" +#line 2364 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2398 "src/libre/dialect/pcre/parser.c" +#line 2375 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2402,9 +2379,9 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st case (TOK_NAMED__CLASS): case (TOK_ESC): case (TOK_CONTROL): case (TOK_OCT): case (TOK_HEX): case (TOK_CHAR): case (TOK_UNSUPPORTED): { - t_pos ZI163; + t_pos ZI162; - p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI163, &ZIend); + p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI162, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2415,7 +2392,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st goto ZL1; } } - /* END OF INLINE: 162 */ + /* END OF INLINE: 161 */ } goto ZL0; ZL1:; @@ -2427,7 +2404,7 @@ ZL0:; } static void -p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI316, t_pos *ZI317, t_ast__expr *ZOnode) +p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI314, t_pos *ZI315, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2436,14 +2413,14 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI316)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI314)); if ((ZInode) == NULL) { goto ZL1; } -#line 2447 "src/libre/dialect/pcre/parser.c" +#line 2424 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2456,15 +2433,15 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char) (*ZI316); + (ZIlower).u.literal.c = (unsigned char) (*ZI314); -#line 2465 "src/libre/dialect/pcre/parser.c" +#line 2442 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ - p_169 (flags, lex_state, act_state, err); + p_168 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -2472,17 +2449,17 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 792 "src/libre/parser.act" +#line 784 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI317)); + mark(&act_state->rangestart, &(*ZI315)); mark(&act_state->rangeend, &(ZIend)); -#line 2481 "src/libre/dialect/pcre/parser.c" +#line 2458 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -2511,7 +2488,7 @@ p_319(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2515 "src/libre/dialect/pcre/parser.c" +#line 2492 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2539,7 +2516,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr ZIe; p_expr_C_Cpiece_C_Catom (flags, lex_state, act_state, err, &ZIe); - /* BEGINNING OF INLINE: 267 */ + /* BEGINNING OF INLINE: 265 */ { switch (CURRENT_TERMINAL) { case (TOK_OPT): case (TOK_PLUS): case (TOK_STAR): case (TOK_OPENCOUNT): @@ -2557,16 +2534,16 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* BEGINNING OF ACTION: count-one */ { -#line 850 "src/libre/parser.act" +#line 832 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2565 "src/libre/dialect/pcre/parser.c" +#line 2542 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2580,7 +2557,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2584 "src/libre/dialect/pcre/parser.c" +#line 2561 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2590,7 +2567,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } } - /* END OF INLINE: 267 */ + /* END OF INLINE: 265 */ } goto ZL0; ZL1:; @@ -2600,6 +2577,86 @@ ZL0:; *ZOnode = ZInode; } +static void +p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI318, t_unsigned *ZIm, t_ast__count *ZOc) +{ + t_ast__count ZIc; + + switch (CURRENT_TERMINAL) { + case (TOK_CLOSECOUNT): + { + t_pos ZI256; + t_pos ZIend; + + /* BEGINNING OF EXTRACT: CLOSECOUNT */ + { +#line 390 "src/libre/parser.act" + + ZI256 = lex_state->lx.start; + ZIend = lex_state->lx.end; + + (void) ZI256; + (void) ZIend; + +#line 2602 "src/libre/dialect/pcre/parser.c" + } + /* END OF EXTRACT: CLOSECOUNT */ + ADVANCE_LEXER; + /* BEGINNING OF ACTION: mark-count */ + { +#line 789 "src/libre/parser.act" + + mark(&act_state->countstart, &(*ZI318)); + mark(&act_state->countend, &(ZIend)); + +#line 2613 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: mark-count */ + /* BEGINNING OF ACTION: count-range */ + { +#line 836 "src/libre/parser.act" + + if ((*ZIm) < (*ZIm)) { + err->e = RE_ENEGCOUNT; + err->m = (*ZIm); + err->n = (*ZIm); + + mark(&act_state->countstart, &(*ZI318)); + mark(&act_state->countend, &(ZIend)); + + goto ZL1; + } + + (ZIc) = ast_make_count((*ZIm), (*ZIm)); + +#line 2633 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: count-range */ + } + break; + case (TOK_SEP): + { + ADVANCE_LEXER; + p_321 (flags, lex_state, act_state, err, ZI318, ZIm, &ZIc); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + case (ERROR_TERMINAL): + return; + default: + goto ZL1; + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOc = ZIc; +} + static void p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { @@ -2611,14 +2668,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2622 "src/libre/dialect/pcre/parser.c" +#line 2679 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2632,26 +2689,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 731 "src/libre/parser.act" +#line 723 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2643 "src/libre/dialect/pcre/parser.c" +#line 2700 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2655 "src/libre/dialect/pcre/parser.c" +#line 2712 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2664,70 +2721,154 @@ ZL0:; } static void -p_322(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI320, t_unsigned *ZIm, t_ast__count *ZOc) +p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI318, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI258; + t_pos ZI261; t_pos ZIend; + t_unsigned ZIn; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 398 "src/libre/parser.act" +#line 390 "src/libre/parser.act" - ZI258 = lex_state->lx.start; + ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI258; + (void) ZI261; (void) ZIend; -#line 2688 "src/libre/dialect/pcre/parser.c" +#line 2746 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 797 "src/libre/parser.act" +#line 789 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI320)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2699 "src/libre/dialect/pcre/parser.c" +#line 2757 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ + /* BEGINNING OF ACTION: count-unbounded */ + { +#line 816 "src/libre/parser.act" + + (ZIn) = AST_COUNT_UNBOUNDED; + +#line 2766 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 854 "src/libre/parser.act" +#line 836 "src/libre/parser.act" - if ((*ZIm) < (*ZIm)) { + if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); - err->n = (*ZIm); + err->n = (ZIn); - mark(&act_state->countstart, &(*ZI320)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - (ZIc) = ast_make_count((*ZIm), (*ZIm)); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2719 "src/libre/dialect/pcre/parser.c" +#line 2786 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } break; - case (TOK_SEP): + case (TOK_COUNT): { + t_unsigned ZIn; + t_pos ZI259; + t_pos ZIend; + + /* BEGINNING OF EXTRACT: COUNT */ + { +#line 639 "src/libre/parser.act" + + unsigned long u; + char *e; + + u = strtoul(lex_state->buf.a, &e, 10); + + if ((u == ULONG_MAX && errno == ERANGE) || u > UINT_MAX) { + err->e = RE_ECOUNTRANGE; + snprintdots(err->esc, sizeof err->esc, lex_state->buf.a); + goto ZL1; + } + + if ((u == ULONG_MAX && errno != 0) || *e != '\0') { + err->e = RE_EXCOUNT; + goto ZL1; + } + + ZIn = (unsigned int) u; + +#line 2819 "src/libre/dialect/pcre/parser.c" + } + /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; - p_323 (flags, lex_state, act_state, err, ZI320, ZIm, &ZIc); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; + switch (CURRENT_TERMINAL) { + case (TOK_CLOSECOUNT): + /* BEGINNING OF EXTRACT: CLOSECOUNT */ + { +#line 390 "src/libre/parser.act" + + ZI259 = lex_state->lx.start; + ZIend = lex_state->lx.end; + + (void) ZI259; + (void) ZIend; + +#line 2835 "src/libre/dialect/pcre/parser.c" + } + /* END OF EXTRACT: CLOSECOUNT */ + break; + default: goto ZL1; } + ADVANCE_LEXER; + /* BEGINNING OF ACTION: mark-count */ + { +#line 789 "src/libre/parser.act" + + mark(&act_state->countstart, &(*ZI318)); + mark(&act_state->countend, &(ZIend)); + +#line 2850 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: mark-count */ + /* BEGINNING OF ACTION: count-range */ + { +#line 836 "src/libre/parser.act" + + if ((ZIn) < (*ZIm)) { + err->e = RE_ENEGCOUNT; + err->m = (*ZIm); + err->n = (ZIn); + + mark(&act_state->countstart, &(*ZI318)); + mark(&act_state->countend, &(ZIend)); + + goto ZL1; + } + + (ZIc) = ast_make_count((*ZIm), (ZIn)); + +#line 2870 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: count-range */ } break; case (ERROR_TERMINAL): @@ -2744,47 +2885,47 @@ ZL0:; } static void -p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) +p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; t_pos ZIrstart; - t_pos ZI196; + t_pos ZI195; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; - ZI196 = lex_state->lx.end; + ZI195 = lex_state->lx.end; (void) ZIc; (void) ZIrstart; - (void) ZI196; + (void) ZI195; -#line 2770 "src/libre/dialect/pcre/parser.c" +#line 2911 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; - /* BEGINNING OF INLINE: 197 */ + /* BEGINNING OF INLINE: 196 */ { switch (CURRENT_TERMINAL) { default: { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2788 "src/libre/dialect/pcre/parser.c" +#line 2929 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2792,35 +2933,35 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp case (TOK_RANGE): { t_endpoint ZIlower; - t_char ZI198; + t_char ZI197; + t_pos ZI198; t_pos ZI199; - t_pos ZI200; t_endpoint ZIupper; t_pos ZIend; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 2809 "src/libre/dialect/pcre/parser.c" +#line 2950 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI198 = '-'; - ZI199 = lex_state->lx.start; - ZI200 = lex_state->lx.end; + ZI197 = '-'; + ZI198 = lex_state->lx.start; + ZI199 = lex_state->lx.end; + (void) ZI197; (void) ZI198; (void) ZI199; - (void) ZI200; -#line 2824 "src/libre/dialect/pcre/parser.c" +#line 2965 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2831,7 +2972,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -2860,23 +3001,23 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 2864 "src/libre/dialect/pcre/parser.c" +#line 3005 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } break; } } - /* END OF INLINE: 197 */ + /* END OF INLINE: 196 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2880 "src/libre/dialect/pcre/parser.c" +#line 3021 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2892,170 +3033,6 @@ ZL1:; return; } -static void -p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI320, t_unsigned *ZIm, t_ast__count *ZOc) -{ - t_ast__count ZIc; - - switch (CURRENT_TERMINAL) { - case (TOK_CLOSECOUNT): - { - t_pos ZI263; - t_pos ZIend; - t_unsigned ZIn; - - /* BEGINNING OF EXTRACT: CLOSECOUNT */ - { -#line 398 "src/libre/parser.act" - - ZI263 = lex_state->lx.start; - ZIend = lex_state->lx.end; - - (void) ZI263; - (void) ZIend; - -#line 2918 "src/libre/dialect/pcre/parser.c" - } - /* END OF EXTRACT: CLOSECOUNT */ - ADVANCE_LEXER; - /* BEGINNING OF ACTION: mark-count */ - { -#line 797 "src/libre/parser.act" - - mark(&act_state->countstart, &(*ZI320)); - mark(&act_state->countend, &(ZIend)); - -#line 2929 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-count */ - /* BEGINNING OF ACTION: count-unbounded */ - { -#line 834 "src/libre/parser.act" - - (ZIn) = AST_COUNT_UNBOUNDED; - -#line 2938 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: count-unbounded */ - /* BEGINNING OF ACTION: count-range */ - { -#line 854 "src/libre/parser.act" - - if ((ZIn) < (*ZIm)) { - err->e = RE_ENEGCOUNT; - err->m = (*ZIm); - err->n = (ZIn); - - mark(&act_state->countstart, &(*ZI320)); - mark(&act_state->countend, &(ZIend)); - - goto ZL1; - } - - (ZIc) = ast_make_count((*ZIm), (ZIn)); - -#line 2958 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: count-range */ - } - break; - case (TOK_COUNT): - { - t_unsigned ZIn; - t_pos ZI261; - t_pos ZIend; - - /* BEGINNING OF EXTRACT: COUNT */ - { -#line 647 "src/libre/parser.act" - - unsigned long u; - char *e; - - u = strtoul(lex_state->buf.a, &e, 10); - - if ((u == ULONG_MAX && errno == ERANGE) || u > UINT_MAX) { - err->e = RE_ECOUNTRANGE; - snprintdots(err->esc, sizeof err->esc, lex_state->buf.a); - goto ZL1; - } - - if ((u == ULONG_MAX && errno != 0) || *e != '\0') { - err->e = RE_EXCOUNT; - goto ZL1; - } - - ZIn = (unsigned int) u; - -#line 2991 "src/libre/dialect/pcre/parser.c" - } - /* END OF EXTRACT: COUNT */ - ADVANCE_LEXER; - switch (CURRENT_TERMINAL) { - case (TOK_CLOSECOUNT): - /* BEGINNING OF EXTRACT: CLOSECOUNT */ - { -#line 398 "src/libre/parser.act" - - ZI261 = lex_state->lx.start; - ZIend = lex_state->lx.end; - - (void) ZI261; - (void) ZIend; - -#line 3007 "src/libre/dialect/pcre/parser.c" - } - /* END OF EXTRACT: CLOSECOUNT */ - break; - default: - goto ZL1; - } - ADVANCE_LEXER; - /* BEGINNING OF ACTION: mark-count */ - { -#line 797 "src/libre/parser.act" - - mark(&act_state->countstart, &(*ZI320)); - mark(&act_state->countend, &(ZIend)); - -#line 3022 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-count */ - /* BEGINNING OF ACTION: count-range */ - { -#line 854 "src/libre/parser.act" - - if ((ZIn) < (*ZIm)) { - err->e = RE_ENEGCOUNT; - err->m = (*ZIm); - err->n = (ZIn); - - mark(&act_state->countstart, &(*ZI320)); - mark(&act_state->countend, &(ZIend)); - - goto ZL1; - } - - (ZIc) = ast_make_count((*ZIm), (ZIn)); - -#line 3042 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: count-range */ - } - break; - case (ERROR_TERMINAL): - return; - default: - goto ZL1; - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOc = ZIc; -} - static void p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { @@ -3079,23 +3056,23 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 826 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; -#line 3087 "src/libre/dialect/pcre/parser.c" +#line 3064 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 826 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; -#line 3096 "src/libre/dialect/pcre/parser.c" +#line 3073 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ - /* BEGINNING OF INLINE: 233 */ + /* BEGINNING OF INLINE: 231 */ { switch (CURRENT_TERMINAL) { case (TOK_FLAG__IGNORE): case (TOK_FLAG__UNKNOWN): case (TOK_FLAG__INSENSITIVE): case (TOK_FLAG__EXTENDED): @@ -3115,8 +3092,8 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, break; } } - /* END OF INLINE: 233 */ - /* BEGINNING OF INLINE: 235 */ + /* END OF INLINE: 231 */ + /* BEGINNING OF INLINE: 233 */ { switch (CURRENT_TERMINAL) { case (TOK_NEGATE): @@ -3136,8 +3113,8 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, break; } } - /* END OF INLINE: 235 */ - /* BEGINNING OF INLINE: 238 */ + /* END OF INLINE: 233 */ + /* BEGINNING OF INLINE: 236 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSE): @@ -3145,7 +3122,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 956 "src/libre/parser.act" +#line 938 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3154,19 +3131,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3158 "src/libre/dialect/pcre/parser.c" +#line 3135 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL5; } -#line 3170 "src/libre/dialect/pcre/parser.c" +#line 3147 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3179,16 +3156,16 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 948 "src/libre/parser.act" +#line 930 "src/libre/parser.act" (ZIflags) = *flags; -#line 3187 "src/libre/dialect/pcre/parser.c" +#line 3164 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 956 "src/libre/parser.act" +#line 938 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3197,7 +3174,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3201 "src/libre/dialect/pcre/parser.c" +#line 3178 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3207,11 +3184,11 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 952 "src/libre/parser.act" +#line 934 "src/libre/parser.act" *flags = (ZIflags); -#line 3215 "src/libre/dialect/pcre/parser.c" +#line 3192 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3232,32 +3209,32 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 766 "src/libre/parser.act" +#line 758 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; } goto ZL1; -#line 3243 "src/libre/dialect/pcre/parser.c" +#line 3220 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 3255 "src/libre/dialect/pcre/parser.c" +#line 3232 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } ZL4:; } - /* END OF INLINE: 238 */ + /* END OF INLINE: 236 */ } goto ZL0; ZL1:; @@ -3285,7 +3262,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3299,10 +3276,10 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3303 "src/libre/dialect/pcre/parser.c" +#line 3280 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ - /* BEGINNING OF INLINE: 266 */ + /* BEGINNING OF INLINE: 264 */ { switch (CURRENT_TERMINAL) { case (TOK_OPT): @@ -3310,14 +3287,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3321 "src/libre/dialect/pcre/parser.c" +#line 3298 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3327,14 +3304,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 780 "src/libre/parser.act" +#line 772 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3338 "src/libre/dialect/pcre/parser.c" +#line 3315 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3343,7 +3320,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state break; } } - /* END OF INLINE: 266 */ + /* END OF INLINE: 264 */ } goto ZL0; ZL1:; @@ -3364,7 +3341,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, return; } { - /* BEGINNING OF INLINE: 159 */ + /* BEGINNING OF INLINE: 158 */ { switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): @@ -3390,7 +3367,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, goto ZL1; } } - /* END OF INLINE: 159 */ + /* END OF INLINE: 158 */ } goto ZL0; ZL1:; @@ -3419,7 +3396,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 667 "src/libre/parser.act" +#line 659 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3433,7 +3410,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3437 "src/libre/dialect/pcre/parser.c" +#line 3414 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3443,14 +3420,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3454 "src/libre/dialect/pcre/parser.c" +#line 3431 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3465,7 +3442,7 @@ ZL0:; } static void -p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) +p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) { t_ast__expr ZInode1; @@ -3474,14 +3451,14 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3485 "src/libre/dialect/pcre/parser.c" +#line 3462 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3489,36 +3466,36 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs case (TOK_RANGE): { t_endpoint ZIr; - t_char ZI211; + t_char ZI210; + t_pos ZI211; t_pos ZI212; - t_pos ZI213; t_endpoint ZIupper; t_pos ZIend; t_endpoint ZIlower; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3507 "src/libre/dialect/pcre/parser.c" +#line 3484 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI211 = '-'; - ZI212 = lex_state->lx.start; - ZI213 = lex_state->lx.end; + ZI210 = '-'; + ZI211 = lex_state->lx.start; + ZI212 = lex_state->lx.end; + (void) ZI210; (void) ZI211; (void) ZI212; - (void) ZI213; -#line 3522 "src/libre/dialect/pcre/parser.c" +#line 3499 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3529,17 +3506,17 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3538 "src/libre/dialect/pcre/parser.c" +#line 3515 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -3568,7 +3545,7 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3572 "src/libre/dialect/pcre/parser.c" +#line 3549 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3601,16 +3578,16 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3611 "src/libre/dialect/pcre/parser.c" +#line 3588 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF INLINE: 280 */ + /* BEGINNING OF INLINE: 278 */ { switch (CURRENT_TERMINAL) { case (TOK_ALT): @@ -3625,21 +3602,21 @@ ZL2_expr_C_Clist_Hof_Halts:; break; } } - /* END OF INLINE: 280 */ + /* END OF INLINE: 278 */ } return; ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 731 "src/libre/parser.act" +#line 723 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 3643 "src/libre/dialect/pcre/parser.c" +#line 3620 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3658,21 +3635,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { - t_pos ZI320; - t_pos ZI321; + t_pos ZI318; + t_pos ZI319; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 390 "src/libre/parser.act" +#line 382 "src/libre/parser.act" - ZI320 = lex_state->lx.start; - ZI321 = lex_state->lx.end; + ZI318 = lex_state->lx.start; + ZI319 = lex_state->lx.end; - (void) ZI320; - (void) ZI321; + (void) ZI318; + (void) ZI319; -#line 3676 "src/libre/dialect/pcre/parser.c" +#line 3653 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3680,7 +3657,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 647 "src/libre/parser.act" +#line 639 "src/libre/parser.act" unsigned long u; char *e; @@ -3700,7 +3677,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3704 "src/libre/dialect/pcre/parser.c" +#line 3681 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3708,7 +3685,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_322 (flags, lex_state, act_state, err, &ZI320, &ZIm, &ZIc); + p_320 (flags, lex_state, act_state, err, &ZI318, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -3720,11 +3697,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 846 "src/libre/parser.act" +#line 828 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 3728 "src/libre/dialect/pcre/parser.c" +#line 3705 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3734,11 +3711,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 842 "src/libre/parser.act" +#line 824 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3742 "src/libre/dialect/pcre/parser.c" +#line 3719 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3748,11 +3725,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 838 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3756 "src/libre/dialect/pcre/parser.c" +#line 3733 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3767,23 +3744,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 717 "src/libre/parser.act" +#line 709 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 3778 "src/libre/dialect/pcre/parser.c" +#line 3755 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 850 "src/libre/parser.act" +#line 832 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 3787 "src/libre/dialect/pcre/parser.c" +#line 3764 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3809,11 +3786,11 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3817 "src/libre/dialect/pcre/parser.c" +#line 3794 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3823,17 +3800,17 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3834 "src/libre/dialect/pcre/parser.c" +#line 3811 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ - /* BEGINNING OF INLINE: 282 */ + /* BEGINNING OF INLINE: 280 */ { { switch (CURRENT_TERMINAL) { @@ -3849,20 +3826,20 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 773 "src/libre/parser.act" +#line 765 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 3860 "src/libre/dialect/pcre/parser.c" +#line 3837 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL2:; } - /* END OF INLINE: 282 */ + /* END OF INLINE: 280 */ } goto ZL0; ZL1:; @@ -3885,24 +3862,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3894 "src/libre/dialect/pcre/parser.c" +#line 3871 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3906 "src/libre/dialect/pcre/parser.c" +#line 3883 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3912,14 +3889,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 972 "src/libre/parser.act" +#line 954 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 3923 "src/libre/dialect/pcre/parser.c" +#line 3900 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3929,7 +3906,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 979 "src/libre/parser.act" +#line 961 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3939,7 +3916,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3943 "src/libre/dialect/pcre/parser.c" +#line 3920 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3957,134 +3934,134 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 817 "src/libre/parser.act" +#line 799 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3966 "src/libre/dialect/pcre/parser.c" +#line 3943 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 3978 "src/libre/dialect/pcre/parser.c" +#line 3955 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { goto ZL1; } -#line 3990 "src/libre/dialect/pcre/parser.c" +#line 3967 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 919 "src/libre/parser.act" +#line 901 "src/libre/parser.act" (ZIcr) = '\r'; -#line 3999 "src/libre/dialect/pcre/parser.c" +#line 3976 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { goto ZL1; } -#line 4011 "src/libre/dialect/pcre/parser.c" +#line 3988 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 923 "src/libre/parser.act" +#line 905 "src/libre/parser.act" (ZInl) = '\n'; -#line 4020 "src/libre/dialect/pcre/parser.c" +#line 3997 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { goto ZL1; } -#line 4032 "src/libre/dialect/pcre/parser.c" +#line 4009 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4043 "src/libre/dialect/pcre/parser.c" +#line 4020 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4054 "src/libre/dialect/pcre/parser.c" +#line 4031 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 4066 "src/libre/dialect/pcre/parser.c" +#line 4043 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4077 "src/libre/dialect/pcre/parser.c" +#line 4054 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4088 "src/libre/dialect/pcre/parser.c" +#line 4065 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4098,20 +4075,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 948 "src/libre/parser.act" +#line 930 "src/libre/parser.act" (ZIflags) = *flags; -#line 4106 "src/libre/dialect/pcre/parser.c" +#line 4083 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 4115 "src/libre/dialect/pcre/parser.c" +#line 4092 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4121,23 +4098,23 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 952 "src/libre/parser.act" +#line 934 "src/libre/parser.act" *flags = (ZIflags); -#line 4129 "src/libre/dialect/pcre/parser.c" +#line 4106 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 4141 "src/libre/dialect/pcre/parser.c" +#line 4118 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4154,14 +4131,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 965 "src/libre/parser.act" +#line 947 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 4165 "src/libre/dialect/pcre/parser.c" +#line 4142 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4213,26 +4190,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 724 "src/libre/parser.act" +#line 716 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 4224 "src/libre/dialect/pcre/parser.c" +#line 4201 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 4236 "src/libre/dialect/pcre/parser.c" +#line 4213 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4258,14 +4235,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4269 "src/libre/dialect/pcre/parser.c" +#line 4246 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4279,14 +4256,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4290 "src/libre/dialect/pcre/parser.c" +#line 4267 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4322,42 +4299,27 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4333 "src/libre/dialect/pcre/parser.c" +#line 4310 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4344 "src/libre/dialect/pcre/parser.c" +#line 4321 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 802 "src/libre/parser.act" - - (void) (ZIstart); - (void) (ZIend); - -/* TODO: reinstate this, applies to an expr node in general - (ZInode)->u.class.start = ast_start; - (ZInode)->u.class.end = ast_end; -*/ - -#line 4359 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-expr */ } goto ZL0; ZL1:; @@ -4369,7 +4331,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1082 "src/libre/parser.act" +#line 1064 "src/libre/parser.act" static int @@ -4527,6 +4489,6 @@ ZL0:; return NULL; } -#line 4531 "src/libre/dialect/pcre/parser.c" +#line 4493 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 733754335..46c464b38 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 301 "src/libre/parser.act" +#line 293 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1239 "src/libre/parser.act" +#line 1221 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/pcre/parser.sid b/src/libre/dialect/pcre/parser.sid index 3a1d34cbc..377060dbe 100644 --- a/src/libre/dialect/pcre/parser.sid +++ b/src/libre/dialect/pcre/parser.sid @@ -140,7 +140,6 @@ : (:pos, :pos) -> (); : (:pos, :pos) -> (); : (:pos, :pos) -> (); - : (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); : () -> (:ast_class_id); @@ -378,10 +377,7 @@ (start, end); ## ; - end = start; /* appease sid */ }; - - (tmp, start, end); }; type: () -> (node :ast_expr) = { @@ -389,8 +385,6 @@ node = ; (node, class); - - (node, start, end); }; flags: () -> (node :ast_expr) [ diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index 4d30cdf44..0a3df7280 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -289,7 +281,7 @@ return s; } -#line 293 "src/libre/dialect/sql/parser.c" +#line 285 "src/libre/dialect/sql/parser.c" #ifndef ERROR_TERMINAL @@ -307,9 +299,9 @@ static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_stat static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); -static void p_205(flags, lex_state, act_state, err, t_ast__expr *); -static void p_209(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); -static void p_212(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_204(flags, lex_state, act_state, err, t_ast__expr *); +static void p_208(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_211(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); static void p_expr_C_Cpiece_C_Catom(flags, lex_state, act_state, err, t_ast__expr *); @@ -327,21 +319,21 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_INVERT): { - t_char ZI204; + t_char ZI203; /* BEGINNING OF EXTRACT: INVERT */ { -#line 322 "src/libre/parser.act" +#line 314 "src/libre/parser.act" - ZI204 = '^'; + ZI203 = '^'; - (void) ZI204; + (void) ZI203; -#line 341 "src/libre/dialect/sql/parser.c" +#line 333 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; - p_205 (flags, lex_state, act_state, err, ZIclass); + p_204 (flags, lex_state, act_state, err, ZIclass); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -351,47 +343,47 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ case (TOK_RANGE): { t_char ZIc; + t_pos ZI114; t_pos ZI115; - t_pos ZI116; t_ast__expr ZInode; /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" ZIc = '-'; - ZI115 = lex_state->lx.start; - ZI116 = lex_state->lx.end; + ZI114 = lex_state->lx.start; + ZI115 = lex_state->lx.end; (void) ZIc; + (void) ZI114; (void) ZI115; - (void) ZI116; -#line 371 "src/libre/dialect/sql/parser.c" +#line 363 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 384 "src/libre/dialect/sql/parser.c" +#line 376 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 395 "src/libre/dialect/sql/parser.c" +#line 387 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -421,11 +413,11 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 429 "src/libre/dialect/sql/parser.c" +#line 421 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -435,17 +427,17 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 446 "src/libre/dialect/sql/parser.c" +#line 438 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ - /* BEGINNING OF INLINE: 192 */ + /* BEGINNING OF INLINE: 191 */ { { switch (CURRENT_TERMINAL) { @@ -461,20 +453,20 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 773 "src/libre/parser.act" +#line 765 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 472 "src/libre/dialect/sql/parser.c" +#line 464 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL2:; } - /* END OF INLINE: 192 */ + /* END OF INLINE: 191 */ } goto ZL0; ZL1:; @@ -492,7 +484,7 @@ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags flags, lex_state lex_ } ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { - /* BEGINNING OF INLINE: 148 */ + /* BEGINNING OF INLINE: 147 */ { { t_ast__expr ZInode; @@ -504,13 +496,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; } -#line 514 "src/libre/dialect/sql/parser.c" +#line 506 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -519,21 +511,21 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 710 "src/libre/parser.act" +#line 702 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; } goto ZL1; -#line 530 "src/libre/dialect/sql/parser.c" +#line 522 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } ZL3:; } - /* END OF INLINE: 148 */ - /* BEGINNING OF INLINE: 149 */ + /* END OF INLINE: 147 */ + /* BEGINNING OF INLINE: 148 */ { switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): case (TOK_CHAR): @@ -547,7 +539,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; break; } } - /* END OF INLINE: 149 */ + /* END OF INLINE: 148 */ } return; ZL1:; @@ -580,16 +572,16 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1070 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 590 "src/libre/dialect/sql/parser.c" +#line 582 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 184 */ + /* BEGINNING OF INLINE: 183 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_OPENSUB): case (TOK_OPENGROUP): @@ -604,7 +596,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; break; } } - /* END OF INLINE: 184 */ + /* END OF INLINE: 183 */ } return; ZL1:; @@ -620,30 +612,30 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI206; + t_char ZI205; + t_pos ZI206; t_pos ZI207; - t_pos ZI208; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI207 = lex_state->lx.start; - ZI208 = lex_state->lx.end; + ZI206 = lex_state->lx.start; + ZI207 = lex_state->lx.end; + (void) ZI206; (void) ZI207; - (void) ZI208; - ZI206 = lex_state->buf.a[0]; + ZI205 = lex_state->buf.a[0]; -#line 643 "src/libre/dialect/sql/parser.c" +#line 635 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_209 (flags, lex_state, act_state, err, &ZI206, &ZI207, &ZInode); + p_208 (flags, lex_state, act_state, err, &ZI205, &ZI206, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -690,7 +682,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 338 "src/libre/parser.act" +#line 330 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -698,7 +690,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 702 "src/libre/dialect/sql/parser.c" +#line 694 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -708,115 +700,100 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { goto ZL1; } -#line 719 "src/libre/dialect/sql/parser.c" +#line 711 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead (flags, lex_state, act_state, err, &ZIclass); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); p_expr_C_Ccharacter_Hclass_C_Cclass_Htail (flags, lex_state, act_state, err, ZItmp); - /* BEGINNING OF INLINE: 154 */ + /* BEGINNING OF INLINE: 153 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSEGROUP): { - t_char ZI155; + t_char ZI154; t_pos ZIclose__start; t_pos ZIclose__end; /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 370 "src/libre/parser.act" +#line 362 "src/libre/parser.act" - ZI155 = ']'; + ZI154 = ']'; ZIclose__start = lex_state->lx.start; ZIclose__end = lex_state->lx.end; - (void) ZI155; + (void) ZI154; (void) ZIclose__start; (void) ZIclose__end; -#line 747 "src/libre/dialect/sql/parser.c" +#line 739 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 787 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); - mark(&act_state->groupend, &(ZIopen__end)); + mark(&act_state->groupend, &(ZIclose__end)); -#line 758 "src/libre/dialect/sql/parser.c" +#line 750 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 802 "src/libre/parser.act" - - (void) (ZIopen__start); - (void) (ZIclose__end); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 773 "src/libre/dialect/sql/parser.c" - } - /* END OF ACTION: mark-expr */ ZInode = ZIclass; } break; case (TOK_INVERT): { - t_char ZI159; + t_char ZI158; t_ast__expr ZImask; t_ast__expr ZImask__tmp; /* BEGINNING OF EXTRACT: INVERT */ { -#line 322 "src/libre/parser.act" +#line 314 "src/libre/parser.act" - ZI159 = '^'; + ZI158 = '^'; - (void) ZI159; + (void) ZI158; -#line 793 "src/libre/dialect/sql/parser.c" +#line 770 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { goto ZL3; } -#line 806 "src/libre/dialect/sql/parser.c" +#line 783 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead (flags, lex_state, act_state, err, &ZImask); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZImask__tmp); p_expr_C_Ccharacter_Hclass_C_Cclass_Htail (flags, lex_state, act_state, err, ZImask__tmp); - /* BEGINNING OF INLINE: 163 */ + /* BEGINNING OF INLINE: 162 */ { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL3; } { - t_char ZI164; + t_char ZI163; t_pos ZIclose__start; t_pos ZIclose__end; @@ -824,17 +801,17 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 370 "src/libre/parser.act" +#line 362 "src/libre/parser.act" - ZI164 = ']'; + ZI163 = ']'; ZIclose__start = lex_state->lx.start; ZIclose__end = lex_state->lx.end; - (void) ZI164; + (void) ZI163; (void) ZIclose__start; (void) ZIclose__end; -#line 838 "src/libre/dialect/sql/parser.c" +#line 815 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -844,74 +821,44 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 787 "src/libre/parser.act" +#line 779 "src/libre/parser.act" - mark(&act_state->groupstart, &(ZIclose__start)); + mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 853 "src/libre/dialect/sql/parser.c" +#line 830 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 802 "src/libre/parser.act" - - (void) (ZIopen__start); - (void) (ZIclose__end); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 868 "src/libre/dialect/sql/parser.c" - } - /* END OF ACTION: mark-expr */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 802 "src/libre/parser.act" - - (void) (ZIopen__start); - (void) (ZIclose__end); - -/* TODO: reinstate this, applies to an expr node in general - (ZImask__tmp)->u.class.start = ast_start; - (ZImask__tmp)->u.class.end = ast_end; -*/ - -#line 883 "src/libre/dialect/sql/parser.c" - } - /* END OF ACTION: mark-expr */ } goto ZL4; ZL5:; { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 745 "src/libre/parser.act" +#line 737 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL3; -#line 899 "src/libre/dialect/sql/parser.c" +#line 846 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } ZL4:; } - /* END OF INLINE: 163 */ + /* END OF INLINE: 162 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 989 "src/libre/parser.act" +#line 971 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { goto ZL3; } -#line 915 "src/libre/dialect/sql/parser.c" +#line 862 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -927,32 +874,32 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 745 "src/libre/parser.act" +#line 737 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 938 "src/libre/dialect/sql/parser.c" +#line 885 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 950 "src/libre/dialect/sql/parser.c" +#line 897 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } ZL2:; } - /* END OF INLINE: 154 */ + /* END OF INLINE: 153 */ } goto ZL0; ZL1:; @@ -982,7 +929,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -996,7 +943,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 1000 "src/libre/dialect/sql/parser.c" +#line 947 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1019,14 +966,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 897 "src/libre/parser.act" +#line 879 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1030 "src/libre/dialect/sql/parser.c" +#line 977 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -1040,26 +987,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 731 "src/libre/parser.act" +#line 723 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 1051 "src/libre/dialect/sql/parser.c" +#line 998 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 1063 "src/libre/dialect/sql/parser.c" +#line 1010 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1072,58 +1019,58 @@ ZL0:; } static void -p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZIclass) +p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZIclass) { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; + t_pos ZI117; t_pos ZI118; - t_pos ZI119; t_ast__expr ZInode; /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" ZIc = '-'; - ZI118 = lex_state->lx.start; - ZI119 = lex_state->lx.end; + ZI117 = lex_state->lx.start; + ZI118 = lex_state->lx.end; (void) ZIc; + (void) ZI117; (void) ZI118; - (void) ZI119; -#line 1098 "src/libre/dialect/sql/parser.c" +#line 1045 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1111 "src/libre/dialect/sql/parser.c" +#line 1058 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 1122 "src/libre/dialect/sql/parser.c" +#line 1069 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 996 "src/libre/parser.act" +#line 978 "src/libre/parser.act" struct ast_expr *any; @@ -1161,7 +1108,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1165 "src/libre/dialect/sql/parser.c" +#line 1112 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1170,7 +1117,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 996 "src/libre/parser.act" +#line 978 "src/libre/parser.act" struct ast_expr *any; @@ -1208,7 +1155,7 @@ p_205(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1212 "src/libre/dialect/sql/parser.c" +#line 1159 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1223,7 +1170,7 @@ ZL1:; } static void -p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI206, t_pos *ZI207, t_ast__expr *ZOnode) +p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI205, t_pos *ZI206, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -1232,14 +1179,14 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI206)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI205)); if ((ZInode) == NULL) { goto ZL1; } -#line 1243 "src/libre/dialect/sql/parser.c" +#line 1190 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1247,37 +1194,37 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_RANGE): { t_endpoint ZIa; - t_char ZI137; + t_char ZI136; + t_pos ZI137; t_pos ZI138; - t_pos ZI139; t_char ZIcz; - t_pos ZI141; + t_pos ZI140; t_pos ZIend; t_endpoint ZIz; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; - (ZIa).u.literal.c = (unsigned char) (*ZI206); + (ZIa).u.literal.c = (unsigned char) (*ZI205); -#line 1266 "src/libre/dialect/sql/parser.c" +#line 1213 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 328 "src/libre/parser.act" +#line 320 "src/libre/parser.act" - ZI137 = '-'; - ZI138 = lex_state->lx.start; - ZI139 = lex_state->lx.end; + ZI136 = '-'; + ZI137 = lex_state->lx.start; + ZI138 = lex_state->lx.end; + (void) ZI136; (void) ZI137; (void) ZI138; - (void) ZI139; -#line 1281 "src/libre/dialect/sql/parser.c" +#line 1228 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1285,20 +1232,20 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI141 = lex_state->lx.start; + ZI140 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI141; + (void) ZI140; (void) ZIend; ZIcz = lex_state->buf.a[0]; -#line 1302 "src/libre/dialect/sql/parser.c" +#line 1249 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1308,27 +1255,27 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 869 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char) (ZIcz); -#line 1317 "src/libre/dialect/sql/parser.c" +#line 1264 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 792 "src/libre/parser.act" +#line 784 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI207)); + mark(&act_state->rangestart, &(*ZI206)); mark(&act_state->rangeend, &(ZIend)); -#line 1327 "src/libre/dialect/sql/parser.c" +#line 1274 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1034 "src/libre/parser.act" +#line 1016 "src/libre/parser.act" unsigned char lower, upper; @@ -1357,7 +1304,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1361 "src/libre/dialect/sql/parser.c" +#line 1308 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1374,50 +1321,50 @@ ZL0:; } static void -p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI210, t_unsigned *ZIm, t_ast__count *ZOc) +p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI209, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI177; + t_pos ZI176; t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 398 "src/libre/parser.act" +#line 390 "src/libre/parser.act" - ZI177 = lex_state->lx.start; + ZI176 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI177; + (void) ZI176; (void) ZIend; -#line 1398 "src/libre/dialect/sql/parser.c" +#line 1345 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 797 "src/libre/parser.act" +#line 789 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI210)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1409 "src/libre/dialect/sql/parser.c" +#line 1356 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 854 "src/libre/parser.act" +#line 836 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI210)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); goto ZL1; @@ -1425,7 +1372,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1429 "src/libre/dialect/sql/parser.c" +#line 1376 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1433,7 +1380,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_SEP): { t_unsigned ZIn; - t_pos ZI180; + t_pos ZI179; t_pos ZIend; ADVANCE_LEXER; @@ -1441,7 +1388,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 647 "src/libre/parser.act" +#line 639 "src/libre/parser.act" unsigned long u; char *e; @@ -1461,7 +1408,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1465 "src/libre/dialect/sql/parser.c" +#line 1412 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1473,15 +1420,15 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 398 "src/libre/parser.act" +#line 390 "src/libre/parser.act" - ZI180 = lex_state->lx.start; + ZI179 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI180; + (void) ZI179; (void) ZIend; -#line 1485 "src/libre/dialect/sql/parser.c" +#line 1432 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1491,24 +1438,24 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 797 "src/libre/parser.act" +#line 789 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI210)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1500 "src/libre/dialect/sql/parser.c" +#line 1447 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 854 "src/libre/parser.act" +#line 836 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI210)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); goto ZL1; @@ -1516,7 +1463,7 @@ p_212(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1520 "src/libre/dialect/sql/parser.c" +#line 1467 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1551,16 +1498,16 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1076 "src/libre/parser.act" +#line 1058 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1561 "src/libre/dialect/sql/parser.c" +#line 1508 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF INLINE: 190 */ + /* BEGINNING OF INLINE: 189 */ { switch (CURRENT_TERMINAL) { case (TOK_ALT): @@ -1575,21 +1522,21 @@ ZL2_expr_C_Clist_Hof_Halts:; break; } } - /* END OF INLINE: 190 */ + /* END OF INLINE: 189 */ } return; ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 731 "src/libre/parser.act" +#line 723 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 1593 "src/libre/dialect/sql/parser.c" +#line 1540 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1608,21 +1555,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { + t_pos ZI209; t_pos ZI210; - t_pos ZI211; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 390 "src/libre/parser.act" +#line 382 "src/libre/parser.act" - ZI210 = lex_state->lx.start; - ZI211 = lex_state->lx.end; + ZI209 = lex_state->lx.start; + ZI210 = lex_state->lx.end; + (void) ZI209; (void) ZI210; - (void) ZI211; -#line 1626 "src/libre/dialect/sql/parser.c" +#line 1573 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1630,7 +1577,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 647 "src/libre/parser.act" +#line 639 "src/libre/parser.act" unsigned long u; char *e; @@ -1650,7 +1597,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1654 "src/libre/dialect/sql/parser.c" +#line 1601 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1658,7 +1605,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_212 (flags, lex_state, act_state, err, &ZI210, &ZIm, &ZIc); + p_211 (flags, lex_state, act_state, err, &ZI209, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1670,11 +1617,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 846 "src/libre/parser.act" +#line 828 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 1678 "src/libre/dialect/sql/parser.c" +#line 1625 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1684,11 +1631,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 842 "src/libre/parser.act" +#line 824 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1692 "src/libre/dialect/sql/parser.c" +#line 1639 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1698,11 +1645,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 838 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1706 "src/libre/dialect/sql/parser.c" +#line 1653 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1711,11 +1658,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 850 "src/libre/parser.act" +#line 832 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1719 "src/libre/dialect/sql/parser.c" +#line 1666 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1728,23 +1675,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 717 "src/libre/parser.act" +#line 709 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 1739 "src/libre/dialect/sql/parser.c" +#line 1686 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 850 "src/libre/parser.act" +#line 832 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1748 "src/libre/dialect/sql/parser.c" +#line 1695 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1769,24 +1716,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1778 "src/libre/dialect/sql/parser.c" +#line 1725 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1790 "src/libre/dialect/sql/parser.c" +#line 1737 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1794,38 +1741,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e case (TOK_CHAR): { t_char ZIa; + t_pos ZI170; t_pos ZI171; - t_pos ZI172; /* BEGINNING OF EXTRACT: CHAR */ { -#line 595 "src/libre/parser.act" +#line 587 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI171 = lex_state->lx.start; - ZI172 = lex_state->lx.end; + ZI170 = lex_state->lx.start; + ZI171 = lex_state->lx.end; + (void) ZI170; (void) ZI171; - (void) ZI172; ZIa = lex_state->buf.a[0]; -#line 1816 "src/libre/dialect/sql/parser.c" +#line 1763 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 904 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1829 "src/libre/dialect/sql/parser.c" +#line 1776 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1839,38 +1786,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 812 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1848 "src/libre/dialect/sql/parser.c" +#line 1795 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1860 "src/libre/dialect/sql/parser.c" +#line 1807 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 838 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1869 "src/libre/dialect/sql/parser.c" +#line 1816 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 927 "src/libre/parser.act" +#line 909 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1884,7 +1831,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1888 "src/libre/dialect/sql/parser.c" +#line 1835 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1897,11 +1844,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 911 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 1905 "src/libre/dialect/sql/parser.c" +#line 1852 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1911,14 +1858,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 941 "src/libre/parser.act" +#line 923 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 1922 "src/libre/dialect/sql/parser.c" +#line 1869 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1949,26 +1896,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 724 "src/libre/parser.act" +#line 716 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 1960 "src/libre/dialect/sql/parser.c" +#line 1907 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 1972 "src/libre/dialect/sql/parser.c" +#line 1919 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1990,14 +1937,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act } { t_ast__class__id ZIid; + t_pos ZI128; t_pos ZI129; - t_pos ZI130; switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 667 "src/libre/parser.act" +#line 659 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -2005,13 +1952,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act goto ZL1; } - ZI129 = lex_state->lx.start; - ZI130 = lex_state->lx.end; + ZI128 = lex_state->lx.start; + ZI129 = lex_state->lx.end; + (void) ZI128; (void) ZI129; - (void) ZI130; -#line 2015 "src/libre/dialect/sql/parser.c" +#line 1962 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2021,14 +1968,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1063 "src/libre/parser.act" +#line 1045 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2032 "src/libre/dialect/sql/parser.c" +#line 1979 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2051,14 +1998,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 890 "src/libre/parser.act" +#line 872 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2062 "src/libre/dialect/sql/parser.c" +#line 2009 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2072,14 +2019,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 883 "src/libre/parser.act" +#line 865 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2083 "src/libre/dialect/sql/parser.c" +#line 2030 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2097,7 +2044,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1082 "src/libre/parser.act" +#line 1064 "src/libre/parser.act" static int @@ -2255,6 +2202,6 @@ ZL0:; return NULL; } -#line 2259 "src/libre/dialect/sql/parser.c" +#line 2206 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 1ca760dd3..9433c21e9 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 301 "src/libre/parser.act" +#line 293 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1239 "src/libre/parser.act" +#line 1221 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/dialect/sql/parser.sid b/src/libre/dialect/sql/parser.sid index cc54ffecd..f8609c6f5 100644 --- a/src/libre/dialect/sql/parser.sid +++ b/src/libre/dialect/sql/parser.sid @@ -139,7 +139,6 @@ : (:pos, :pos) -> (); : (:pos, :pos) -> (); : (:pos, :pos) -> (); - : (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -253,9 +252,7 @@ { (!, close_start, close_end) = CLOSEGROUP; - (open_start, open_end); - - (tmp, open_start, close_end); + (open_start, close_end); node = class; || @@ -287,11 +284,8 @@ { (!, close_start, close_end) = CLOSEGROUP; - (close_start, close_end); - /* TODO: would prefer to demark class and mask separately */ - (tmp, open_start, close_end); - (mask_tmp, open_start, close_end); + (open_start, close_end); ## ; }; diff --git a/src/libre/parser.act b/src/libre/parser.act index c22197dcd..418d01469 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -77,14 +77,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -799,16 +791,6 @@ mark(&act_state->countend, &@end); @}; - : (node :ast_expr, start :pos, end :pos) -> () = @{ - (void) @start; - (void) @end; - -/* TODO: reinstate this, applies to an expr node in general - @node->u.class.start = ast_start; - @node->u.class.end = ast_end; -*/ - @}; - : () -> (c :ast_class_id) = @{ /* TODO: or the unicode equivalent */ @c = (*flags & RE_SINGLE) ? &class_any : &class_notnl; From 94ffdcceb38b6202e912e3df512f8fcf4d085a49 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 29 May 2024 15:48:00 -0400 Subject: [PATCH 110/373] Add fsm_vacuum, which reduces the state array when over-allocated. --- include/fsm/fsm.h | 9 +++++++++ src/libfsm/Makefile | 1 + src/libfsm/fsm.c | 2 +- src/libfsm/internal.h | 3 +++ src/libfsm/libfsm.syms | 1 + src/libfsm/vacuum.c | 40 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 src/libfsm/vacuum.c diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 2b4c438f1..a5e6ef388 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -493,5 +493,14 @@ int fsm_fgetc(void *opaque); /* expects opaque to be FILE * */ int fsm_shuffle(struct fsm *fsm, unsigned seed); +/* Attempt to reclaim memory if an FSM has significantly reduced its + * state count. Returns 1 if */ +enum fsm_vacuum_res { + FSM_VACUUM_NO_CHANGE, + FSM_VACUUM_REDUCED_MEMORY, + FSM_VACUUM_ERROC_REALLOC = -1, +} +fsm_vacuum(struct fsm *fsm); + #endif diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 005f7d4c6..2b7aba005 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -22,6 +22,7 @@ SRC += src/libfsm/trim.c SRC += src/libfsm/example.c SRC += src/libfsm/getc.c SRC += src/libfsm/shuffle.c +SRC += src/libfsm/vacuum.c SRC += src/libfsm/vm.c # graph things diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index 6a7bb790f..b9f092973 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -57,7 +57,7 @@ fsm_new(const struct fsm_options *opt) return NULL; } - new->statealloc = 128; /* guess */ + new->statealloc = FSM_DEFAULT_STATEALLOC; new->statecount = 0; new->endcount = 0; new->capture_info = NULL; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 6e77510a7..024bf99df 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -44,6 +44,9 @@ struct state_array; #define FSM_CAPTURE_MAX INT_MAX +/* guess for default state allocation */ +#define FSM_DEFAULT_STATEALLOC 128 + struct fsm_edge { fsm_state_t state; /* destination */ unsigned char symbol; diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index f905e1f04..6e1f72dbe 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -68,6 +68,7 @@ fsm_addstate fsm_addstate_bulk fsm_removestate fsm_shuffle +fsm_vacuum fsm_addedge_any fsm_addedge_epsilon diff --git a/src/libfsm/vacuum.c b/src/libfsm/vacuum.c new file mode 100644 index 000000000..2c349c210 --- /dev/null +++ b/src/libfsm/vacuum.c @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#include "internal.h" + +enum fsm_vacuum_res +fsm_vacuum(struct fsm *fsm) +{ + assert(fsm != NULL); + assert(fsm->statecount <= fsm->statealloc); + size_t nceil = fsm->statealloc; + + while ((fsm->statecount < nceil/2) && nceil > 1) { + nceil /= 2; + } + + if (nceil == fsm->statealloc) { + return FSM_VACUUM_NO_CHANGE; + } + + struct fsm_state *nstates = f_realloc(fsm->opt->alloc, + fsm->states, nceil * sizeof(nstates[0])); + if (nstates == NULL) { + return FSM_VACUUM_ERROC_REALLOC; + } + + fsm->states = nstates; + fsm->statealloc = nceil; + + return FSM_VACUUM_REDUCED_MEMORY; +} From 68c5710548d3687a00b15815af199bae01188e74 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 30 May 2024 11:56:43 +0100 Subject: [PATCH 111/373] Whitespace. --- src/libre/dialect/glob/parser.c | 227 ++++----- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/like/parser.c | 227 ++++----- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/literal/parser.c | 203 ++++---- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/native/parser.c | 551 ++++++++++---------- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/pcre/parser.c | 779 ++++++++++++++--------------- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/sql/parser.c | 451 +++++++++-------- src/libre/dialect/sql/parser.h | 4 +- src/libre/parser.act | 157 +++--- 13 files changed, 1292 insertions(+), 1327 deletions(-) diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index d9d30b552..f6504b854 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -124,93 +124,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -281,7 +201,82 @@ return s; } -#line 285 "src/libre/dialect/glob/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 280 "src/libre/dialect/glob/parser.c" #ifndef ERROR_TERMINAL @@ -316,13 +311,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 326 "src/libre/dialect/glob/parser.c" +#line 321 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 115 */ @@ -360,24 +355,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 369 "src/libre/dialect/glob/parser.c" +#line 364 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 381 "src/libre/dialect/glob/parser.c" +#line 376 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -390,7 +385,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -403,20 +398,20 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 407 "src/libre/dialect/glob/parser.c" +#line 402 "src/libre/dialect/glob/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 420 "src/libre/dialect/glob/parser.c" +#line 415 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -430,38 +425,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 439 "src/libre/dialect/glob/parser.c" +#line 434 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 451 "src/libre/dialect/glob/parser.c" +#line 446 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 820 "src/libre/parser.act" +#line 815 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 460 "src/libre/dialect/glob/parser.c" +#line 455 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -475,7 +470,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 479 "src/libre/dialect/glob/parser.c" +#line 474 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -490,26 +485,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 716 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 501 "src/libre/dialect/glob/parser.c" +#line 496 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 513 "src/libre/dialect/glob/parser.c" +#line 508 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -535,11 +530,11 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 543 "src/libre/dialect/glob/parser.c" +#line 538 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 119 */ @@ -549,14 +544,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 560 "src/libre/dialect/glob/parser.c" +#line 555 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -570,14 +565,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 581 "src/libre/dialect/glob/parser.c" +#line 576 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -587,14 +582,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 598 "src/libre/dialect/glob/parser.c" +#line 593 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ @@ -613,14 +608,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 765 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 624 "src/libre/dialect/glob/parser.c" +#line 619 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -638,7 +633,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1064 "src/libre/parser.act" +#line 1059 "src/libre/parser.act" static int @@ -796,6 +791,6 @@ ZL0:; return NULL; } -#line 800 "src/libre/dialect/glob/parser.c" +#line 795 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 074b0a37c..105c27e82 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 293 "src/libre/parser.act" +#line 288 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1221 "src/libre/parser.act" +#line 1216 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index c9f1f7769..12dae617a 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -124,93 +124,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -281,7 +201,82 @@ return s; } -#line 285 "src/libre/dialect/like/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 280 "src/libre/dialect/like/parser.c" #ifndef ERROR_TERMINAL @@ -316,13 +311,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 326 "src/libre/dialect/like/parser.c" +#line 321 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 115 */ @@ -360,24 +355,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 369 "src/libre/dialect/like/parser.c" +#line 364 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 381 "src/libre/dialect/like/parser.c" +#line 376 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -390,7 +385,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -403,20 +398,20 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 407 "src/libre/dialect/like/parser.c" +#line 402 "src/libre/dialect/like/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 420 "src/libre/dialect/like/parser.c" +#line 415 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -430,38 +425,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 439 "src/libre/dialect/like/parser.c" +#line 434 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 451 "src/libre/dialect/like/parser.c" +#line 446 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 820 "src/libre/parser.act" +#line 815 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 460 "src/libre/dialect/like/parser.c" +#line 455 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -475,7 +470,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 479 "src/libre/dialect/like/parser.c" +#line 474 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -490,26 +485,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 716 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 501 "src/libre/dialect/like/parser.c" +#line 496 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 513 "src/libre/dialect/like/parser.c" +#line 508 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -535,11 +530,11 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 543 "src/libre/dialect/like/parser.c" +#line 538 "src/libre/dialect/like/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 119 */ @@ -549,14 +544,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 560 "src/libre/dialect/like/parser.c" +#line 555 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -570,14 +565,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 581 "src/libre/dialect/like/parser.c" +#line 576 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -587,14 +582,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 598 "src/libre/dialect/like/parser.c" +#line 593 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ @@ -613,14 +608,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 765 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 624 "src/libre/dialect/like/parser.c" +#line 619 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -638,7 +633,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1064 "src/libre/parser.act" +#line 1059 "src/libre/parser.act" static int @@ -796,6 +791,6 @@ ZL0:; return NULL; } -#line 800 "src/libre/dialect/like/parser.c" +#line 795 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index c7e00987d..bb01f69f1 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 293 "src/libre/parser.act" +#line 288 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1221 "src/libre/parser.act" +#line 1216 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 118aa0393..404859678 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -124,93 +124,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -281,7 +201,82 @@ return s; } -#line 285 "src/libre/dialect/literal/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 280 "src/libre/dialect/literal/parser.c" #ifndef ERROR_TERMINAL @@ -316,13 +311,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 326 "src/libre/dialect/literal/parser.c" +#line 321 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 114 */ @@ -361,11 +356,11 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 369 "src/libre/dialect/literal/parser.c" +#line 364 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 117 */ @@ -375,14 +370,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 386 "src/libre/dialect/literal/parser.c" +#line 381 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -396,14 +391,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 407 "src/libre/dialect/literal/parser.c" +#line 402 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -413,14 +408,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* END OF INLINE: 117 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 424 "src/libre/dialect/literal/parser.c" +#line 419 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 119 */ @@ -439,14 +434,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 765 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 450 "src/libre/dialect/literal/parser.c" +#line 445 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -479,7 +474,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -492,7 +487,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 496 "src/libre/dialect/literal/parser.c" +#line 491 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -502,14 +497,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 513 "src/libre/dialect/literal/parser.c" +#line 508 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -518,26 +513,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 716 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 529 "src/libre/dialect/literal/parser.c" +#line 524 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 541 "src/libre/dialect/literal/parser.c" +#line 536 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -551,7 +546,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1064 "src/libre/parser.act" +#line 1059 "src/libre/parser.act" static int @@ -709,6 +704,6 @@ ZL0:; return NULL; } -#line 713 "src/libre/dialect/literal/parser.c" +#line 708 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index 2b6ef4b73..0a98a2362 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 293 "src/libre/parser.act" +#line 288 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1221 "src/libre/parser.act" +#line 1216 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 01f4a4a8a..2b204045a 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -124,93 +124,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -281,7 +201,82 @@ return s; } -#line 285 "src/libre/dialect/native/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 280 "src/libre/dialect/native/parser.c" #ifndef ERROR_TERMINAL @@ -338,7 +333,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -351,7 +346,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 355 "src/libre/dialect/native/parser.c" +#line 350 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -361,7 +356,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 398 "src/libre/parser.act" +#line 393 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -387,7 +382,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 391 "src/libre/dialect/native/parser.c" +#line 386 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -397,7 +392,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 539 "src/libre/parser.act" +#line 534 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -445,7 +440,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 449 "src/libre/dialect/native/parser.c" +#line 444 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -455,7 +450,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 496 "src/libre/parser.act" +#line 491 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -498,7 +493,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 502 "src/libre/dialect/native/parser.c" +#line 497 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -511,12 +506,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 141 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 520 "src/libre/dialect/native/parser.c" +#line 515 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -543,7 +538,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI216 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -551,23 +546,23 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI216; (void) ZIend; -#line 555 "src/libre/dialect/native/parser.c" +#line 550 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 566 "src/libre/dialect/native/parser.c" +#line 561 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -582,7 +577,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 586 "src/libre/dialect/native/parser.c" +#line 581 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -598,7 +593,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 639 "src/libre/parser.act" +#line 634 "src/libre/parser.act" unsigned long u; char *e; @@ -618,7 +613,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 622 "src/libre/dialect/native/parser.c" +#line 617 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -630,7 +625,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI219 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -638,7 +633,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI219; (void) ZIend; -#line 642 "src/libre/dialect/native/parser.c" +#line 637 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -648,17 +643,17 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 657 "src/libre/dialect/native/parser.c" +#line 652 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -673,7 +668,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 677 "src/libre/dialect/native/parser.c" +#line 672 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -708,13 +703,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 718 "src/libre/dialect/native/parser.c" +#line 713 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -748,7 +743,7 @@ p_154(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI155 = '-'; ZI156 = lex_state->lx.start; @@ -758,7 +753,7 @@ p_154(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI156; (void) ZI157; -#line 762 "src/libre/dialect/native/parser.c" +#line 757 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -772,14 +767,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 730 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 783 "src/libre/dialect/native/parser.c" +#line 778 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -807,13 +802,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 817 "src/libre/dialect/native/parser.c" +#line 812 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 223 */ @@ -861,7 +856,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -874,7 +869,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 878 "src/libre/dialect/native/parser.c" +#line 873 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -887,7 +882,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 398 "src/libre/parser.act" +#line 393 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -913,7 +908,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI111; (void) ZI112; -#line 917 "src/libre/dialect/native/parser.c" +#line 912 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -926,7 +921,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 539 "src/libre/parser.act" +#line 534 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -974,7 +969,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 978 "src/libre/dialect/native/parser.c" +#line 973 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -987,7 +982,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 496 "src/libre/parser.act" +#line 491 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1030,7 +1025,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1034 "src/libre/dialect/native/parser.c" +#line 1029 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1043,14 +1038,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 109 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1054 "src/libre/dialect/native/parser.c" +#line 1049 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1076,7 +1071,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1089,7 +1084,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI259 = lex_state->buf.a[0]; -#line 1093 "src/libre/dialect/native/parser.c" +#line 1088 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1108,7 +1103,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 398 "src/libre/parser.act" +#line 393 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1134,7 +1129,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI248; (void) ZI249; -#line 1138 "src/libre/dialect/native/parser.c" +#line 1133 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1153,7 +1148,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 539 "src/libre/parser.act" +#line 534 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1201,7 +1196,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI255 = (char) (unsigned char) u; -#line 1205 "src/libre/dialect/native/parser.c" +#line 1200 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1220,7 +1215,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 659 "src/libre/parser.act" +#line 654 "src/libre/parser.act" ZI243 = DIALECT_CLASS(lex_state->buf.a); if (ZI243 == NULL) { @@ -1234,7 +1229,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI244; (void) ZI245; -#line 1238 "src/libre/dialect/native/parser.c" +#line 1233 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1253,7 +1248,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 496 "src/libre/parser.act" +#line 491 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1296,7 +1291,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI251 = (char) (unsigned char) u; -#line 1300 "src/libre/dialect/native/parser.c" +#line 1295 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1337,7 +1332,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 659 "src/libre/parser.act" +#line 654 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1351,7 +1346,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1355 "src/libre/dialect/native/parser.c" +#line 1350 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1361,12 +1356,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 856 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1370 "src/libre/dialect/native/parser.c" +#line 1365 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1401,7 +1396,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 330 "src/libre/parser.act" +#line 325 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI167 = lex_state->lx.end; @@ -1409,20 +1404,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI167; -#line 1413 "src/libre/dialect/native/parser.c" +#line 1408 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1426 "src/libre/dialect/native/parser.c" +#line 1421 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1442,7 +1437,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 346 "src/libre/parser.act" +#line 341 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI186 = lex_state->lx.end; @@ -1450,30 +1445,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI186; -#line 1454 "src/libre/dialect/native/parser.c" +#line 1449 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1467 "src/libre/dialect/native/parser.c" +#line 1462 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 897 "src/libre/parser.act" +#line 892 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1477 "src/libre/dialect/native/parser.c" +#line 1472 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1483,13 +1478,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1493 "src/libre/dialect/native/parser.c" +#line 1488 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1505,7 +1500,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 338 "src/libre/parser.act" +#line 333 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI178 = lex_state->lx.end; @@ -1513,26 +1508,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI178; -#line 1517 "src/libre/dialect/native/parser.c" +#line 1512 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1530 "src/libre/dialect/native/parser.c" +#line 1525 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 978 "src/libre/parser.act" +#line 973 "src/libre/parser.act" struct ast_expr *any; @@ -1570,7 +1565,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1574 "src/libre/dialect/native/parser.c" +#line 1569 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ p_180 (flags, lex_state, act_state, err, &ZItmp); @@ -1589,7 +1584,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 354 "src/libre/parser.act" +#line 349 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI193 = lex_state->lx.end; @@ -1597,26 +1592,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI193; -#line 1601 "src/libre/dialect/native/parser.c" +#line 1596 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1614 "src/libre/dialect/native/parser.c" +#line 1609 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 978 "src/libre/parser.act" +#line 973 "src/libre/parser.act" struct ast_expr *any; @@ -1654,16 +1649,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1658 "src/libre/dialect/native/parser.c" +#line 1653 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 897 "src/libre/parser.act" +#line 892 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1667 "src/libre/dialect/native/parser.c" +#line 1662 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1673,13 +1668,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1683 "src/libre/dialect/native/parser.c" +#line 1678 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1705,7 +1700,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 362 "src/libre/parser.act" +#line 357 "src/libre/parser.act" ZI200 = ']'; ZI201 = lex_state->lx.start; @@ -1715,18 +1710,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI201; (void) ZIend; -#line 1719 "src/libre/dialect/native/parser.c" +#line 1714 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 779 "src/libre/parser.act" +#line 774 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1730 "src/libre/dialect/native/parser.c" +#line 1725 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1740,7 +1735,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 372 "src/libre/parser.act" +#line 367 "src/libre/parser.act" ZIcrange = '-'; ZI203 = lex_state->lx.start; @@ -1750,41 +1745,41 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI203; (void) ZIend; -#line 1754 "src/libre/dialect/native/parser.c" +#line 1749 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 1767 "src/libre/dialect/native/parser.c" +#line 1762 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 1778 "src/libre/dialect/native/parser.c" +#line 1773 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 779 "src/libre/parser.act" +#line 774 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1788 "src/libre/dialect/native/parser.c" +#line 1783 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1797,14 +1792,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 737 "src/libre/parser.act" +#line 732 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 1808 "src/libre/dialect/native/parser.c" +#line 1803 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -1833,7 +1828,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -1843,7 +1838,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI181; -#line 1847 "src/libre/dialect/native/parser.c" +#line 1842 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1854,14 +1849,14 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 1865 "src/libre/dialect/native/parser.c" +#line 1860 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1877,17 +1872,17 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 1886 "src/libre/dialect/native/parser.c" +#line 1881 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI183 = '-'; ZI184 = lex_state->lx.start; @@ -1897,7 +1892,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI184; (void) ZI185; -#line 1901 "src/libre/dialect/native/parser.c" +#line 1896 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1908,7 +1903,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -1937,7 +1932,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1941 "src/libre/dialect/native/parser.c" +#line 1936 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1947,13 +1942,13 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 182 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 1957 "src/libre/dialect/native/parser.c" +#line 1952 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -1989,7 +1984,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZIc = '-'; ZI149 = lex_state->lx.start; @@ -1999,18 +1994,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI149; (void) ZIend; -#line 2003 "src/libre/dialect/native/parser.c" +#line 1998 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2014 "src/libre/dialect/native/parser.c" +#line 2009 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2062,7 +2057,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2076,7 +2071,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2080 "src/libre/dialect/native/parser.c" +#line 2075 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2100,14 +2095,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2111 "src/libre/dialect/native/parser.c" +#line 2106 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2121,14 +2116,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2132 "src/libre/dialect/native/parser.c" +#line 2127 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2141,26 +2136,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 723 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2152 "src/libre/dialect/native/parser.c" +#line 2147 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2164 "src/libre/dialect/native/parser.c" +#line 2159 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2182,14 +2177,14 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2193 "src/libre/dialect/native/parser.c" +#line 2188 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2206,17 +2201,17 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2215 "src/libre/dialect/native/parser.c" +#line 2210 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI196 = '-'; ZI197 = lex_state->lx.start; @@ -2226,7 +2221,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI197; (void) ZI198; -#line 2230 "src/libre/dialect/native/parser.c" +#line 2225 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2237,17 +2232,17 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2246 "src/libre/dialect/native/parser.c" +#line 2241 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -2276,7 +2271,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2280 "src/libre/dialect/native/parser.c" +#line 2275 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2306,11 +2301,11 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2314 "src/libre/dialect/native/parser.c" +#line 2309 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -2320,14 +2315,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2331 "src/libre/dialect/native/parser.c" +#line 2326 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 231 */ @@ -2346,14 +2341,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 765 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 2357 "src/libre/dialect/native/parser.c" +#line 2352 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2434,13 +2429,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2444 "src/libre/dialect/native/parser.c" +#line 2439 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 229 */ @@ -2465,14 +2460,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 723 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 2476 "src/libre/dialect/native/parser.c" +#line 2471 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2497,7 +2492,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 382 "src/libre/parser.act" +#line 377 "src/libre/parser.act" ZI263 = lex_state->lx.start; ZI264 = lex_state->lx.end; @@ -2505,7 +2500,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI263; (void) ZI264; -#line 2509 "src/libre/dialect/native/parser.c" +#line 2504 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2513,7 +2508,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 639 "src/libre/parser.act" +#line 634 "src/libre/parser.act" unsigned long u; char *e; @@ -2533,7 +2528,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2537 "src/libre/dialect/native/parser.c" +#line 2532 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2553,11 +2548,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 828 "src/libre/parser.act" +#line 823 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 2561 "src/libre/dialect/native/parser.c" +#line 2556 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2567,11 +2562,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 824 "src/libre/parser.act" +#line 819 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2575 "src/libre/dialect/native/parser.c" +#line 2570 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2581,11 +2576,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 820 "src/libre/parser.act" +#line 815 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2589 "src/libre/dialect/native/parser.c" +#line 2584 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2594,11 +2589,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 832 "src/libre/parser.act" +#line 827 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2602 "src/libre/dialect/native/parser.c" +#line 2597 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2611,23 +2606,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 709 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 2622 "src/libre/dialect/native/parser.c" +#line 2617 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 832 "src/libre/parser.act" +#line 827 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2631 "src/libre/dialect/native/parser.c" +#line 2626 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2652,24 +2647,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2661 "src/libre/dialect/native/parser.c" +#line 2656 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2673 "src/libre/dialect/native/parser.c" +#line 2668 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2679,14 +2674,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 954 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 2690 "src/libre/dialect/native/parser.c" +#line 2685 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2699,11 +2694,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2707 "src/libre/dialect/native/parser.c" +#line 2702 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2713,14 +2708,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 2724 "src/libre/dialect/native/parser.c" +#line 2719 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2737,14 +2732,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 947 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 2748 "src/libre/dialect/native/parser.c" +#line 2743 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2777,26 +2772,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 716 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 2788 "src/libre/dialect/native/parser.c" +#line 2783 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2800 "src/libre/dialect/native/parser.c" +#line 2795 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2818,14 +2813,14 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI243)); if ((ZInode) == NULL) { goto ZL1; } -#line 2829 "src/libre/dialect/native/parser.c" +#line 2824 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2838,12 +2833,12 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 856 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI243); -#line 2847 "src/libre/dialect/native/parser.c" +#line 2842 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_154 (flags, lex_state, act_state, err); @@ -2854,17 +2849,17 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 784 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); -#line 2863 "src/libre/dialect/native/parser.c" +#line 2858 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -2893,7 +2888,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2897 "src/libre/dialect/native/parser.c" +#line 2892 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2920,14 +2915,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2931 "src/libre/dialect/native/parser.c" +#line 2926 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2954,14 +2949,14 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI247)); if ((ZInode) == NULL) { goto ZL1; } -#line 2965 "src/libre/dialect/native/parser.c" +#line 2960 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2974,12 +2969,12 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI247); -#line 2983 "src/libre/dialect/native/parser.c" +#line 2978 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_154 (flags, lex_state, act_state, err); @@ -2990,17 +2985,17 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 784 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); -#line 2999 "src/libre/dialect/native/parser.c" +#line 2994 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -3029,7 +3024,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3033 "src/libre/dialect/native/parser.c" +#line 3028 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3047,7 +3042,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1064 "src/libre/parser.act" +#line 1059 "src/libre/parser.act" static int @@ -3205,6 +3200,6 @@ ZL0:; return NULL; } -#line 3209 "src/libre/dialect/native/parser.c" +#line 3204 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 6aaf2a576..408a2c546 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 293 "src/libre/parser.act" +#line 288 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1221 "src/libre/parser.act" +#line 1216 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 8536441e3..137be4555 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -124,93 +124,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -281,7 +201,82 @@ return s; } -#line 285 "src/libre/dialect/pcre/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 280 "src/libre/dialect/pcre/parser.c" #ifndef ERROR_TERMINAL @@ -337,21 +332,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 677 "src/libre/parser.act" +#line 672 "src/libre/parser.act" ZIc = RE_EXTENDED; -#line 345 "src/libre/dialect/pcre/parser.c" +#line 340 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_EXTENDED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 812 "src/libre/parser.act" +#line 807 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 355 "src/libre/dialect/pcre/parser.c" +#line 350 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -368,21 +363,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 673 "src/libre/parser.act" +#line 668 "src/libre/parser.act" ZIc = RE_ICASE; -#line 376 "src/libre/dialect/pcre/parser.c" +#line 371 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_INSENSITIVE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 812 "src/libre/parser.act" +#line 807 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 386 "src/libre/dialect/pcre/parser.c" +#line 381 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -393,21 +388,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 681 "src/libre/parser.act" +#line 676 "src/libre/parser.act" ZIc = RE_SINGLE; -#line 401 "src/libre/dialect/pcre/parser.c" +#line 396 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_SINGLE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 812 "src/libre/parser.act" +#line 807 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 411 "src/libre/dialect/pcre/parser.c" +#line 406 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -418,14 +413,14 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 751 "src/libre/parser.act" +#line 746 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; } goto ZL1; -#line 429 "src/libre/dialect/pcre/parser.c" +#line 424 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unknown-flag */ } @@ -463,7 +458,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -476,7 +471,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 480 "src/libre/dialect/pcre/parser.c" +#line 475 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -486,7 +481,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 454 "src/libre/parser.act" +#line 449 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -528,20 +523,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 532 "src/libre/dialect/pcre/parser.c" +#line 527 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 545 "src/libre/dialect/pcre/parser.c" +#line 540 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -550,7 +545,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 398 "src/libre/parser.act" +#line 393 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -576,7 +571,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 580 "src/libre/dialect/pcre/parser.c" +#line 575 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -586,7 +581,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 539 "src/libre/parser.act" +#line 534 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -634,7 +629,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 638 "src/libre/dialect/pcre/parser.c" +#line 633 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -644,7 +639,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 496 "src/libre/parser.act" +#line 491 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -687,7 +682,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 691 "src/libre/dialect/pcre/parser.c" +#line 686 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -697,7 +692,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 438 "src/libre/parser.act" +#line 433 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -713,20 +708,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 717 "src/libre/dialect/pcre/parser.c" +#line 712 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 730 "src/libre/dialect/pcre/parser.c" +#line 725 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -738,12 +733,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 155 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 747 "src/libre/dialect/pcre/parser.c" +#line 742 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -774,13 +769,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 784 "src/libre/dialect/pcre/parser.c" +#line 779 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -834,13 +829,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 844 "src/libre/dialect/pcre/parser.c" +#line 839 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ } @@ -886,14 +881,14 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI290)); if ((ZInode) == NULL) { goto ZL1; } -#line 897 "src/libre/dialect/pcre/parser.c" +#line 892 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -906,12 +901,12 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 856 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI290); -#line 915 "src/libre/dialect/pcre/parser.c" +#line 910 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_168 (flags, lex_state, act_state, err); @@ -922,17 +917,17 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 784 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI291)); mark(&act_state->rangeend, &(ZIend)); -#line 931 "src/libre/dialect/pcre/parser.c" +#line 926 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -961,7 +956,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 965 "src/libre/dialect/pcre/parser.c" +#line 960 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -992,7 +987,7 @@ p_168(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI169 = '-'; ZI170 = lex_state->lx.start; @@ -1002,7 +997,7 @@ p_168(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI170; (void) ZI171; -#line 1006 "src/libre/dialect/pcre/parser.c" +#line 1001 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1016,14 +1011,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 730 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 1027 "src/libre/dialect/pcre/parser.c" +#line 1022 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1055,7 +1050,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1068,7 +1063,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 1072 "src/libre/dialect/pcre/parser.c" +#line 1067 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1081,7 +1076,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CONTROL */ { -#line 454 "src/libre/parser.act" +#line 449 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1123,7 +1118,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI123; (void) ZI124; -#line 1127 "src/libre/dialect/pcre/parser.c" +#line 1122 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1136,7 +1131,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 398 "src/libre/parser.act" +#line 393 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1162,7 +1157,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI113; (void) ZI114; -#line 1166 "src/libre/dialect/pcre/parser.c" +#line 1161 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1175,7 +1170,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 539 "src/libre/parser.act" +#line 534 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1223,7 +1218,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1227 "src/libre/dialect/pcre/parser.c" +#line 1222 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1236,7 +1231,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: NOESC */ { -#line 424 "src/libre/parser.act" +#line 419 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1250,7 +1245,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI115; (void) ZI116; -#line 1254 "src/libre/dialect/pcre/parser.c" +#line 1249 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1263,7 +1258,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 496 "src/libre/parser.act" +#line 491 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1306,7 +1301,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1310 "src/libre/dialect/pcre/parser.c" +#line 1305 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1319,7 +1314,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 438 "src/libre/parser.act" +#line 433 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1335,20 +1330,20 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI125; (void) ZI126; -#line 1339 "src/libre/dialect/pcre/parser.c" +#line 1334 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1352 "src/libre/dialect/pcre/parser.c" +#line 1347 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1360,14 +1355,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 111 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1371 "src/libre/dialect/pcre/parser.c" +#line 1366 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1393,7 +1388,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1406,7 +1401,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI306 = lex_state->buf.a[0]; -#line 1410 "src/libre/dialect/pcre/parser.c" +#line 1405 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1425,7 +1420,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CONTROL */ { -#line 454 "src/libre/parser.act" +#line 449 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1467,20 +1462,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI311; (void) ZI312; -#line 1471 "src/libre/dialect/pcre/parser.c" +#line 1466 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1484 "src/libre/dialect/pcre/parser.c" +#line 1479 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_317 (flags, lex_state, act_state, err, &ZI310, &ZI311, &ZInode); @@ -1498,7 +1493,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 398 "src/libre/parser.act" +#line 393 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1524,7 +1519,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI295; (void) ZI296; -#line 1528 "src/libre/dialect/pcre/parser.c" +#line 1523 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1543,7 +1538,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 539 "src/libre/parser.act" +#line 534 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1591,7 +1586,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI302 = (char) (unsigned char) u; -#line 1595 "src/libre/dialect/pcre/parser.c" +#line 1590 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1610,7 +1605,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 659 "src/libre/parser.act" +#line 654 "src/libre/parser.act" ZI290 = DIALECT_CLASS(lex_state->buf.a); if (ZI290 == NULL) { @@ -1624,7 +1619,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI291; (void) ZI292; -#line 1628 "src/libre/dialect/pcre/parser.c" +#line 1623 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1643,7 +1638,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NOESC */ { -#line 424 "src/libre/parser.act" +#line 419 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1657,20 +1652,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI134; (void) ZI135; -#line 1661 "src/libre/dialect/pcre/parser.c" +#line 1656 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1674 "src/libre/dialect/pcre/parser.c" +#line 1669 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1683,7 +1678,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 496 "src/libre/parser.act" +#line 491 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1726,7 +1721,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI298 = (char) (unsigned char) u; -#line 1730 "src/libre/dialect/pcre/parser.c" +#line 1725 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1745,7 +1740,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 438 "src/libre/parser.act" +#line 433 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1761,20 +1756,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI315; (void) ZI316; -#line 1765 "src/libre/dialect/pcre/parser.c" +#line 1760 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1778 "src/libre/dialect/pcre/parser.c" +#line 1773 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_317 (flags, lex_state, act_state, err, &ZI314, &ZI315, &ZInode); @@ -1813,14 +1808,14 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 695 "src/libre/parser.act" +#line 690 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; } goto ZL1; -#line 1824 "src/libre/dialect/pcre/parser.c" +#line 1819 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1847,7 +1842,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 659 "src/libre/parser.act" +#line 654 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1861,7 +1856,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1865 "src/libre/dialect/pcre/parser.c" +#line 1860 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1871,12 +1866,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 856 "src/libre/parser.act" +#line 851 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1880 "src/libre/dialect/pcre/parser.c" +#line 1875 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1911,7 +1906,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 330 "src/libre/parser.act" +#line 325 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI181 = lex_state->lx.end; @@ -1919,20 +1914,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI181; -#line 1923 "src/libre/dialect/pcre/parser.c" +#line 1918 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1936 "src/libre/dialect/pcre/parser.c" +#line 1931 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1952,7 +1947,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 346 "src/libre/parser.act" +#line 341 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI200 = lex_state->lx.end; @@ -1960,30 +1955,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI200; -#line 1964 "src/libre/dialect/pcre/parser.c" +#line 1959 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1977 "src/libre/dialect/pcre/parser.c" +#line 1972 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 897 "src/libre/parser.act" +#line 892 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1987 "src/libre/dialect/pcre/parser.c" +#line 1982 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1993,13 +1988,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2003 "src/libre/dialect/pcre/parser.c" +#line 1998 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2015,7 +2010,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 338 "src/libre/parser.act" +#line 333 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI192 = lex_state->lx.end; @@ -2023,26 +2018,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI192; -#line 2027 "src/libre/dialect/pcre/parser.c" +#line 2022 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2040 "src/libre/dialect/pcre/parser.c" +#line 2035 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 978 "src/libre/parser.act" +#line 973 "src/libre/parser.act" struct ast_expr *any; @@ -2080,7 +2075,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2084 "src/libre/dialect/pcre/parser.c" +#line 2079 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ p_194 (flags, lex_state, act_state, err, &ZItmp); @@ -2099,7 +2094,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 354 "src/libre/parser.act" +#line 349 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI207 = lex_state->lx.end; @@ -2107,26 +2102,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI207; -#line 2111 "src/libre/dialect/pcre/parser.c" +#line 2106 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2124 "src/libre/dialect/pcre/parser.c" +#line 2119 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 978 "src/libre/parser.act" +#line 973 "src/libre/parser.act" struct ast_expr *any; @@ -2164,16 +2159,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2168 "src/libre/dialect/pcre/parser.c" +#line 2163 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 897 "src/libre/parser.act" +#line 892 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 2177 "src/libre/dialect/pcre/parser.c" +#line 2172 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -2183,13 +2178,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2193 "src/libre/dialect/pcre/parser.c" +#line 2188 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2215,7 +2210,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 362 "src/libre/parser.act" +#line 357 "src/libre/parser.act" ZI214 = ']'; ZI215 = lex_state->lx.start; @@ -2225,18 +2220,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI215; (void) ZIend; -#line 2229 "src/libre/dialect/pcre/parser.c" +#line 2224 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 779 "src/libre/parser.act" +#line 774 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2240 "src/libre/dialect/pcre/parser.c" +#line 2235 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2250,7 +2245,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 372 "src/libre/parser.act" +#line 367 "src/libre/parser.act" ZIcrange = '-'; ZI217 = lex_state->lx.start; @@ -2260,41 +2255,41 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI217; (void) ZIend; -#line 2264 "src/libre/dialect/pcre/parser.c" +#line 2259 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 2277 "src/libre/dialect/pcre/parser.c" +#line 2272 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 2288 "src/libre/dialect/pcre/parser.c" +#line 2283 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 779 "src/libre/parser.act" +#line 774 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2298 "src/libre/dialect/pcre/parser.c" +#line 2293 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2307,14 +2302,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 737 "src/libre/parser.act" +#line 732 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 2318 "src/libre/dialect/pcre/parser.c" +#line 2313 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -2350,7 +2345,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZIc = '-'; ZI163 = lex_state->lx.start; @@ -2360,18 +2355,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI163; (void) ZIend; -#line 2364 "src/libre/dialect/pcre/parser.c" +#line 2359 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2375 "src/libre/dialect/pcre/parser.c" +#line 2370 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2413,14 +2408,14 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI314)); if ((ZInode) == NULL) { goto ZL1; } -#line 2424 "src/libre/dialect/pcre/parser.c" +#line 2419 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2433,12 +2428,12 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI314); -#line 2442 "src/libre/dialect/pcre/parser.c" +#line 2437 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_168 (flags, lex_state, act_state, err); @@ -2449,17 +2444,17 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 784 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI315)); mark(&act_state->rangeend, &(ZIend)); -#line 2458 "src/libre/dialect/pcre/parser.c" +#line 2453 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -2488,7 +2483,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2492 "src/libre/dialect/pcre/parser.c" +#line 2487 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2534,16 +2529,16 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* BEGINNING OF ACTION: count-one */ { -#line 832 "src/libre/parser.act" +#line 827 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2542 "src/libre/dialect/pcre/parser.c" +#line 2537 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2557,7 +2552,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2561 "src/libre/dialect/pcre/parser.c" +#line 2556 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2590,7 +2585,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI256 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2598,23 +2593,23 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI256; (void) ZIend; -#line 2602 "src/libre/dialect/pcre/parser.c" +#line 2597 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2613 "src/libre/dialect/pcre/parser.c" +#line 2608 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2629,7 +2624,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 2633 "src/libre/dialect/pcre/parser.c" +#line 2628 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2668,14 +2663,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2679 "src/libre/dialect/pcre/parser.c" +#line 2674 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2689,26 +2684,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 723 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2700 "src/libre/dialect/pcre/parser.c" +#line 2695 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2712 "src/libre/dialect/pcre/parser.c" +#line 2707 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2734,7 +2729,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2742,32 +2737,32 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI261; (void) ZIend; -#line 2746 "src/libre/dialect/pcre/parser.c" +#line 2741 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2757 "src/libre/dialect/pcre/parser.c" +#line 2752 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ { -#line 816 "src/libre/parser.act" +#line 811 "src/libre/parser.act" (ZIn) = AST_COUNT_UNBOUNDED; -#line 2766 "src/libre/dialect/pcre/parser.c" +#line 2761 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2782,7 +2777,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2786 "src/libre/dialect/pcre/parser.c" +#line 2781 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2795,7 +2790,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: COUNT */ { -#line 639 "src/libre/parser.act" +#line 634 "src/libre/parser.act" unsigned long u; char *e; @@ -2815,7 +2810,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 2819 "src/libre/dialect/pcre/parser.c" +#line 2814 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -2823,7 +2818,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI259 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2831,7 +2826,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI259; (void) ZIend; -#line 2835 "src/libre/dialect/pcre/parser.c" +#line 2830 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -2841,17 +2836,17 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2850 "src/libre/dialect/pcre/parser.c" +#line 2845 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2866,7 +2861,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2870 "src/libre/dialect/pcre/parser.c" +#line 2865 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2897,7 +2892,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -2907,7 +2902,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI195; -#line 2911 "src/libre/dialect/pcre/parser.c" +#line 2906 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2918,14 +2913,14 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2929 "src/libre/dialect/pcre/parser.c" +#line 2924 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2941,17 +2936,17 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 2950 "src/libre/dialect/pcre/parser.c" +#line 2945 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI197 = '-'; ZI198 = lex_state->lx.start; @@ -2961,7 +2956,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI198; (void) ZI199; -#line 2965 "src/libre/dialect/pcre/parser.c" +#line 2960 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2972,7 +2967,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -3001,7 +2996,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 3005 "src/libre/dialect/pcre/parser.c" +#line 3000 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3011,13 +3006,13 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 196 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 3021 "src/libre/dialect/pcre/parser.c" +#line 3016 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -3056,20 +3051,20 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 808 "src/libre/parser.act" +#line 803 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; -#line 3064 "src/libre/dialect/pcre/parser.c" +#line 3059 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 808 "src/libre/parser.act" +#line 803 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; -#line 3073 "src/libre/dialect/pcre/parser.c" +#line 3068 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF INLINE: 231 */ @@ -3122,7 +3117,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 938 "src/libre/parser.act" +#line 933 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3131,19 +3126,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3135 "src/libre/dialect/pcre/parser.c" +#line 3130 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL5; } -#line 3147 "src/libre/dialect/pcre/parser.c" +#line 3142 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3156,16 +3151,16 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 930 "src/libre/parser.act" +#line 925 "src/libre/parser.act" (ZIflags) = *flags; -#line 3164 "src/libre/dialect/pcre/parser.c" +#line 3159 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 938 "src/libre/parser.act" +#line 933 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3174,7 +3169,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3178 "src/libre/dialect/pcre/parser.c" +#line 3173 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3184,11 +3179,11 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 934 "src/libre/parser.act" +#line 929 "src/libre/parser.act" *flags = (ZIflags); -#line 3192 "src/libre/dialect/pcre/parser.c" +#line 3187 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3209,26 +3204,26 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 758 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; } goto ZL1; -#line 3220 "src/libre/dialect/pcre/parser.c" +#line 3215 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 3232 "src/libre/dialect/pcre/parser.c" +#line 3227 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3262,7 +3257,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3276,7 +3271,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3280 "src/libre/dialect/pcre/parser.c" +#line 3275 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ /* BEGINNING OF INLINE: 264 */ @@ -3287,14 +3282,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3298 "src/libre/dialect/pcre/parser.c" +#line 3293 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3304,14 +3299,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 772 "src/libre/parser.act" +#line 767 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3315 "src/libre/dialect/pcre/parser.c" +#line 3310 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3396,7 +3391,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 659 "src/libre/parser.act" +#line 654 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3410,7 +3405,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3414 "src/libre/dialect/pcre/parser.c" +#line 3409 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3420,14 +3415,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3431 "src/libre/dialect/pcre/parser.c" +#line 3426 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3451,14 +3446,14 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3462 "src/libre/dialect/pcre/parser.c" +#line 3457 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3475,17 +3470,17 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3484 "src/libre/dialect/pcre/parser.c" +#line 3479 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI210 = '-'; ZI211 = lex_state->lx.start; @@ -3495,7 +3490,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI211; (void) ZI212; -#line 3499 "src/libre/dialect/pcre/parser.c" +#line 3494 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3506,17 +3501,17 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3515 "src/libre/dialect/pcre/parser.c" +#line 3510 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -3545,7 +3540,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3549 "src/libre/dialect/pcre/parser.c" +#line 3544 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3578,13 +3573,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3588 "src/libre/dialect/pcre/parser.c" +#line 3583 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 278 */ @@ -3609,14 +3604,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 723 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 3620 "src/libre/dialect/pcre/parser.c" +#line 3615 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3641,7 +3636,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 382 "src/libre/parser.act" +#line 377 "src/libre/parser.act" ZI318 = lex_state->lx.start; ZI319 = lex_state->lx.end; @@ -3649,7 +3644,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI318; (void) ZI319; -#line 3653 "src/libre/dialect/pcre/parser.c" +#line 3648 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3657,7 +3652,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 639 "src/libre/parser.act" +#line 634 "src/libre/parser.act" unsigned long u; char *e; @@ -3677,7 +3672,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3681 "src/libre/dialect/pcre/parser.c" +#line 3676 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3697,11 +3692,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 828 "src/libre/parser.act" +#line 823 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 3705 "src/libre/dialect/pcre/parser.c" +#line 3700 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3711,11 +3706,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 824 "src/libre/parser.act" +#line 819 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3719 "src/libre/dialect/pcre/parser.c" +#line 3714 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3725,11 +3720,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 820 "src/libre/parser.act" +#line 815 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3733 "src/libre/dialect/pcre/parser.c" +#line 3728 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3744,23 +3739,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 709 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 3755 "src/libre/dialect/pcre/parser.c" +#line 3750 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 832 "src/libre/parser.act" +#line 827 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 3764 "src/libre/dialect/pcre/parser.c" +#line 3759 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3786,11 +3781,11 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3794 "src/libre/dialect/pcre/parser.c" +#line 3789 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3800,14 +3795,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3811 "src/libre/dialect/pcre/parser.c" +#line 3806 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 280 */ @@ -3826,14 +3821,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 765 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 3837 "src/libre/dialect/pcre/parser.c" +#line 3832 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -3862,24 +3857,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3871 "src/libre/dialect/pcre/parser.c" +#line 3866 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3883 "src/libre/dialect/pcre/parser.c" +#line 3878 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3889,14 +3884,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 954 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 3900 "src/libre/dialect/pcre/parser.c" +#line 3895 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3906,7 +3901,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 961 "src/libre/parser.act" +#line 956 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3916,7 +3911,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3920 "src/libre/dialect/pcre/parser.c" +#line 3915 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3934,134 +3929,134 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 799 "src/libre/parser.act" +#line 794 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3943 "src/libre/dialect/pcre/parser.c" +#line 3938 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 3955 "src/libre/dialect/pcre/parser.c" +#line 3950 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { goto ZL1; } -#line 3967 "src/libre/dialect/pcre/parser.c" +#line 3962 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 901 "src/libre/parser.act" +#line 896 "src/libre/parser.act" (ZIcr) = '\r'; -#line 3976 "src/libre/dialect/pcre/parser.c" +#line 3971 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { goto ZL1; } -#line 3988 "src/libre/dialect/pcre/parser.c" +#line 3983 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 905 "src/libre/parser.act" +#line 900 "src/libre/parser.act" (ZInl) = '\n'; -#line 3997 "src/libre/dialect/pcre/parser.c" +#line 3992 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { goto ZL1; } -#line 4009 "src/libre/dialect/pcre/parser.c" +#line 4004 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4020 "src/libre/dialect/pcre/parser.c" +#line 4015 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4031 "src/libre/dialect/pcre/parser.c" +#line 4026 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 4043 "src/libre/dialect/pcre/parser.c" +#line 4038 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4054 "src/libre/dialect/pcre/parser.c" +#line 4049 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4065 "src/libre/dialect/pcre/parser.c" +#line 4060 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4075,20 +4070,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 930 "src/libre/parser.act" +#line 925 "src/libre/parser.act" (ZIflags) = *flags; -#line 4083 "src/libre/dialect/pcre/parser.c" +#line 4078 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 4092 "src/libre/dialect/pcre/parser.c" +#line 4087 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4098,23 +4093,23 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 934 "src/libre/parser.act" +#line 929 "src/libre/parser.act" *flags = (ZIflags); -#line 4106 "src/libre/dialect/pcre/parser.c" +#line 4101 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 4118 "src/libre/dialect/pcre/parser.c" +#line 4113 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4131,14 +4126,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 947 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 4142 "src/libre/dialect/pcre/parser.c" +#line 4137 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4190,26 +4185,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 716 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 4201 "src/libre/dialect/pcre/parser.c" +#line 4196 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 4213 "src/libre/dialect/pcre/parser.c" +#line 4208 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4235,14 +4230,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4246 "src/libre/dialect/pcre/parser.c" +#line 4241 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4256,14 +4251,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4267 "src/libre/dialect/pcre/parser.c" +#line 4262 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4299,25 +4294,25 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4310 "src/libre/dialect/pcre/parser.c" +#line 4305 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4321 "src/libre/dialect/pcre/parser.c" +#line 4316 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4331,7 +4326,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1064 "src/libre/parser.act" +#line 1059 "src/libre/parser.act" static int @@ -4489,6 +4484,6 @@ ZL0:; return NULL; } -#line 4493 "src/libre/dialect/pcre/parser.c" +#line 4488 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 46c464b38..b808f0eb6 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 293 "src/libre/parser.act" +#line 288 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1221 "src/libre/parser.act" +#line 1216 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index 0a3df7280..a1158231a 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -124,93 +124,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -281,7 +201,82 @@ return s; } -#line 285 "src/libre/dialect/sql/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 280 "src/libre/dialect/sql/parser.c" #ifndef ERROR_TERMINAL @@ -323,13 +318,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: INVERT */ { -#line 314 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI203 = '^'; (void) ZI203; -#line 333 "src/libre/dialect/sql/parser.c" +#line 328 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; @@ -349,7 +344,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZIc = '-'; ZI114 = lex_state->lx.start; @@ -359,31 +354,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ (void) ZI114; (void) ZI115; -#line 363 "src/libre/dialect/sql/parser.c" +#line 358 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 376 "src/libre/dialect/sql/parser.c" +#line 371 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 387 "src/libre/dialect/sql/parser.c" +#line 382 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -413,11 +408,11 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 421 "src/libre/dialect/sql/parser.c" +#line 416 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -427,14 +422,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 438 "src/libre/dialect/sql/parser.c" +#line 433 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 191 */ @@ -453,14 +448,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 765 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 464 "src/libre/dialect/sql/parser.c" +#line 459 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -496,13 +491,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; } -#line 506 "src/libre/dialect/sql/parser.c" +#line 501 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -511,14 +506,14 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 702 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; } goto ZL1; -#line 522 "src/libre/dialect/sql/parser.c" +#line 517 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } @@ -572,13 +567,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1052 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 582 "src/libre/dialect/sql/parser.c" +#line 577 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 183 */ @@ -618,7 +613,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -631,7 +626,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI205 = lex_state->buf.a[0]; -#line 635 "src/libre/dialect/sql/parser.c" +#line 630 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -682,7 +677,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 330 "src/libre/parser.act" +#line 325 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -690,7 +685,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 694 "src/libre/dialect/sql/parser.c" +#line 689 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -700,14 +695,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { goto ZL1; } -#line 711 "src/libre/dialect/sql/parser.c" +#line 706 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; @@ -725,7 +720,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 362 "src/libre/parser.act" +#line 357 "src/libre/parser.act" ZI154 = ']'; ZIclose__start = lex_state->lx.start; @@ -735,18 +730,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 739 "src/libre/dialect/sql/parser.c" +#line 734 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 779 "src/libre/parser.act" +#line 774 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 750 "src/libre/dialect/sql/parser.c" +#line 745 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ ZInode = ZIclass; @@ -760,26 +755,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: INVERT */ { -#line 314 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI158 = '^'; (void) ZI158; -#line 770 "src/libre/dialect/sql/parser.c" +#line 765 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { goto ZL3; } -#line 783 "src/libre/dialect/sql/parser.c" +#line 778 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; @@ -801,7 +796,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 362 "src/libre/parser.act" +#line 357 "src/libre/parser.act" ZI163 = ']'; ZIclose__start = lex_state->lx.start; @@ -811,7 +806,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 815 "src/libre/dialect/sql/parser.c" +#line 810 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -821,12 +816,12 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 779 "src/libre/parser.act" +#line 774 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 830 "src/libre/dialect/sql/parser.c" +#line 825 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ } @@ -835,14 +830,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 737 "src/libre/parser.act" +#line 732 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL3; -#line 846 "src/libre/dialect/sql/parser.c" +#line 841 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -851,14 +846,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 162 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 971 "src/libre/parser.act" +#line 966 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { goto ZL3; } -#line 862 "src/libre/dialect/sql/parser.c" +#line 857 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -874,26 +869,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 737 "src/libre/parser.act" +#line 732 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 885 "src/libre/dialect/sql/parser.c" +#line 880 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 897 "src/libre/dialect/sql/parser.c" +#line 892 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -929,7 +924,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -943,7 +938,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 947 "src/libre/dialect/sql/parser.c" +#line 942 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -966,14 +961,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 879 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 977 "src/libre/dialect/sql/parser.c" +#line 972 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -987,26 +982,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 723 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 998 "src/libre/dialect/sql/parser.c" +#line 993 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 1010 "src/libre/dialect/sql/parser.c" +#line 1005 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1031,7 +1026,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZIc = '-'; ZI117 = lex_state->lx.start; @@ -1041,36 +1036,36 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI117; (void) ZI118; -#line 1045 "src/libre/dialect/sql/parser.c" +#line 1040 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1058 "src/libre/dialect/sql/parser.c" +#line 1053 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 1069 "src/libre/dialect/sql/parser.c" +#line 1064 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 978 "src/libre/parser.act" +#line 973 "src/libre/parser.act" struct ast_expr *any; @@ -1108,7 +1103,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1112 "src/libre/dialect/sql/parser.c" +#line 1107 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1117,7 +1112,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 978 "src/libre/parser.act" +#line 973 "src/libre/parser.act" struct ast_expr *any; @@ -1155,7 +1150,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1159 "src/libre/dialect/sql/parser.c" +#line 1154 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1179,14 +1174,14 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI205)); if ((ZInode) == NULL) { goto ZL1; } -#line 1190 "src/libre/dialect/sql/parser.c" +#line 1185 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1204,17 +1199,17 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; (ZIa).u.literal.c = (unsigned char) (*ZI205); -#line 1213 "src/libre/dialect/sql/parser.c" +#line 1208 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 320 "src/libre/parser.act" +#line 315 "src/libre/parser.act" ZI136 = '-'; ZI137 = lex_state->lx.start; @@ -1224,7 +1219,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (void) ZI137; (void) ZI138; -#line 1228 "src/libre/dialect/sql/parser.c" +#line 1223 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1232,7 +1227,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1245,7 +1240,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ZIcz = lex_state->buf.a[0]; -#line 1249 "src/libre/dialect/sql/parser.c" +#line 1244 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1255,27 +1250,27 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 851 "src/libre/parser.act" +#line 846 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char) (ZIcz); -#line 1264 "src/libre/dialect/sql/parser.c" +#line 1259 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 784 "src/libre/parser.act" +#line 779 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI206)); mark(&act_state->rangeend, &(ZIend)); -#line 1274 "src/libre/dialect/sql/parser.c" +#line 1269 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1016 "src/libre/parser.act" +#line 1011 "src/libre/parser.act" unsigned char lower, upper; @@ -1304,7 +1299,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1308 "src/libre/dialect/sql/parser.c" +#line 1303 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1333,7 +1328,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI176 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1341,23 +1336,23 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI176; (void) ZIend; -#line 1345 "src/libre/dialect/sql/parser.c" +#line 1340 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1356 "src/libre/dialect/sql/parser.c" +#line 1351 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1372,7 +1367,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1376 "src/libre/dialect/sql/parser.c" +#line 1371 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1388,7 +1383,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 639 "src/libre/parser.act" +#line 634 "src/libre/parser.act" unsigned long u; char *e; @@ -1408,7 +1403,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1412 "src/libre/dialect/sql/parser.c" +#line 1407 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1420,7 +1415,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 390 "src/libre/parser.act" +#line 385 "src/libre/parser.act" ZI179 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1428,7 +1423,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI179; (void) ZIend; -#line 1432 "src/libre/dialect/sql/parser.c" +#line 1427 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1438,17 +1433,17 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 789 "src/libre/parser.act" +#line 784 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1447 "src/libre/dialect/sql/parser.c" +#line 1442 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 836 "src/libre/parser.act" +#line 831 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1463,7 +1458,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1467 "src/libre/dialect/sql/parser.c" +#line 1462 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1498,13 +1493,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1058 "src/libre/parser.act" +#line 1053 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1508 "src/libre/dialect/sql/parser.c" +#line 1503 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 189 */ @@ -1529,14 +1524,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 723 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 1540 "src/libre/dialect/sql/parser.c" +#line 1535 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1561,7 +1556,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 382 "src/libre/parser.act" +#line 377 "src/libre/parser.act" ZI209 = lex_state->lx.start; ZI210 = lex_state->lx.end; @@ -1569,7 +1564,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI209; (void) ZI210; -#line 1573 "src/libre/dialect/sql/parser.c" +#line 1568 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1577,7 +1572,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 639 "src/libre/parser.act" +#line 634 "src/libre/parser.act" unsigned long u; char *e; @@ -1597,7 +1592,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1601 "src/libre/dialect/sql/parser.c" +#line 1596 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1617,11 +1612,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 828 "src/libre/parser.act" +#line 823 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 1625 "src/libre/dialect/sql/parser.c" +#line 1620 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1631,11 +1626,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 824 "src/libre/parser.act" +#line 819 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1639 "src/libre/dialect/sql/parser.c" +#line 1634 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1645,11 +1640,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 820 "src/libre/parser.act" +#line 815 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1653 "src/libre/dialect/sql/parser.c" +#line 1648 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1658,11 +1653,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 832 "src/libre/parser.act" +#line 827 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1666 "src/libre/dialect/sql/parser.c" +#line 1661 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1675,23 +1670,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 709 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 1686 "src/libre/dialect/sql/parser.c" +#line 1681 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 832 "src/libre/parser.act" +#line 827 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1695 "src/libre/dialect/sql/parser.c" +#line 1690 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1716,24 +1711,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1725 "src/libre/dialect/sql/parser.c" +#line 1720 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1737 "src/libre/dialect/sql/parser.c" +#line 1732 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1746,7 +1741,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* BEGINNING OF EXTRACT: CHAR */ { -#line 587 "src/libre/parser.act" +#line 582 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1759,20 +1754,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1763 "src/libre/dialect/sql/parser.c" +#line 1758 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 886 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1776 "src/libre/dialect/sql/parser.c" +#line 1771 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1786,38 +1781,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 794 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1795 "src/libre/dialect/sql/parser.c" +#line 1790 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1807 "src/libre/dialect/sql/parser.c" +#line 1802 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 820 "src/libre/parser.act" +#line 815 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1816 "src/libre/dialect/sql/parser.c" +#line 1811 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 909 "src/libre/parser.act" +#line 904 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1831,7 +1826,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1835 "src/libre/dialect/sql/parser.c" +#line 1830 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1844,11 +1839,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 893 "src/libre/parser.act" +#line 888 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 1852 "src/libre/dialect/sql/parser.c" +#line 1847 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1858,14 +1853,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 923 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 1869 "src/libre/dialect/sql/parser.c" +#line 1864 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1896,26 +1891,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 716 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 1907 "src/libre/dialect/sql/parser.c" +#line 1902 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 1919 "src/libre/dialect/sql/parser.c" +#line 1914 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1944,7 +1939,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 659 "src/libre/parser.act" +#line 654 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1958,7 +1953,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI128; (void) ZI129; -#line 1962 "src/libre/dialect/sql/parser.c" +#line 1957 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1968,14 +1963,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1045 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 1979 "src/libre/dialect/sql/parser.c" +#line 1974 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1998,14 +1993,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 872 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2009 "src/libre/dialect/sql/parser.c" +#line 2004 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2019,14 +2014,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 865 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2030 "src/libre/dialect/sql/parser.c" +#line 2025 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2044,7 +2039,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1064 "src/libre/parser.act" +#line 1059 "src/libre/parser.act" static int @@ -2202,6 +2197,6 @@ ZL0:; return NULL; } -#line 2206 "src/libre/dialect/sql/parser.c" +#line 2201 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 9433c21e9..7d292714c 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 293 "src/libre/parser.act" +#line 288 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1221 "src/libre/parser.act" +#line 1216 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index 418d01469..af2d5724e 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -133,93 +133,13 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; + #if PCRE_DIALECT int extended_mode_comment; enum re_flags *flags_ptr; #endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL @$ERROR - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -290,6 +210,81 @@ return s; } + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state) + { +#ifndef PCRE_DIALECT + (void) lex_state; + return act_state->lex_tok; +#else + enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; + enum LX_TOKEN tok; + + if ((fl & RE_EXTENDED) == 0) { + tok = act_state->lex_tok; + + switch (tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return tok; + } + } + + restart: + + tok = act_state->lex_tok; + if (lex_state->extended_mode_comment) { + switch (tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return tok; + + case TOK_NEWLINE: + lex_state->extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (tok) { + case TOK_MAYBE_COMMENT: + lex_state->extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define ERROR_TERMINAL @$ERROR + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + @}, @{ #include From e9d190a663b7ec9e411889337453432be70e7f6e Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 30 May 2024 13:05:09 +0100 Subject: [PATCH 112/373] Refactoring: No need to keep dialect-dependent parser state here. --- src/libre/dialect/glob/parser.c | 114 +++-- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/like/parser.c | 114 +++-- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/literal/parser.c | 90 ++-- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/native/parser.c | 438 +++++++++---------- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/pcre/parser.c | 666 ++++++++++++++--------------- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/sql/parser.c | 338 +++++++-------- src/libre/dialect/sql/parser.h | 4 +- src/libre/parser.act | 44 +- 13 files changed, 872 insertions(+), 956 deletions(-) diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index f6504b854..0de67cf26 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -124,11 +124,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -210,51 +205,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -262,21 +255,21 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL (TOK_ERROR) #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ act_state->lex_tok = tok; } while (0) #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) -#line 280 "src/libre/dialect/glob/parser.c" +#line 273 "src/libre/dialect/glob/parser.c" #ifndef ERROR_TERMINAL @@ -311,13 +304,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 321 "src/libre/dialect/glob/parser.c" +#line 314 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 115 */ @@ -355,24 +348,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 364 "src/libre/dialect/glob/parser.c" +#line 357 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 376 "src/libre/dialect/glob/parser.c" +#line 369 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -385,7 +378,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -398,20 +391,20 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 402 "src/libre/dialect/glob/parser.c" +#line 395 "src/libre/dialect/glob/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 415 "src/libre/dialect/glob/parser.c" +#line 408 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -425,38 +418,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 434 "src/libre/dialect/glob/parser.c" +#line 427 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 446 "src/libre/dialect/glob/parser.c" +#line 439 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 815 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 455 "src/libre/dialect/glob/parser.c" +#line 448 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -470,7 +463,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 474 "src/libre/dialect/glob/parser.c" +#line 467 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -485,26 +478,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 711 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 496 "src/libre/dialect/glob/parser.c" +#line 489 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 508 "src/libre/dialect/glob/parser.c" +#line 501 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -530,11 +523,11 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 538 "src/libre/dialect/glob/parser.c" +#line 531 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 119 */ @@ -544,14 +537,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 555 "src/libre/dialect/glob/parser.c" +#line 548 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -565,14 +558,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 576 "src/libre/dialect/glob/parser.c" +#line 569 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -582,14 +575,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 593 "src/libre/dialect/glob/parser.c" +#line 586 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ @@ -608,14 +601,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 760 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 619 "src/libre/dialect/glob/parser.c" +#line 612 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -633,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1059 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -691,11 +684,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -791,6 +779,6 @@ ZL0:; return NULL; } -#line 795 "src/libre/dialect/glob/parser.c" +#line 783 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 105c27e82..cfcce16b7 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 288 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1216 "src/libre/parser.act" +#line 1204 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index 12dae617a..eec930c0f 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -124,11 +124,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -210,51 +205,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -262,21 +255,21 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL (TOK_ERROR) #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ act_state->lex_tok = tok; } while (0) #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) -#line 280 "src/libre/dialect/like/parser.c" +#line 273 "src/libre/dialect/like/parser.c" #ifndef ERROR_TERMINAL @@ -311,13 +304,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 321 "src/libre/dialect/like/parser.c" +#line 314 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 115 */ @@ -355,24 +348,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 364 "src/libre/dialect/like/parser.c" +#line 357 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 376 "src/libre/dialect/like/parser.c" +#line 369 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -385,7 +378,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -398,20 +391,20 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 402 "src/libre/dialect/like/parser.c" +#line 395 "src/libre/dialect/like/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 415 "src/libre/dialect/like/parser.c" +#line 408 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -425,38 +418,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 434 "src/libre/dialect/like/parser.c" +#line 427 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 446 "src/libre/dialect/like/parser.c" +#line 439 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 815 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 455 "src/libre/dialect/like/parser.c" +#line 448 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -470,7 +463,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 474 "src/libre/dialect/like/parser.c" +#line 467 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -485,26 +478,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 711 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 496 "src/libre/dialect/like/parser.c" +#line 489 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 508 "src/libre/dialect/like/parser.c" +#line 501 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -530,11 +523,11 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 538 "src/libre/dialect/like/parser.c" +#line 531 "src/libre/dialect/like/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 119 */ @@ -544,14 +537,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 555 "src/libre/dialect/like/parser.c" +#line 548 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -565,14 +558,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 576 "src/libre/dialect/like/parser.c" +#line 569 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -582,14 +575,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 593 "src/libre/dialect/like/parser.c" +#line 586 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ @@ -608,14 +601,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 760 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 619 "src/libre/dialect/like/parser.c" +#line 612 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -633,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1059 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -691,11 +684,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -791,6 +779,6 @@ ZL0:; return NULL; } -#line 795 "src/libre/dialect/like/parser.c" +#line 783 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index bb01f69f1..4c7949714 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 288 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1216 "src/libre/parser.act" +#line 1204 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 404859678..7b25cbc6b 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -124,11 +124,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -210,51 +205,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -262,21 +255,21 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL (TOK_ERROR) #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ act_state->lex_tok = tok; } while (0) #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) -#line 280 "src/libre/dialect/literal/parser.c" +#line 273 "src/libre/dialect/literal/parser.c" #ifndef ERROR_TERMINAL @@ -311,13 +304,13 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 321 "src/libre/dialect/literal/parser.c" +#line 314 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 114 */ @@ -356,11 +349,11 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 364 "src/libre/dialect/literal/parser.c" +#line 357 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 117 */ @@ -370,14 +363,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 381 "src/libre/dialect/literal/parser.c" +#line 374 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); @@ -391,14 +384,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 402 "src/libre/dialect/literal/parser.c" +#line 395 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -408,14 +401,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* END OF INLINE: 117 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 419 "src/libre/dialect/literal/parser.c" +#line 412 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 119 */ @@ -434,14 +427,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 760 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 445 "src/libre/dialect/literal/parser.c" +#line 438 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -474,7 +467,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -487,7 +480,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ZIc = lex_state->buf.a[0]; -#line 491 "src/libre/dialect/literal/parser.c" +#line 484 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -497,14 +490,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 508 "src/libre/dialect/literal/parser.c" +#line 501 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -513,26 +506,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 711 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 524 "src/libre/dialect/literal/parser.c" +#line 517 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 536 "src/libre/dialect/literal/parser.c" +#line 529 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -546,7 +539,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1059 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -604,11 +597,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -704,6 +692,6 @@ ZL0:; return NULL; } -#line 708 "src/libre/dialect/literal/parser.c" +#line 696 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index 0a98a2362..b1c20e4bb 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 288 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1216 "src/libre/parser.act" +#line 1204 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 2b204045a..7aa79896c 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -124,11 +124,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -210,51 +205,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -262,21 +255,21 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL (TOK_ERROR) #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ act_state->lex_tok = tok; } while (0) #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) -#line 280 "src/libre/dialect/native/parser.c" +#line 273 "src/libre/dialect/native/parser.c" #ifndef ERROR_TERMINAL @@ -333,7 +326,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -346,7 +339,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 350 "src/libre/dialect/native/parser.c" +#line 343 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -356,7 +349,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 393 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -382,7 +375,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 386 "src/libre/dialect/native/parser.c" +#line 379 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -392,7 +385,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 534 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -440,7 +433,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 444 "src/libre/dialect/native/parser.c" +#line 437 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -450,7 +443,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 491 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -493,7 +486,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 497 "src/libre/dialect/native/parser.c" +#line 490 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -506,12 +499,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 141 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 515 "src/libre/dialect/native/parser.c" +#line 508 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -538,7 +531,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI216 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -546,23 +539,23 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI216; (void) ZIend; -#line 550 "src/libre/dialect/native/parser.c" +#line 543 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 561 "src/libre/dialect/native/parser.c" +#line 554 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -577,7 +570,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 581 "src/libre/dialect/native/parser.c" +#line 574 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -593,7 +586,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 634 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -613,7 +606,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 617 "src/libre/dialect/native/parser.c" +#line 610 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -625,7 +618,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI219 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -633,7 +626,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI219; (void) ZIend; -#line 637 "src/libre/dialect/native/parser.c" +#line 630 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -643,17 +636,17 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 652 "src/libre/dialect/native/parser.c" +#line 645 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -668,7 +661,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 672 "src/libre/dialect/native/parser.c" +#line 665 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -703,13 +696,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 713 "src/libre/dialect/native/parser.c" +#line 706 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -743,7 +736,7 @@ p_154(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI155 = '-'; ZI156 = lex_state->lx.start; @@ -753,7 +746,7 @@ p_154(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI156; (void) ZI157; -#line 757 "src/libre/dialect/native/parser.c" +#line 750 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -767,14 +760,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 725 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 778 "src/libre/dialect/native/parser.c" +#line 771 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -802,13 +795,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 812 "src/libre/dialect/native/parser.c" +#line 805 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 223 */ @@ -856,7 +849,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -869,7 +862,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 873 "src/libre/dialect/native/parser.c" +#line 866 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -882,7 +875,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 393 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -908,7 +901,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI111; (void) ZI112; -#line 912 "src/libre/dialect/native/parser.c" +#line 905 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -921,7 +914,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 534 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -969,7 +962,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 973 "src/libre/dialect/native/parser.c" +#line 966 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -982,7 +975,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 491 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1025,7 +1018,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1029 "src/libre/dialect/native/parser.c" +#line 1022 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1038,14 +1031,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 109 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1049 "src/libre/dialect/native/parser.c" +#line 1042 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1071,7 +1064,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1084,7 +1077,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI259 = lex_state->buf.a[0]; -#line 1088 "src/libre/dialect/native/parser.c" +#line 1081 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1103,7 +1096,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 393 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1129,7 +1122,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI248; (void) ZI249; -#line 1133 "src/libre/dialect/native/parser.c" +#line 1126 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1148,7 +1141,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 534 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1196,7 +1189,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI255 = (char) (unsigned char) u; -#line 1200 "src/libre/dialect/native/parser.c" +#line 1193 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1215,7 +1208,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 654 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZI243 = DIALECT_CLASS(lex_state->buf.a); if (ZI243 == NULL) { @@ -1229,7 +1222,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI244; (void) ZI245; -#line 1233 "src/libre/dialect/native/parser.c" +#line 1226 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1248,7 +1241,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 491 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1291,7 +1284,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI251 = (char) (unsigned char) u; -#line 1295 "src/libre/dialect/native/parser.c" +#line 1288 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1332,7 +1325,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 654 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1346,7 +1339,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1350 "src/libre/dialect/native/parser.c" +#line 1343 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1356,12 +1349,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 851 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1365 "src/libre/dialect/native/parser.c" +#line 1358 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1396,7 +1389,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 325 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI167 = lex_state->lx.end; @@ -1404,20 +1397,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI167; -#line 1408 "src/libre/dialect/native/parser.c" +#line 1401 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1421 "src/libre/dialect/native/parser.c" +#line 1414 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1437,7 +1430,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 341 "src/libre/parser.act" +#line 334 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI186 = lex_state->lx.end; @@ -1445,30 +1438,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI186; -#line 1449 "src/libre/dialect/native/parser.c" +#line 1442 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1462 "src/libre/dialect/native/parser.c" +#line 1455 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 892 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1472 "src/libre/dialect/native/parser.c" +#line 1465 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1478,13 +1471,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1488 "src/libre/dialect/native/parser.c" +#line 1481 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1500,7 +1493,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 333 "src/libre/parser.act" +#line 326 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI178 = lex_state->lx.end; @@ -1508,26 +1501,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI178; -#line 1512 "src/libre/dialect/native/parser.c" +#line 1505 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1525 "src/libre/dialect/native/parser.c" +#line 1518 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 973 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1565,7 +1558,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1569 "src/libre/dialect/native/parser.c" +#line 1562 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ p_180 (flags, lex_state, act_state, err, &ZItmp); @@ -1584,7 +1577,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 349 "src/libre/parser.act" +#line 342 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI193 = lex_state->lx.end; @@ -1592,26 +1585,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI193; -#line 1596 "src/libre/dialect/native/parser.c" +#line 1589 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1609 "src/libre/dialect/native/parser.c" +#line 1602 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 973 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1649,16 +1642,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1653 "src/libre/dialect/native/parser.c" +#line 1646 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 892 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1662 "src/libre/dialect/native/parser.c" +#line 1655 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1668,13 +1661,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1678 "src/libre/dialect/native/parser.c" +#line 1671 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1700,7 +1693,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 357 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI200 = ']'; ZI201 = lex_state->lx.start; @@ -1710,18 +1703,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI201; (void) ZIend; -#line 1714 "src/libre/dialect/native/parser.c" +#line 1707 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 774 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1725 "src/libre/dialect/native/parser.c" +#line 1718 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1735,7 +1728,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 367 "src/libre/parser.act" +#line 360 "src/libre/parser.act" ZIcrange = '-'; ZI203 = lex_state->lx.start; @@ -1745,41 +1738,41 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI203; (void) ZIend; -#line 1749 "src/libre/dialect/native/parser.c" +#line 1742 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 1762 "src/libre/dialect/native/parser.c" +#line 1755 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 1773 "src/libre/dialect/native/parser.c" +#line 1766 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 774 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1783 "src/libre/dialect/native/parser.c" +#line 1776 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1792,14 +1785,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 732 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 1803 "src/libre/dialect/native/parser.c" +#line 1796 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -1828,7 +1821,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -1838,7 +1831,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI181; -#line 1842 "src/libre/dialect/native/parser.c" +#line 1835 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1849,14 +1842,14 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 1860 "src/libre/dialect/native/parser.c" +#line 1853 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1872,17 +1865,17 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 1881 "src/libre/dialect/native/parser.c" +#line 1874 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI183 = '-'; ZI184 = lex_state->lx.start; @@ -1892,7 +1885,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI184; (void) ZI185; -#line 1896 "src/libre/dialect/native/parser.c" +#line 1889 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1903,7 +1896,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -1932,7 +1925,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1936 "src/libre/dialect/native/parser.c" +#line 1929 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1942,13 +1935,13 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 182 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 1952 "src/libre/dialect/native/parser.c" +#line 1945 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -1984,7 +1977,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI149 = lex_state->lx.start; @@ -1994,18 +1987,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI149; (void) ZIend; -#line 1998 "src/libre/dialect/native/parser.c" +#line 1991 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2009 "src/libre/dialect/native/parser.c" +#line 2002 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2057,7 +2050,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2071,7 +2064,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2075 "src/libre/dialect/native/parser.c" +#line 2068 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2095,14 +2088,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2106 "src/libre/dialect/native/parser.c" +#line 2099 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2116,14 +2109,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2127 "src/libre/dialect/native/parser.c" +#line 2120 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2136,26 +2129,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 718 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2147 "src/libre/dialect/native/parser.c" +#line 2140 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2159 "src/libre/dialect/native/parser.c" +#line 2152 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2177,14 +2170,14 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2188 "src/libre/dialect/native/parser.c" +#line 2181 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2201,17 +2194,17 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2210 "src/libre/dialect/native/parser.c" +#line 2203 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI196 = '-'; ZI197 = lex_state->lx.start; @@ -2221,7 +2214,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI197; (void) ZI198; -#line 2225 "src/libre/dialect/native/parser.c" +#line 2218 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2232,17 +2225,17 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2241 "src/libre/dialect/native/parser.c" +#line 2234 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2271,7 +2264,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 2275 "src/libre/dialect/native/parser.c" +#line 2268 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2301,11 +2294,11 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2309 "src/libre/dialect/native/parser.c" +#line 2302 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -2315,14 +2308,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2326 "src/libre/dialect/native/parser.c" +#line 2319 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 231 */ @@ -2341,14 +2334,14 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 760 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 2352 "src/libre/dialect/native/parser.c" +#line 2345 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2429,13 +2422,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2439 "src/libre/dialect/native/parser.c" +#line 2432 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 229 */ @@ -2460,14 +2453,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 718 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 2471 "src/libre/dialect/native/parser.c" +#line 2464 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2492,7 +2485,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 377 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI263 = lex_state->lx.start; ZI264 = lex_state->lx.end; @@ -2500,7 +2493,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI263; (void) ZI264; -#line 2504 "src/libre/dialect/native/parser.c" +#line 2497 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2508,7 +2501,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 634 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -2528,7 +2521,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2532 "src/libre/dialect/native/parser.c" +#line 2525 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2548,11 +2541,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 823 "src/libre/parser.act" +#line 816 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 2556 "src/libre/dialect/native/parser.c" +#line 2549 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2562,11 +2555,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 819 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2570 "src/libre/dialect/native/parser.c" +#line 2563 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2576,11 +2569,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 815 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2584 "src/libre/dialect/native/parser.c" +#line 2577 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2589,11 +2582,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 827 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2597 "src/libre/dialect/native/parser.c" +#line 2590 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2606,23 +2599,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 704 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 2617 "src/libre/dialect/native/parser.c" +#line 2610 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 827 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2626 "src/libre/dialect/native/parser.c" +#line 2619 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2647,24 +2640,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2656 "src/libre/dialect/native/parser.c" +#line 2649 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2668 "src/libre/dialect/native/parser.c" +#line 2661 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2674,14 +2667,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 949 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 2685 "src/libre/dialect/native/parser.c" +#line 2678 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -2694,11 +2687,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2702 "src/libre/dialect/native/parser.c" +#line 2695 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2708,14 +2701,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 2719 "src/libre/dialect/native/parser.c" +#line 2712 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2732,14 +2725,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 942 "src/libre/parser.act" +#line 935 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 2743 "src/libre/dialect/native/parser.c" +#line 2736 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2772,26 +2765,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 711 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 2783 "src/libre/dialect/native/parser.c" +#line 2776 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2795 "src/libre/dialect/native/parser.c" +#line 2788 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2813,14 +2806,14 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI243)); if ((ZInode) == NULL) { goto ZL1; } -#line 2824 "src/libre/dialect/native/parser.c" +#line 2817 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2833,12 +2826,12 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 851 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI243); -#line 2842 "src/libre/dialect/native/parser.c" +#line 2835 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_154 (flags, lex_state, act_state, err); @@ -2849,17 +2842,17 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 779 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); -#line 2858 "src/libre/dialect/native/parser.c" +#line 2851 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2888,7 +2881,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2892 "src/libre/dialect/native/parser.c" +#line 2885 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2915,14 +2908,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2926 "src/libre/dialect/native/parser.c" +#line 2919 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2949,14 +2942,14 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI247)); if ((ZInode) == NULL) { goto ZL1; } -#line 2960 "src/libre/dialect/native/parser.c" +#line 2953 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2969,12 +2962,12 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI247); -#line 2978 "src/libre/dialect/native/parser.c" +#line 2971 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_154 (flags, lex_state, act_state, err); @@ -2985,17 +2978,17 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 779 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); -#line 2994 "src/libre/dialect/native/parser.c" +#line 2987 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -3024,7 +3017,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3028 "src/libre/dialect/native/parser.c" +#line 3021 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3042,7 +3035,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1059 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -3100,11 +3093,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -3200,6 +3188,6 @@ ZL0:; return NULL; } -#line 3204 "src/libre/dialect/native/parser.c" +#line 3192 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 408a2c546..c833c9629 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 288 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1216 "src/libre/parser.act" +#line 1204 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 137be4555..12987d311 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -124,11 +124,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -210,51 +205,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -262,21 +255,21 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL (TOK_ERROR) #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ act_state->lex_tok = tok; } while (0) #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) -#line 280 "src/libre/dialect/pcre/parser.c" +#line 273 "src/libre/dialect/pcre/parser.c" #ifndef ERROR_TERMINAL @@ -332,21 +325,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 672 "src/libre/parser.act" +#line 665 "src/libre/parser.act" ZIc = RE_EXTENDED; -#line 340 "src/libre/dialect/pcre/parser.c" +#line 333 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_EXTENDED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 807 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 350 "src/libre/dialect/pcre/parser.c" +#line 343 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -363,21 +356,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 668 "src/libre/parser.act" +#line 661 "src/libre/parser.act" ZIc = RE_ICASE; -#line 371 "src/libre/dialect/pcre/parser.c" +#line 364 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_INSENSITIVE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 807 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 381 "src/libre/dialect/pcre/parser.c" +#line 374 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -388,21 +381,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 676 "src/libre/parser.act" +#line 669 "src/libre/parser.act" ZIc = RE_SINGLE; -#line 396 "src/libre/dialect/pcre/parser.c" +#line 389 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_SINGLE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 807 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 406 "src/libre/dialect/pcre/parser.c" +#line 399 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -413,14 +406,14 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 746 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; } goto ZL1; -#line 424 "src/libre/dialect/pcre/parser.c" +#line 417 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unknown-flag */ } @@ -458,7 +451,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -471,7 +464,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 475 "src/libre/dialect/pcre/parser.c" +#line 468 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -481,7 +474,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 449 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -523,20 +516,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 527 "src/libre/dialect/pcre/parser.c" +#line 520 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 540 "src/libre/dialect/pcre/parser.c" +#line 533 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -545,7 +538,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 393 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -571,7 +564,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 575 "src/libre/dialect/pcre/parser.c" +#line 568 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -581,7 +574,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 534 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -629,7 +622,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 633 "src/libre/dialect/pcre/parser.c" +#line 626 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -639,7 +632,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 491 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -682,7 +675,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 686 "src/libre/dialect/pcre/parser.c" +#line 679 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -692,7 +685,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 433 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -708,20 +701,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 712 "src/libre/dialect/pcre/parser.c" +#line 705 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 725 "src/libre/dialect/pcre/parser.c" +#line 718 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -733,12 +726,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 155 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 742 "src/libre/dialect/pcre/parser.c" +#line 735 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -769,13 +762,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 779 "src/libre/dialect/pcre/parser.c" +#line 772 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -829,13 +822,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 839 "src/libre/dialect/pcre/parser.c" +#line 832 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ } @@ -881,14 +874,14 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI290)); if ((ZInode) == NULL) { goto ZL1; } -#line 892 "src/libre/dialect/pcre/parser.c" +#line 885 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -901,12 +894,12 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 851 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI290); -#line 910 "src/libre/dialect/pcre/parser.c" +#line 903 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_168 (flags, lex_state, act_state, err); @@ -917,17 +910,17 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 779 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI291)); mark(&act_state->rangeend, &(ZIend)); -#line 926 "src/libre/dialect/pcre/parser.c" +#line 919 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -956,7 +949,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 960 "src/libre/dialect/pcre/parser.c" +#line 953 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -987,7 +980,7 @@ p_168(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI169 = '-'; ZI170 = lex_state->lx.start; @@ -997,7 +990,7 @@ p_168(flags flags, lex_state lex_state, act_state act_state, err err) (void) ZI170; (void) ZI171; -#line 1001 "src/libre/dialect/pcre/parser.c" +#line 994 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1011,14 +1004,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 725 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 1022 "src/libre/dialect/pcre/parser.c" +#line 1015 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1050,7 +1043,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1063,7 +1056,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; -#line 1067 "src/libre/dialect/pcre/parser.c" +#line 1060 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1076,7 +1069,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CONTROL */ { -#line 449 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1118,7 +1111,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI123; (void) ZI124; -#line 1122 "src/libre/dialect/pcre/parser.c" +#line 1115 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1131,7 +1124,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 393 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1157,7 +1150,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI113; (void) ZI114; -#line 1161 "src/libre/dialect/pcre/parser.c" +#line 1154 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1170,7 +1163,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 534 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1218,7 +1211,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1222 "src/libre/dialect/pcre/parser.c" +#line 1215 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1231,7 +1224,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: NOESC */ { -#line 419 "src/libre/parser.act" +#line 412 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1245,7 +1238,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI115; (void) ZI116; -#line 1249 "src/libre/dialect/pcre/parser.c" +#line 1242 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1258,7 +1251,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 491 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1301,7 +1294,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1305 "src/libre/dialect/pcre/parser.c" +#line 1298 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1314,7 +1307,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 433 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1330,20 +1323,20 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err (void) ZI125; (void) ZI126; -#line 1334 "src/libre/dialect/pcre/parser.c" +#line 1327 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1347 "src/libre/dialect/pcre/parser.c" +#line 1340 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1355,14 +1348,14 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 111 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1366 "src/libre/dialect/pcre/parser.c" +#line 1359 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1388,7 +1381,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1401,7 +1394,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI306 = lex_state->buf.a[0]; -#line 1405 "src/libre/dialect/pcre/parser.c" +#line 1398 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1420,7 +1413,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CONTROL */ { -#line 449 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1462,20 +1455,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI311; (void) ZI312; -#line 1466 "src/libre/dialect/pcre/parser.c" +#line 1459 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1479 "src/libre/dialect/pcre/parser.c" +#line 1472 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_317 (flags, lex_state, act_state, err, &ZI310, &ZI311, &ZInode); @@ -1493,7 +1486,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 393 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1519,7 +1512,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI295; (void) ZI296; -#line 1523 "src/libre/dialect/pcre/parser.c" +#line 1516 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1538,7 +1531,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 534 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1586,7 +1579,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI302 = (char) (unsigned char) u; -#line 1590 "src/libre/dialect/pcre/parser.c" +#line 1583 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1605,7 +1598,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 654 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZI290 = DIALECT_CLASS(lex_state->buf.a); if (ZI290 == NULL) { @@ -1619,7 +1612,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI291; (void) ZI292; -#line 1623 "src/libre/dialect/pcre/parser.c" +#line 1616 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; @@ -1638,7 +1631,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NOESC */ { -#line 419 "src/libre/parser.act" +#line 412 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1652,20 +1645,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI134; (void) ZI135; -#line 1656 "src/libre/dialect/pcre/parser.c" +#line 1649 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1669 "src/libre/dialect/pcre/parser.c" +#line 1662 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1678,7 +1671,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 491 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1721,7 +1714,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI298 = (char) (unsigned char) u; -#line 1725 "src/libre/dialect/pcre/parser.c" +#line 1718 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1740,7 +1733,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 433 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1756,20 +1749,20 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ (void) ZI315; (void) ZI316; -#line 1760 "src/libre/dialect/pcre/parser.c" +#line 1753 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1773 "src/libre/dialect/pcre/parser.c" +#line 1766 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ p_317 (flags, lex_state, act_state, err, &ZI314, &ZI315, &ZInode); @@ -1808,14 +1801,14 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 690 "src/libre/parser.act" +#line 683 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; } goto ZL1; -#line 1819 "src/libre/dialect/pcre/parser.c" +#line 1812 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1842,7 +1835,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 654 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1856,7 +1849,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1860 "src/libre/dialect/pcre/parser.c" +#line 1853 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1866,12 +1859,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 851 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1875 "src/libre/dialect/pcre/parser.c" +#line 1868 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1906,7 +1899,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 325 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI181 = lex_state->lx.end; @@ -1914,20 +1907,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI181; -#line 1918 "src/libre/dialect/pcre/parser.c" +#line 1911 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1931 "src/libre/dialect/pcre/parser.c" +#line 1924 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; @@ -1947,7 +1940,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 341 "src/libre/parser.act" +#line 334 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI200 = lex_state->lx.end; @@ -1955,30 +1948,30 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI200; -#line 1959 "src/libre/dialect/pcre/parser.c" +#line 1952 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1972 "src/libre/dialect/pcre/parser.c" +#line 1965 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 892 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1982 "src/libre/dialect/pcre/parser.c" +#line 1975 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -1988,13 +1981,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1998 "src/libre/dialect/pcre/parser.c" +#line 1991 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2010,7 +2003,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 333 "src/libre/parser.act" +#line 326 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI192 = lex_state->lx.end; @@ -2018,26 +2011,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI192; -#line 2022 "src/libre/dialect/pcre/parser.c" +#line 2015 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2035 "src/libre/dialect/pcre/parser.c" +#line 2028 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 973 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -2075,7 +2068,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2079 "src/libre/dialect/pcre/parser.c" +#line 2072 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ p_194 (flags, lex_state, act_state, err, &ZItmp); @@ -2094,7 +2087,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 349 "src/libre/parser.act" +#line 342 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI207 = lex_state->lx.end; @@ -2102,26 +2095,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIstart; (void) ZI207; -#line 2106 "src/libre/dialect/pcre/parser.c" +#line 2099 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2119 "src/libre/dialect/pcre/parser.c" +#line 2112 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 973 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -2159,16 +2152,16 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2163 "src/libre/dialect/pcre/parser.c" +#line 2156 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 892 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 2172 "src/libre/dialect/pcre/parser.c" +#line 2165 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); @@ -2178,13 +2171,13 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2188 "src/libre/dialect/pcre/parser.c" +#line 2181 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2210,7 +2203,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 357 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI214 = ']'; ZI215 = lex_state->lx.start; @@ -2220,18 +2213,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI215; (void) ZIend; -#line 2224 "src/libre/dialect/pcre/parser.c" +#line 2217 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 774 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2235 "src/libre/dialect/pcre/parser.c" +#line 2228 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2245,7 +2238,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 367 "src/libre/parser.act" +#line 360 "src/libre/parser.act" ZIcrange = '-'; ZI217 = lex_state->lx.start; @@ -2255,41 +2248,41 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZI217; (void) ZIend; -#line 2259 "src/libre/dialect/pcre/parser.c" +#line 2252 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 2272 "src/libre/dialect/pcre/parser.c" +#line 2265 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 2283 "src/libre/dialect/pcre/parser.c" +#line 2276 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 774 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2293 "src/libre/dialect/pcre/parser.c" +#line 2286 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2302,14 +2295,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 732 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 2313 "src/libre/dialect/pcre/parser.c" +#line 2306 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -2345,7 +2338,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI163 = lex_state->lx.start; @@ -2355,18 +2348,18 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st (void) ZI163; (void) ZIend; -#line 2359 "src/libre/dialect/pcre/parser.c" +#line 2352 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2370 "src/libre/dialect/pcre/parser.c" +#line 2363 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2408,14 +2401,14 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI314)); if ((ZInode) == NULL) { goto ZL1; } -#line 2419 "src/libre/dialect/pcre/parser.c" +#line 2412 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2428,12 +2421,12 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI314); -#line 2437 "src/libre/dialect/pcre/parser.c" +#line 2430 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_168 (flags, lex_state, act_state, err); @@ -2444,17 +2437,17 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 779 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI315)); mark(&act_state->rangeend, &(ZIend)); -#line 2453 "src/libre/dialect/pcre/parser.c" +#line 2446 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2483,7 +2476,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2487 "src/libre/dialect/pcre/parser.c" +#line 2480 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2529,16 +2522,16 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* BEGINNING OF ACTION: count-one */ { -#line 827 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 2537 "src/libre/dialect/pcre/parser.c" +#line 2530 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2552,7 +2545,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2556 "src/libre/dialect/pcre/parser.c" +#line 2549 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2585,7 +2578,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI256 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2593,23 +2586,23 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI256; (void) ZIend; -#line 2597 "src/libre/dialect/pcre/parser.c" +#line 2590 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2608 "src/libre/dialect/pcre/parser.c" +#line 2601 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2624,7 +2617,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 2628 "src/libre/dialect/pcre/parser.c" +#line 2621 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2663,14 +2656,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2674 "src/libre/dialect/pcre/parser.c" +#line 2667 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2684,26 +2677,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 718 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2695 "src/libre/dialect/pcre/parser.c" +#line 2688 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2707 "src/libre/dialect/pcre/parser.c" +#line 2700 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2729,7 +2722,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2737,32 +2730,32 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI261; (void) ZIend; -#line 2741 "src/libre/dialect/pcre/parser.c" +#line 2734 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2752 "src/libre/dialect/pcre/parser.c" +#line 2745 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ { -#line 811 "src/libre/parser.act" +#line 804 "src/libre/parser.act" (ZIn) = AST_COUNT_UNBOUNDED; -#line 2761 "src/libre/dialect/pcre/parser.c" +#line 2754 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2777,7 +2770,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2781 "src/libre/dialect/pcre/parser.c" +#line 2774 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2790,7 +2783,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: COUNT */ { -#line 634 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -2810,7 +2803,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 2814 "src/libre/dialect/pcre/parser.c" +#line 2807 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -2818,7 +2811,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI259 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2826,7 +2819,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (void) ZI259; (void) ZIend; -#line 2830 "src/libre/dialect/pcre/parser.c" +#line 2823 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -2836,17 +2829,17 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2845 "src/libre/dialect/pcre/parser.c" +#line 2838 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2861,7 +2854,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2865 "src/libre/dialect/pcre/parser.c" +#line 2858 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2892,7 +2885,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -2902,7 +2895,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZIrstart; (void) ZI195; -#line 2906 "src/libre/dialect/pcre/parser.c" +#line 2899 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2913,14 +2906,14 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2924 "src/libre/dialect/pcre/parser.c" +#line 2917 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2936,17 +2929,17 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 2945 "src/libre/dialect/pcre/parser.c" +#line 2938 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI197 = '-'; ZI198 = lex_state->lx.start; @@ -2956,7 +2949,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI198; (void) ZI199; -#line 2960 "src/libre/dialect/pcre/parser.c" +#line 2953 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2967,7 +2960,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2996,7 +2989,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 3000 "src/libre/dialect/pcre/parser.c" +#line 2993 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3006,13 +2999,13 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 196 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 3016 "src/libre/dialect/pcre/parser.c" +#line 3009 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -3051,20 +3044,20 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 803 "src/libre/parser.act" +#line 796 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; -#line 3059 "src/libre/dialect/pcre/parser.c" +#line 3052 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 803 "src/libre/parser.act" +#line 796 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; -#line 3068 "src/libre/dialect/pcre/parser.c" +#line 3061 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF INLINE: 231 */ @@ -3117,7 +3110,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 933 "src/libre/parser.act" +#line 926 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3126,19 +3119,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3130 "src/libre/dialect/pcre/parser.c" +#line 3123 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL5; } -#line 3142 "src/libre/dialect/pcre/parser.c" +#line 3135 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3151,16 +3144,16 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 925 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIflags) = *flags; -#line 3159 "src/libre/dialect/pcre/parser.c" +#line 3152 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 933 "src/libre/parser.act" +#line 926 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3169,7 +3162,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3173 "src/libre/dialect/pcre/parser.c" +#line 3166 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3179,11 +3172,11 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 929 "src/libre/parser.act" +#line 922 "src/libre/parser.act" *flags = (ZIflags); -#line 3187 "src/libre/dialect/pcre/parser.c" +#line 3180 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3204,26 +3197,26 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 753 "src/libre/parser.act" +#line 746 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; } goto ZL1; -#line 3215 "src/libre/dialect/pcre/parser.c" +#line 3208 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 3227 "src/libre/dialect/pcre/parser.c" +#line 3220 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3257,7 +3250,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3271,7 +3264,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3275 "src/libre/dialect/pcre/parser.c" +#line 3268 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ /* BEGINNING OF INLINE: 264 */ @@ -3282,14 +3275,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3293 "src/libre/dialect/pcre/parser.c" +#line 3286 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3299,14 +3292,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 767 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3310 "src/libre/dialect/pcre/parser.c" +#line 3303 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3391,7 +3384,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 654 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3405,7 +3398,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3409 "src/libre/dialect/pcre/parser.c" +#line 3402 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3415,14 +3408,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3426 "src/libre/dialect/pcre/parser.c" +#line 3419 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3446,14 +3439,14 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3457 "src/libre/dialect/pcre/parser.c" +#line 3450 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3470,17 +3463,17 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3479 "src/libre/dialect/pcre/parser.c" +#line 3472 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI210 = '-'; ZI211 = lex_state->lx.start; @@ -3490,7 +3483,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs (void) ZI211; (void) ZI212; -#line 3494 "src/libre/dialect/pcre/parser.c" +#line 3487 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3501,17 +3494,17 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3510 "src/libre/dialect/pcre/parser.c" +#line 3503 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -3540,7 +3533,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } -#line 3544 "src/libre/dialect/pcre/parser.c" +#line 3537 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3573,13 +3566,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3583 "src/libre/dialect/pcre/parser.c" +#line 3576 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 278 */ @@ -3604,14 +3597,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 718 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 3615 "src/libre/dialect/pcre/parser.c" +#line 3608 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3636,7 +3629,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 377 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI318 = lex_state->lx.start; ZI319 = lex_state->lx.end; @@ -3644,7 +3637,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI318; (void) ZI319; -#line 3648 "src/libre/dialect/pcre/parser.c" +#line 3641 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3652,7 +3645,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 634 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -3672,7 +3665,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3676 "src/libre/dialect/pcre/parser.c" +#line 3669 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3692,11 +3685,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 823 "src/libre/parser.act" +#line 816 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 3700 "src/libre/dialect/pcre/parser.c" +#line 3693 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3706,11 +3699,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 819 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3714 "src/libre/dialect/pcre/parser.c" +#line 3707 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3720,11 +3713,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 815 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3728 "src/libre/dialect/pcre/parser.c" +#line 3721 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3739,23 +3732,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 704 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 3750 "src/libre/dialect/pcre/parser.c" +#line 3743 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 827 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 3759 "src/libre/dialect/pcre/parser.c" +#line 3752 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3781,11 +3774,11 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3789 "src/libre/dialect/pcre/parser.c" +#line 3782 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3795,14 +3788,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3806 "src/libre/dialect/pcre/parser.c" +#line 3799 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 280 */ @@ -3821,14 +3814,14 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 760 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 3832 "src/libre/dialect/pcre/parser.c" +#line 3825 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -3857,24 +3850,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3866 "src/libre/dialect/pcre/parser.c" +#line 3859 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3878 "src/libre/dialect/pcre/parser.c" +#line 3871 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3884,14 +3877,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 949 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 3895 "src/libre/dialect/pcre/parser.c" +#line 3888 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3901,7 +3894,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 956 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3911,7 +3904,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3915 "src/libre/dialect/pcre/parser.c" +#line 3908 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3929,134 +3922,134 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 794 "src/libre/parser.act" +#line 787 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3938 "src/libre/dialect/pcre/parser.c" +#line 3931 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 3950 "src/libre/dialect/pcre/parser.c" +#line 3943 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { goto ZL1; } -#line 3962 "src/libre/dialect/pcre/parser.c" +#line 3955 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 896 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIcr) = '\r'; -#line 3971 "src/libre/dialect/pcre/parser.c" +#line 3964 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { goto ZL1; } -#line 3983 "src/libre/dialect/pcre/parser.c" +#line 3976 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 900 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZInl) = '\n'; -#line 3992 "src/libre/dialect/pcre/parser.c" +#line 3985 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { goto ZL1; } -#line 4004 "src/libre/dialect/pcre/parser.c" +#line 3997 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4015 "src/libre/dialect/pcre/parser.c" +#line 4008 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4026 "src/libre/dialect/pcre/parser.c" +#line 4019 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 4038 "src/libre/dialect/pcre/parser.c" +#line 4031 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4049 "src/libre/dialect/pcre/parser.c" +#line 4042 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4060 "src/libre/dialect/pcre/parser.c" +#line 4053 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4070,20 +4063,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 925 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIflags) = *flags; -#line 4078 "src/libre/dialect/pcre/parser.c" +#line 4071 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 4087 "src/libre/dialect/pcre/parser.c" +#line 4080 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4093,23 +4086,23 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 929 "src/libre/parser.act" +#line 922 "src/libre/parser.act" *flags = (ZIflags); -#line 4101 "src/libre/dialect/pcre/parser.c" +#line 4094 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 4113 "src/libre/dialect/pcre/parser.c" +#line 4106 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4126,14 +4119,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 942 "src/libre/parser.act" +#line 935 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 4137 "src/libre/dialect/pcre/parser.c" +#line 4130 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4185,26 +4178,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 711 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 4196 "src/libre/dialect/pcre/parser.c" +#line 4189 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 4208 "src/libre/dialect/pcre/parser.c" +#line 4201 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4230,14 +4223,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4241 "src/libre/dialect/pcre/parser.c" +#line 4234 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4251,14 +4244,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4262 "src/libre/dialect/pcre/parser.c" +#line 4255 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4294,25 +4287,25 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4305 "src/libre/dialect/pcre/parser.c" +#line 4298 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4316 "src/libre/dialect/pcre/parser.c" +#line 4309 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4326,7 +4319,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1059 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -4384,11 +4377,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -4484,6 +4472,6 @@ ZL0:; return NULL; } -#line 4488 "src/libre/dialect/pcre/parser.c" +#line 4476 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index b808f0eb6..17a017297 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 288 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1216 "src/libre/parser.act" +#line 1204 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index a1158231a..dd60efe03 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -124,11 +124,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -210,51 +205,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -262,21 +255,21 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL (TOK_ERROR) #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ act_state->lex_tok = tok; } while (0) #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) -#line 280 "src/libre/dialect/sql/parser.c" +#line 273 "src/libre/dialect/sql/parser.c" #ifndef ERROR_TERMINAL @@ -318,13 +311,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: INVERT */ { -#line 309 "src/libre/parser.act" +#line 302 "src/libre/parser.act" ZI203 = '^'; (void) ZI203; -#line 328 "src/libre/dialect/sql/parser.c" +#line 321 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; @@ -344,7 +337,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI114 = lex_state->lx.start; @@ -354,31 +347,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ (void) ZI114; (void) ZI115; -#line 358 "src/libre/dialect/sql/parser.c" +#line 351 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 371 "src/libre/dialect/sql/parser.c" +#line 364 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 382 "src/libre/dialect/sql/parser.c" +#line 375 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -408,11 +401,11 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 416 "src/libre/dialect/sql/parser.c" +#line 409 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -422,14 +415,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 433 "src/libre/dialect/sql/parser.c" +#line 426 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 191 */ @@ -448,14 +441,14 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 760 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 459 "src/libre/dialect/sql/parser.c" +#line 452 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -491,13 +484,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; } -#line 501 "src/libre/dialect/sql/parser.c" +#line 494 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -506,14 +499,14 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 697 "src/libre/parser.act" +#line 690 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; } goto ZL1; -#line 517 "src/libre/dialect/sql/parser.c" +#line 510 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } @@ -567,13 +560,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1047 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 577 "src/libre/dialect/sql/parser.c" +#line 570 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF INLINE: 183 */ @@ -613,7 +606,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -626,7 +619,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZI205 = lex_state->buf.a[0]; -#line 630 "src/libre/dialect/sql/parser.c" +#line 623 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -677,7 +670,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 325 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -685,7 +678,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 689 "src/libre/dialect/sql/parser.c" +#line 682 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -695,14 +688,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { goto ZL1; } -#line 706 "src/libre/dialect/sql/parser.c" +#line 699 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; @@ -720,7 +713,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 357 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI154 = ']'; ZIclose__start = lex_state->lx.start; @@ -730,18 +723,18 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 734 "src/libre/dialect/sql/parser.c" +#line 727 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 774 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 745 "src/libre/dialect/sql/parser.c" +#line 738 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ ZInode = ZIclass; @@ -755,26 +748,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: INVERT */ { -#line 309 "src/libre/parser.act" +#line 302 "src/libre/parser.act" ZI158 = '^'; (void) ZI158; -#line 765 "src/libre/dialect/sql/parser.c" +#line 758 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { goto ZL3; } -#line 778 "src/libre/dialect/sql/parser.c" +#line 771 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; @@ -796,7 +789,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 357 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI163 = ']'; ZIclose__start = lex_state->lx.start; @@ -806,7 +799,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIclose__start; (void) ZIclose__end; -#line 810 "src/libre/dialect/sql/parser.c" +#line 803 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -816,12 +809,12 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 774 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); -#line 825 "src/libre/dialect/sql/parser.c" +#line 818 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-group */ } @@ -830,14 +823,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 732 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL3; -#line 841 "src/libre/dialect/sql/parser.c" +#line 834 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } @@ -846,14 +839,14 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 162 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 966 "src/libre/parser.act" +#line 959 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { goto ZL3; } -#line 857 "src/libre/dialect/sql/parser.c" +#line 850 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -869,26 +862,26 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 732 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 880 "src/libre/dialect/sql/parser.c" +#line 873 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 892 "src/libre/dialect/sql/parser.c" +#line 885 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -924,7 +917,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -938,7 +931,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 942 "src/libre/dialect/sql/parser.c" +#line 935 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -961,14 +954,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 874 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 972 "src/libre/dialect/sql/parser.c" +#line 965 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -982,26 +975,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 718 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 993 "src/libre/dialect/sql/parser.c" +#line 986 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 1005 "src/libre/dialect/sql/parser.c" +#line 998 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1026,7 +1019,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI117 = lex_state->lx.start; @@ -1036,36 +1029,36 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp (void) ZI117; (void) ZI118; -#line 1040 "src/libre/dialect/sql/parser.c" +#line 1033 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1053 "src/libre/dialect/sql/parser.c" +#line 1046 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 1064 "src/libre/dialect/sql/parser.c" +#line 1057 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 973 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1103,7 +1096,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1107 "src/libre/dialect/sql/parser.c" +#line 1100 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1112,7 +1105,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 973 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1150,7 +1143,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1154 "src/libre/dialect/sql/parser.c" +#line 1147 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1174,14 +1167,14 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI205)); if ((ZInode) == NULL) { goto ZL1; } -#line 1185 "src/libre/dialect/sql/parser.c" +#line 1178 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1199,17 +1192,17 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; (ZIa).u.literal.c = (unsigned char) (*ZI205); -#line 1208 "src/libre/dialect/sql/parser.c" +#line 1201 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 315 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI136 = '-'; ZI137 = lex_state->lx.start; @@ -1219,7 +1212,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (void) ZI137; (void) ZI138; -#line 1223 "src/libre/dialect/sql/parser.c" +#line 1216 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1227,7 +1220,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1240,7 +1233,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ZIcz = lex_state->buf.a[0]; -#line 1244 "src/libre/dialect/sql/parser.c" +#line 1237 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1250,27 +1243,27 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 846 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char) (ZIcz); -#line 1259 "src/libre/dialect/sql/parser.c" +#line 1252 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 779 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI206)); mark(&act_state->rangeend, &(ZIend)); -#line 1269 "src/libre/dialect/sql/parser.c" +#line 1262 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1011 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -1299,7 +1292,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 1303 "src/libre/dialect/sql/parser.c" +#line 1296 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1328,7 +1321,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI176 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1336,23 +1329,23 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI176; (void) ZIend; -#line 1340 "src/libre/dialect/sql/parser.c" +#line 1333 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1351 "src/libre/dialect/sql/parser.c" +#line 1344 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1367,7 +1360,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1371 "src/libre/dialect/sql/parser.c" +#line 1364 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1383,7 +1376,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 634 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -1403,7 +1396,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1407 "src/libre/dialect/sql/parser.c" +#line 1400 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1415,7 +1408,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 385 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI179 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1423,7 +1416,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (void) ZI179; (void) ZIend; -#line 1427 "src/libre/dialect/sql/parser.c" +#line 1420 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1433,17 +1426,17 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 784 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1442 "src/libre/dialect/sql/parser.c" +#line 1435 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 831 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1458,7 +1451,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1462 "src/libre/dialect/sql/parser.c" +#line 1455 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1493,13 +1486,13 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1053 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1503 "src/libre/dialect/sql/parser.c" +#line 1496 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: 189 */ @@ -1524,14 +1517,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 718 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 1535 "src/libre/dialect/sql/parser.c" +#line 1528 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1556,7 +1549,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 377 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI209 = lex_state->lx.start; ZI210 = lex_state->lx.end; @@ -1564,7 +1557,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, (void) ZI209; (void) ZI210; -#line 1568 "src/libre/dialect/sql/parser.c" +#line 1561 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1572,7 +1565,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 634 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -1592,7 +1585,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1596 "src/libre/dialect/sql/parser.c" +#line 1589 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1612,11 +1605,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 823 "src/libre/parser.act" +#line 816 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); -#line 1620 "src/libre/dialect/sql/parser.c" +#line 1613 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1626,11 +1619,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 819 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1634 "src/libre/dialect/sql/parser.c" +#line 1627 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1640,11 +1633,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 815 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1648 "src/libre/dialect/sql/parser.c" +#line 1641 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1653,11 +1646,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 827 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1661 "src/libre/dialect/sql/parser.c" +#line 1654 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1670,23 +1663,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 704 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 1681 "src/libre/dialect/sql/parser.c" +#line 1674 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 827 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); -#line 1690 "src/libre/dialect/sql/parser.c" +#line 1683 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1711,24 +1704,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1720 "src/libre/dialect/sql/parser.c" +#line 1713 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1732 "src/libre/dialect/sql/parser.c" +#line 1725 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1741,7 +1734,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* BEGINNING OF EXTRACT: CHAR */ { -#line 582 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1754,20 +1747,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ZIa = lex_state->buf.a[0]; -#line 1758 "src/libre/dialect/sql/parser.c" +#line 1751 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 881 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1771 "src/libre/dialect/sql/parser.c" +#line 1764 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1781,38 +1774,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 789 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1790 "src/libre/dialect/sql/parser.c" +#line 1783 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1802 "src/libre/dialect/sql/parser.c" +#line 1795 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 815 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1811 "src/libre/dialect/sql/parser.c" +#line 1804 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 904 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1826,7 +1819,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1830 "src/libre/dialect/sql/parser.c" +#line 1823 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1839,11 +1832,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 888 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 1847 "src/libre/dialect/sql/parser.c" +#line 1840 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1853,14 +1846,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 918 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 1864 "src/libre/dialect/sql/parser.c" +#line 1857 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1891,26 +1884,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 711 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 1902 "src/libre/dialect/sql/parser.c" +#line 1895 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 1914 "src/libre/dialect/sql/parser.c" +#line 1907 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1939,7 +1932,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 654 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1953,7 +1946,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act (void) ZI128; (void) ZI129; -#line 1957 "src/libre/dialect/sql/parser.c" +#line 1950 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1963,14 +1956,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1040 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 1974 "src/libre/dialect/sql/parser.c" +#line 1967 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1993,14 +1986,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 867 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2004 "src/libre/dialect/sql/parser.c" +#line 1997 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2014,14 +2007,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 860 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2025 "src/libre/dialect/sql/parser.c" +#line 2018 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2039,7 +2032,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1059 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -2097,11 +2090,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -2197,6 +2185,6 @@ ZL0:; return NULL; } -#line 2201 "src/libre/dialect/sql/parser.c" +#line 2189 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 7d292714c..cbbd5b111 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 288 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1216 "src/libre/parser.act" +#line 1204 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index af2d5724e..0156fbf08 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -133,11 +133,6 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; - -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; static void @@ -219,51 +214,49 @@ } static enum LX_TOKEN - current_token(struct lex_state *lex_state, struct act_state *act_state) + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) { #ifndef PCRE_DIALECT (void) lex_state; + (void) flags; return act_state->lex_tok; #else - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { case TOK_WHITESPACE: case TOK_NEWLINE: case TOK_MAYBE_COMMENT: return TOK_CHAR; default: - return tok; + return act_state->lex_tok; } } + int extended_mode_comment = 0; + restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { + if (extended_mode_comment) { + switch (act_state->lex_tok) { case TOK_EOF: case TOK_ERROR: case TOK_UNKNOWN: /* don't eat EOF or errors */ - return tok; + return act_state->lex_tok; case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; + extended_mode_comment = 0; /* fall through */ default: advance_lexer(lex_state, act_state); goto restart; } } else { - switch (tok) { + switch (act_state->lex_tok) { case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; + extended_mode_comment = 1; /* fall through */ case TOK_WHITESPACE: case TOK_NEWLINE: @@ -271,14 +264,14 @@ goto restart; default: - return tok; + return act_state->lex_tok; } } #endif } #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) - #define CURRENT_TERMINAL (current_token(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) #define ERROR_TERMINAL @$ERROR #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ @@ -1113,11 +1106,6 @@ lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; From 3435f508afe08c606e700d9ddc48efe58d449d74 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 30 May 2024 08:49:21 -0400 Subject: [PATCH 113/373] Move `FSM_DEFAULT_STATEALLOC` to fsm.c. --- src/libfsm/fsm.c | 3 +++ src/libfsm/internal.h | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index b9f092973..d027825ff 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -22,6 +22,9 @@ #include "capture.h" #include "endids.h" +/* guess for default state allocation */ +#define FSM_DEFAULT_STATEALLOC 128 + void free_contents(struct fsm *fsm) { diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 024bf99df..6e77510a7 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -44,9 +44,6 @@ struct state_array; #define FSM_CAPTURE_MAX INT_MAX -/* guess for default state allocation */ -#define FSM_DEFAULT_STATEALLOC 128 - struct fsm_edge { fsm_state_t state; /* destination */ unsigned char symbol; From 0a5e6fb873d4aa9e10f26e3d00363ac784cac1f4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 27 May 2024 17:45:40 +0100 Subject: [PATCH 114/373] Consolidate PCRE unsupported cases. My thinking here is that we don't need to categorise these independently from anything else we consider unsupported, because to the caller the reason something is unsupported doesn't matter. This way there's only one situation for a caller to keep track of, and in particular to not need to remember to update whenever we introduce more unsupported things. --- include/re/re.h | 2 -- src/libre/ast_analysis.c | 21 ++++++++++----------- src/libre/ast_analysis.h | 3 +-- src/libre/re.c | 6 ++---- src/libre/strerror.c | 2 -- tests/pcre/out48.err | 2 +- tests/pcre/out49.err | 2 +- 7 files changed, 15 insertions(+), 23 deletions(-) diff --git a/include/re/re.h b/include/re/re.h index f876f635a..deab6caed 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -46,8 +46,6 @@ enum re_errno { RE_EERRNO = 1 | RE_MISC, RE_EBADDIALECT = 2 | RE_MISC, RE_EBADGROUP = 3 | RE_MISC, - RE_EUNSUPCAPTUR = 4 | RE_MISC, - RE_EUNSUPPPCRE = 5 | RE_MISC, RE_ENEGRANGE = 0 | RE_MARK | RE_GROUP, RE_ENEGCOUNT = 1 | RE_MARK | RE_GROUP, diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 24db8b6d7..b2381e01b 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -783,7 +783,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: LITERAL: rejecting non-optional newline match after $ as unsupported\n", __func__); set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } } break; @@ -1010,8 +1010,8 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) all_set_past_always_consuming &= child_env.past_always_consuming; any_sat = 1; } - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE - || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED + || res == AST_ANALYSIS_ERROR_UNSUPPORTED) { continue; } else { return res; @@ -1201,7 +1201,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: SUBTRACT: rejecting non-optional newline match after $ as unsupported\n", __func__); set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } if (res != AST_ANALYSIS_OK) { @@ -1287,9 +1287,9 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) if (env->followed_by_consuming) { if (env->followed_by_consuming_newline) { LOG(3 - LOG_ANCHORING, - "%s: END anchor & followed_by_consuming, returning UNSUPPORTED_PCRE\n", + "%s: END anchor & followed_by_consuming, returning UNSUPPORTED\n", __func__); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } else { LOG(3 - LOG_ANCHORING, "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", @@ -1446,9 +1446,8 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) any_set_followed_by_consuming_newline |= child_env.followed_by_consuming_newline; any_sat = 1; } - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE - || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { - LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED_*, bubbling up\n", __func__); + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED, bubbling up\n", __func__); return res; } else { return res; @@ -1883,7 +1882,7 @@ analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_p * * An example input that triggers this is '^(($)|)+$' . */ set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } } @@ -1994,7 +1993,7 @@ analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_p && repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); LOG(3 - LOG_REPETITION_CASES, "%s: not yet implemented, skipping\n", __func__); - /* return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; */ + /* return AST_ANALYSIS_ERROR_UNSUPPORTED; */ } res = analysis_iter_repetition(n->u.group.e, outermost_repeat_parent, diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index 5390cce57..0628bd127 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -31,8 +31,7 @@ enum ast_analysis_res { AST_ANALYSIS_ERROR_NULL = -1, AST_ANALYSIS_ERROR_MEMORY = -2, - AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE = -3, - AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE = -4 + AST_ANALYSIS_ERROR_UNSUPPORTED = -3 }; enum ast_analysis_res diff --git a/src/libre/re.c b/src/libre/re.c index 499bbf194..0a2217264 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -126,16 +126,14 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, if (res < 0) { ast_free(ast); if (err != NULL) { - if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { - err->e = RE_EUNSUPPPCRE; + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + err->e = RE_EUNSUPPORTED; } else if (res == AST_ANALYSIS_ERROR_MEMORY) { /* This case comes up during fuzzing. */ if (err->e == RE_ESUCCESS) { err->e = RE_EERRNO; errno = ENOMEM; } - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE) { - err->e = RE_EUNSUPCAPTUR; } else if (err->e == RE_ESUCCESS) { err->e = RE_EERRNO; } diff --git a/src/libre/strerror.c b/src/libre/strerror.c index d66e750a4..009d61df2 100644 --- a/src/libre/strerror.c +++ b/src/libre/strerror.c @@ -20,8 +20,6 @@ re_strerror(enum re_errno e) case RE_EERRNO: return strerror(errno); case RE_EBADDIALECT: return "Bad dialect"; case RE_EBADGROUP: return "Bad group"; - case RE_EUNSUPCAPTUR: return "Cannot support captures in this case"; - case RE_EUNSUPPPCRE: return "Unsupported PCRE edge case"; case RE_ENEGRANGE: return "Negative group range"; case RE_ENEGCOUNT: return "Negative count range"; diff --git a/tests/pcre/out48.err b/tests/pcre/out48.err index b03d96d7e..4922f0d59 100644 --- a/tests/pcre/out48.err +++ b/tests/pcre/out48.err @@ -1 +1 @@ -tests/pcre/in48.re: Unsupported PCRE edge case +tests/pcre/in48.re:1: Unsupported operator diff --git a/tests/pcre/out49.err b/tests/pcre/out49.err index d22e4cd6d..e7b5bc93f 100644 --- a/tests/pcre/out49.err +++ b/tests/pcre/out49.err @@ -1 +1 @@ -tests/pcre/in49.re: Unsupported PCRE edge case +tests/pcre/in49.re:1: Unsupported operator From 8330d5733ee6a50e53796010721d1d20751c6589 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 30 May 2024 02:12:27 +0100 Subject: [PATCH 115/373] Fabricate error reporting for RE_EUNSUPPORTED through AST analysis. --- src/libre/ast_analysis.c | 3 +-- src/libre/dialect/comp.h | 2 +- src/libre/dialect/glob/parser.c | 8 ++++++-- src/libre/dialect/glob/parser.h | 2 +- src/libre/dialect/like/parser.c | 8 ++++++-- src/libre/dialect/like/parser.h | 2 +- src/libre/dialect/literal/parser.c | 8 ++++++-- src/libre/dialect/literal/parser.h | 2 +- src/libre/dialect/native/parser.c | 8 ++++++-- src/libre/dialect/native/parser.h | 2 +- src/libre/dialect/pcre/parser.c | 8 ++++++-- src/libre/dialect/pcre/parser.h | 2 +- src/libre/dialect/sql/parser.c | 8 ++++++-- src/libre/dialect/sql/parser.h | 2 +- src/libre/parser.act | 6 +++++- src/libre/re.c | 14 +++++++++++++- tests/pcre/out48.err | 2 +- tests/pcre/out49.err | 2 +- 18 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index b2381e01b..03d6d79b7 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1010,8 +1010,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) all_set_past_always_consuming &= child_env.past_always_consuming; any_sat = 1; } - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED - || res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { continue; } else { return res; diff --git a/src/libre/dialect/comp.h b/src/libre/dialect/comp.h index 7841f776b..d223ab90e 100644 --- a/src/libre/dialect/comp.h +++ b/src/libre/dialect/comp.h @@ -17,7 +17,7 @@ typedef struct ast * re_dialect_parse_fun(re_getchar_fun *getchar, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err); + struct re_err *err, struct re_pos *end); re_dialect_parse_fun parse_re_literal; re_dialect_parse_fun parse_re_glob; diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 0de67cf26..746220d4c 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -648,7 +648,7 @@ ZL0:; DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -661,6 +661,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -714,6 +715,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -779,6 +783,6 @@ ZL0:; return NULL; } -#line 783 "src/libre/dialect/glob/parser.c" +#line 787 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index cfcce16b7..c0ce31ed6 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1204 "src/libre/parser.act" +#line 1208 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index eec930c0f..6543095e3 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -648,7 +648,7 @@ ZL0:; DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -661,6 +661,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -714,6 +715,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -779,6 +783,6 @@ ZL0:; return NULL; } -#line 783 "src/libre/dialect/like/parser.c" +#line 787 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index 4c7949714..8563a04a6 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1204 "src/libre/parser.act" +#line 1208 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 7b25cbc6b..f52793a0c 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -561,7 +561,7 @@ ZL0:; DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -574,6 +574,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -627,6 +628,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -692,6 +696,6 @@ ZL0:; return NULL; } -#line 696 "src/libre/dialect/literal/parser.c" +#line 700 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index b1c20e4bb..672d23e8e 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1204 "src/libre/parser.act" +#line 1208 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 7aa79896c..05a3ff0e7 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -3057,7 +3057,7 @@ ZL0:; DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -3070,6 +3070,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -3123,6 +3124,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -3188,6 +3192,6 @@ ZL0:; return NULL; } -#line 3192 "src/libre/dialect/native/parser.c" +#line 3196 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index c833c9629..8b1c4d866 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1204 "src/libre/parser.act" +#line 1208 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 12987d311..2400a0812 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -4341,7 +4341,7 @@ ZL0:; DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -4354,6 +4354,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -4407,6 +4408,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -4472,6 +4476,6 @@ ZL0:; return NULL; } -#line 4476 "src/libre/dialect/pcre/parser.c" +#line 4480 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 17a017297..a0543aebe 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1204 "src/libre/parser.act" +#line 1208 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index dd60efe03..078b8f67d 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -2054,7 +2054,7 @@ ZL0:; DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -2067,6 +2067,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -2120,6 +2121,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -2185,6 +2189,6 @@ ZL0:; return NULL; } -#line 2189 "src/libre/dialect/sql/parser.c" +#line 2193 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index cbbd5b111..e9850510a 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1204 "src/libre/parser.act" +#line 1208 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index 0156fbf08..d8d0e2ac4 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -1070,7 +1070,7 @@ DIALECT_PARSE(re_getchar_fun *f, void *opaque, const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -1083,6 +1083,7 @@ struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -1136,6 +1137,9 @@ DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { diff --git a/src/libre/re.c b/src/libre/re.c index 0a2217264..ca7451867 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -98,6 +98,7 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, const struct dialect *m; struct ast *ast = NULL; enum ast_analysis_res res; + struct re_pos end; assert(getc != NULL); @@ -109,7 +110,7 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = m->parse(getc, opaque, opt, flags, m->overlap, err); + ast = m->parse(getc, opaque, opt, flags, m->overlap, err, &end); if (ast == NULL) { return NULL; @@ -128,6 +129,17 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, if (err != NULL) { if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { err->e = RE_EUNSUPPORTED; + + /* + * We can't tag AST nodes with re_pos, because it's + * also possible to construct an AST from an .fsm file. + * We detect RE_EUNSUPPORTED from annotations (e.g. nullable) + * on arbitary nodes. So at best we could tag an expr. + * But since in general we'd need to fabricate a pos anyway, + * I'm blaming the entire expression here. + */ + err->start.byte = 0; + err->end.byte = end.byte; } else if (res == AST_ANALYSIS_ERROR_MEMORY) { /* This case comes up during fuzzing. */ if (err->e == RE_ESUCCESS) { diff --git a/tests/pcre/out48.err b/tests/pcre/out48.err index 4922f0d59..d96141a09 100644 --- a/tests/pcre/out48.err +++ b/tests/pcre/out48.err @@ -1 +1 @@ -tests/pcre/in48.re:1: Unsupported operator +tests/pcre/in48.re:1-11: Unsupported operator diff --git a/tests/pcre/out49.err b/tests/pcre/out49.err index e7b5bc93f..b92cc933b 100644 --- a/tests/pcre/out49.err +++ b/tests/pcre/out49.err @@ -1 +1 @@ -tests/pcre/in49.re:1: Unsupported operator +tests/pcre/in49.re:1-17: Unsupported operator From 61772d0c253d3c057b1ee7a9ee6a2160095a2eb0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 30 May 2024 02:40:16 +0100 Subject: [PATCH 116/373] Whitespace. --- src/libre/re.c | 69 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/src/libre/re.c b/src/libre/re.c index ca7451867..2923b7d3d 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -123,34 +123,8 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, /* Do a complete pass over the AST, filling in other details. */ res = ast_analysis(ast, flags); - if (res < 0) { - ast_free(ast); - if (err != NULL) { - if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { - err->e = RE_EUNSUPPORTED; - - /* - * We can't tag AST nodes with re_pos, because it's - * also possible to construct an AST from an .fsm file. - * We detect RE_EUNSUPPORTED from annotations (e.g. nullable) - * on arbitary nodes. So at best we could tag an expr. - * But since in general we'd need to fabricate a pos anyway, - * I'm blaming the entire expression here. - */ - err->start.byte = 0; - err->end.byte = end.byte; - } else if (res == AST_ANALYSIS_ERROR_MEMORY) { - /* This case comes up during fuzzing. */ - if (err->e == RE_ESUCCESS) { - err->e = RE_EERRNO; - errno = ENOMEM; - } - } else if (err->e == RE_ESUCCESS) { - err->e = RE_EERRNO; - } - } - return NULL; + goto error; } if (unsatisfiable != NULL) { @@ -158,6 +132,47 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, } return ast; + +error: + + ast_free(ast); + + if (err == NULL) { + return NULL; + } + + switch (res) { + case AST_ANALYSIS_ERROR_MEMORY: + /* This case comes up during fuzzing. */ + if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + errno = ENOMEM; + } + break; + + case AST_ANALYSIS_ERROR_UNSUPPORTED: + err->e = RE_EUNSUPPORTED; + + /* + * We can't tag AST nodes with re_pos, because it's + * also possible to construct an AST from an .fsm file. + * We detect RE_EUNSUPPORTED from annotations (e.g. nullable) + * on arbitary nodes. So at best we could tag an expr. + * But since in general we'd need to fabricate a pos anyway, + * I'm blaming the entire expression here. + */ + err->start.byte = 0; + err->end.byte = end.byte; + break; + + default: + if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + } + break; + } + + return NULL; } struct fsm * From 8feff1405390e05dc099a41233c2ad095096e49f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 30 May 2024 11:24:02 -0400 Subject: [PATCH 117/373] fsm_vacuum: Change return type to bool. --- include/fsm/fsm.h | 11 +++++------ src/libfsm/vacuum.c | 8 ++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index a5e6ef388..1904814bd 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -15,6 +15,8 @@ * be atomic. */ +#include + struct fsm; struct fsm_options; struct path; /* XXX */ @@ -494,12 +496,9 @@ int fsm_shuffle(struct fsm *fsm, unsigned seed); /* Attempt to reclaim memory if an FSM has significantly reduced its - * state count. Returns 1 if */ -enum fsm_vacuum_res { - FSM_VACUUM_NO_CHANGE, - FSM_VACUUM_REDUCED_MEMORY, - FSM_VACUUM_ERROC_REALLOC = -1, -} + * state count. Returns true if the vacuuming attempt succeeded (including + * deciding not to do anything), false if realloc failed. */ +bool fsm_vacuum(struct fsm *fsm); #endif diff --git a/src/libfsm/vacuum.c b/src/libfsm/vacuum.c index 2c349c210..abecf6301 100644 --- a/src/libfsm/vacuum.c +++ b/src/libfsm/vacuum.c @@ -12,7 +12,7 @@ #include "internal.h" -enum fsm_vacuum_res +bool fsm_vacuum(struct fsm *fsm) { assert(fsm != NULL); @@ -24,17 +24,17 @@ fsm_vacuum(struct fsm *fsm) } if (nceil == fsm->statealloc) { - return FSM_VACUUM_NO_CHANGE; + return true; } struct fsm_state *nstates = f_realloc(fsm->opt->alloc, fsm->states, nceil * sizeof(nstates[0])); if (nstates == NULL) { - return FSM_VACUUM_ERROC_REALLOC; + return false; } fsm->states = nstates; fsm->statealloc = nceil; - return FSM_VACUUM_REDUCED_MEMORY; + return true; } From 61478603941bba05a8d42644a518590b67e0e726 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 30 May 2024 11:29:00 -0400 Subject: [PATCH 118/373] Call fsm_vacuum at the end of fsm_minimise. This also means that many existing tests will exercise fsm_vacuum. --- src/libfsm/minimise.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 96d92b4e8..94dfdf80b 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -157,6 +157,12 @@ fsm_minimise(struct fsm *fsm) fsm_move(fsm, dst); + /* The FSM state count should be settled now, so if the state + * allocation is unnecessarily large move to a smaller one. */ + if (!fsm_vacuum(fsm)) { /* realloc failure */ + r = 0; + } + cleanup: if (mapping != NULL) { f_free(fsm->opt->alloc, mapping); From 748b7afc934164fa5b0e2e3aaf9ead198a708901 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 30 May 2024 15:27:51 +0100 Subject: [PATCH 119/373] Add fsm_new_statealloc(), to pre-allocate to a known number of states. --- include/fsm/fsm.h | 9 +++++++++ src/libfsm/charset.c | 3 +-- src/libfsm/clone.c | 2 +- src/libfsm/fsm.c | 14 ++++++++++++-- src/libfsm/reverse.c | 2 +- src/libre/re_strings.c | 1 + 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 1904814bd..2fea117cd 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -60,6 +60,15 @@ struct fsm_end_ids { struct fsm * fsm_new(const struct fsm_options *opt); +/* + * As fsm_new(), but with an explicit number of pre-allocated states. + * This is intended to save on internal reallocations for situations + * where it's known in advance exactly how many states an FSM will have. + * You can still add more states per usual. + */ +struct fsm * +fsm_new_statealloc(const struct fsm_options *opt, size_t statealloc); + /* * Free a structure created by fsm_new(), and all of its contents. * No other pointers returned by this API are to be freed individually. diff --git a/src/libfsm/charset.c b/src/libfsm/charset.c index dd8a679ed..662c0ac1f 100644 --- a/src/libfsm/charset.c +++ b/src/libfsm/charset.c @@ -36,8 +36,7 @@ fsm_intersect_charset(struct fsm *a, size_t n, const char *charset) { fsm_state_t state; - // TODO: pass .statealloc explicitly, we know it's 1. the default is overkill - b = fsm_new(a->opt); + b = fsm_new_statealloc(a->opt, 1); if (b == NULL) { return NULL; } diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index bec16bb0f..78511b12c 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -35,7 +35,7 @@ fsm_clone(const struct fsm *fsm) assert(fsm != NULL); assert(fsm->opt != NULL); - new = fsm_new(fsm->opt); + new = fsm_new_statealloc(fsm->opt, fsm->statecount); if (new == NULL) { return NULL; } diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index d027825ff..0e46962d1 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -44,11 +44,15 @@ free_contents(struct fsm *fsm) } struct fsm * -fsm_new(const struct fsm_options *opt) +fsm_new_statealloc(const struct fsm_options *opt, size_t statealloc) { static const struct fsm_options defaults; struct fsm *new, f; + if (statealloc == 0) { + return fsm_new(opt); + } + if (opt == NULL) { opt = &defaults; } @@ -60,7 +64,7 @@ fsm_new(const struct fsm_options *opt) return NULL; } - new->statealloc = FSM_DEFAULT_STATEALLOC; + new->statealloc = statealloc; new->statecount = 0; new->endcount = 0; new->capture_info = NULL; @@ -92,6 +96,12 @@ fsm_new(const struct fsm_options *opt) return new; } +struct fsm * +fsm_new(const struct fsm_options *opt) +{ + return fsm_new_statealloc(opt, FSM_DEFAULT_STATEALLOC); +} + void fsm_free(struct fsm *fsm) { diff --git a/src/libfsm/reverse.c b/src/libfsm/reverse.c index e896a1bff..b802f364d 100644 --- a/src/libfsm/reverse.c +++ b/src/libfsm/reverse.c @@ -54,7 +54,7 @@ fsm_reverse(struct fsm *fsm) if (fsm->endcount == 0 || !fsm_getstart(fsm, &prevstart)) { struct fsm *new; - new = fsm_new(fsm->opt); + new = fsm_new_statealloc(fsm->opt, 1); if (new == NULL) { return 0; } diff --git a/src/libre/re_strings.c b/src/libre/re_strings.c index b2f04fec1..ff8b2b4ab 100644 --- a/src/libre/re_strings.c +++ b/src/libre/re_strings.c @@ -96,6 +96,7 @@ re_strings_build(struct re_strings *g, } } + /* TODO: count trie nodes and fsm_new_statealloc() */ fsm = fsm_new(opt); if (fsm == NULL) { goto error; From a3fb7d7dc02f481ddade1d88f2a2dd5bbb344f4e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 31 May 2024 10:55:01 -0400 Subject: [PATCH 120/373] Add `void* opaque` to vmc codegen too, disable unused warning. This is a follow-up to #465: - Also add the extra `void *opaque` to vmc's codegen. - Add a `(void)` cast to suppress warnings if the extra opaque void pointer isn't used by the generated code. --- src/libfsm/print/c.c | 3 +++ src/libfsm/print/vmc.c | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 29365c5e6..dbca1d3fa 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -427,6 +427,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, case FSM_IO_GETC: fprintf(f, "(int (*fsm_getc)(void *opaque), void *opaque)\n"); fprintf(f, "{\n"); + fprintf(f, "\t(void)opaque;\n"); if (ir->n > 0) { fprintf(f, "\tint c;\n"); fprintf(f, "\n"); @@ -436,6 +437,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, case FSM_IO_STR: fprintf(f, "(const char *s, void *opaque)\n"); fprintf(f, "{\n"); + fprintf(f, "\t(void)opaque;\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); @@ -445,6 +447,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, case FSM_IO_PAIR: fprintf(f, "(const char *b, const char *e, void *opaque)\n"); fprintf(f, "{\n"); + fprintf(f, "\t(void)opaque;\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index b92d68f6d..911619a7f 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -472,13 +472,15 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt break; case FSM_IO_STR: - fprintf(f, "(const char *s)\n"); + fprintf(f, "(const char *s, void *opaque)\n"); fprintf(f, "{\n"); + fprintf(f, "\t(void)opaque;\n"); break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e)\n"); + fprintf(f, "(const char *b, const char *e, void *opaque)\n"); fprintf(f, "{\n"); + fprintf(f, "\t(void)opaque;\n"); break; } From 7af68d4bf02a2ed7116bb0b25ce733e914acbef7 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 31 May 2024 07:34:21 -0700 Subject: [PATCH 121/373] Allow xrealloc() to free() on a size of 0. This matches the standard behaviour. I'd disallowed it to avoid confusion, but in rx I do actually want to realloc down to nothing and return NULL when size is 0. And that's what free() does, but it seems cumbersome to have a conditional around that in the caller. --- include/adt/xalloc.h | 1 + src/adt/xalloc.c | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/adt/xalloc.h b/include/adt/xalloc.h index 373e72d46..d36132687 100644 --- a/include/adt/xalloc.h +++ b/include/adt/xalloc.h @@ -23,6 +23,7 @@ xmalloc(size_t sz); void * xcalloc(size_t count, size_t sz); +/* equivalent to free(p) when sz is 0, returns NULL */ void * xrealloc(void *p, size_t sz); diff --git a/src/adt/xalloc.c b/src/adt/xalloc.c index b80723ffd..86a83ead4 100644 --- a/src/adt/xalloc.c +++ b/src/adt/xalloc.c @@ -57,11 +57,8 @@ xrealloc(void *p, size_t sz) { void *q; - /* This is legal and frees p, but confusing; use free() instead */ - assert(sz != 0); - q = realloc(p, sz); - if (q == NULL) { + if (sz > 0 && q == NULL) { perror("realloc"); abort(); } From 88877cd858b5665d2752342dff8a85b5eadc506c Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 31 May 2024 07:35:14 -0700 Subject: [PATCH 122/373] xstrndup() --- include/adt/xalloc.h | 3 +++ src/adt/xalloc.c | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/include/adt/xalloc.h b/include/adt/xalloc.h index d36132687..e398b3925 100644 --- a/include/adt/xalloc.h +++ b/include/adt/xalloc.h @@ -17,6 +17,9 @@ xstrdup(const char *s); * The following functions are for CLI use only; they exit on error */ +char * +xstrndup(const char *s, size_t n); + void * xmalloc(size_t sz); diff --git a/src/adt/xalloc.c b/src/adt/xalloc.c index 86a83ead4..b077c3b8f 100644 --- a/src/adt/xalloc.c +++ b/src/adt/xalloc.c @@ -4,6 +4,8 @@ * See LICENCE for the full copyright terms. */ +#define _POSIX_C_SOURCE 200809L + #include #include #include @@ -24,6 +26,18 @@ xstrdup(const char *s) return strcpy(new, s); } +char * +xstrndup(const char *s, size_t n) +{ + char *new = strndup(s, n); + if (new == NULL) { + perror("strndup"); + abort(); + } + + return new; +} + void * xmalloc(size_t sz) { From 4ec219a73b1a79186f5d3705cb0fbe60f7f5739d Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 31 May 2024 07:53:29 -0700 Subject: [PATCH 123/373] Normalization; all x*() xalloc.h interfaces exit on error. There's no need to have xstrdup() behave differently here, it's just confusing. --- include/adt/xalloc.h | 9 +- src/adt/xalloc.c | 2 +- src/libfsm/parser.act | 10 +- src/libfsm/parser.c | 173 +++++++++----------- src/libfsm/parser.h | 4 +- src/lx/parser.act | 8 - src/lx/parser.c | 360 ++++++++++++++++++++---------------------- src/lx/parser.h | 4 +- src/retest/cvtpcre.c | 4 - 9 files changed, 253 insertions(+), 321 deletions(-) diff --git a/include/adt/xalloc.h b/include/adt/xalloc.h index e398b3925..cf7fd7b11 100644 --- a/include/adt/xalloc.h +++ b/include/adt/xalloc.h @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -8,15 +8,12 @@ #define XALLOC_H /* - * Duplicate a string. Returns NULL on error. + * These functions are for CLI use only; they exit on error */ + char * xstrdup(const char *s); -/* - * The following functions are for CLI use only; they exit on error - */ - char * xstrndup(const char *s, size_t n); diff --git a/src/adt/xalloc.c b/src/adt/xalloc.c index b077c3b8f..bdce10a49 100644 --- a/src/adt/xalloc.c +++ b/src/adt/xalloc.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index 14ad7f169..80ace7fdf 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -241,11 +241,8 @@ @}; IDENT: () -> (s :string) = @{ + /* XXX: don't exit in library code */ @s = xstrdup(lex_state->buf.a); - if (@s == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } @}; %actions% @@ -288,11 +285,8 @@ exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup(@n); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &@s)) { perror("fsm_addstate"); diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 81f1cc030..19d5664f3 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 150 "src/libfsm/parser.act" +#line 151 "src/libfsm/parser.act" #include @@ -135,7 +135,7 @@ htab->longest_chain_length = max_chain_length; } -#line 138 "src/libfsm/parser.c" +#line 139 "src/libfsm/parser.c" #ifndef ERROR_TERMINAL @@ -175,14 +175,14 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: CHAR */ { -#line 239 "src/libfsm/parser.act" +#line 240 "src/libfsm/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 185 "src/libfsm/parser.c" +#line 186 "src/libfsm/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -192,7 +192,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: ESC */ { -#line 167 "src/libfsm/parser.act" +#line 168 "src/libfsm/parser.act" assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(2 == strlen(lex_state->buf.a)); @@ -210,7 +210,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) default: break; } -#line 213 "src/libfsm/parser.c" +#line 214 "src/libfsm/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -220,7 +220,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: HEX */ { -#line 232 "src/libfsm/parser.act" +#line 233 "src/libfsm/parser.act" unsigned long u; char *e; @@ -247,7 +247,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) ZIc = (char) (unsigned char) u; -#line 250 "src/libfsm/parser.c" +#line 251 "src/libfsm/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -257,7 +257,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: OCT */ { -#line 205 "src/libfsm/parser.act" +#line 206 "src/libfsm/parser.act" unsigned long u; char *e; @@ -284,7 +284,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) ZIc = (char) (unsigned char) u; -#line 287 "src/libfsm/parser.c" +#line 288 "src/libfsm/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -330,15 +330,12 @@ ZL2_items:; case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 245 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIa = xstrdup(lex_state->buf.a); - if (ZIa == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 341 "src/libfsm/parser.c" +#line 339 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -398,11 +395,11 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-start */ { -#line 371 "src/libfsm/parser.act" +#line 366 "src/libfsm/parser.act" err_expected(lex_state, "'start:'"); -#line 405 "src/libfsm/parser.c" +#line 403 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-start */ } @@ -416,15 +413,12 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 245 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 427 "src/libfsm/parser.c" +#line 422 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -442,7 +436,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) } /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -481,11 +475,8 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((ZIn)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIs))) { perror("fsm_addstate"); @@ -498,25 +489,25 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 501 "src/libfsm/parser.c" +#line 493 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ { -#line 309 "src/libfsm/parser.act" +#line 304 "src/libfsm/parser.act" fsm_setstart(fsm, (ZIs)); -#line 510 "src/libfsm/parser.c" +#line 502 "src/libfsm/parser.c" } /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 312 "src/libfsm/parser.act" free((ZIn)); -#line 519 "src/libfsm/parser.c" +#line 511 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -554,11 +545,11 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-end */ { -#line 375 "src/libfsm/parser.act" +#line 370 "src/libfsm/parser.act" err_expected(lex_state, "'end:'"); -#line 561 "src/libfsm/parser.c" +#line 553 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-end */ } @@ -604,11 +595,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-sep */ { -#line 359 "src/libfsm/parser.act" +#line 354 "src/libfsm/parser.act" err_expected(lex_state, "';'"); -#line 611 "src/libfsm/parser.c" +#line 603 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-sep */ } @@ -632,15 +623,12 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 245 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); - if (ZIb == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 643 "src/libfsm/parser.c" +#line 632 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -653,7 +641,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF INLINE: id */ /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -692,11 +680,8 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((*ZIa)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIx))) { perror("fsm_addstate"); @@ -709,12 +694,12 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 712 "src/libfsm/parser.c" +#line 698 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIb)); @@ -753,11 +738,8 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((ZIb)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIy))) { perror("fsm_addstate"); @@ -770,25 +752,25 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 773 "src/libfsm/parser.c" +#line 756 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 312 "src/libfsm/parser.act" free((*ZIa)); -#line 782 "src/libfsm/parser.c" +#line 765 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 312 "src/libfsm/parser.act" free((ZIb)); -#line 791 "src/libfsm/parser.c" +#line 774 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF INLINE: 42 */ @@ -799,14 +781,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; /* BEGINNING OF ACTION: add-edge-any */ { -#line 343 "src/libfsm/parser.act" +#line 338 "src/libfsm/parser.act" if (!fsm_addedge_any(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_any"); exit(EXIT_FAILURE); } -#line 809 "src/libfsm/parser.c" +#line 792 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-any */ } @@ -822,14 +804,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) } /* BEGINNING OF ACTION: add-edge-literal */ { -#line 336 "src/libfsm/parser.act" +#line 331 "src/libfsm/parser.act" if (!fsm_addedge_literal(fsm, (ZIx), (ZIy), (ZIc))) { perror("fsm_addedge_literal"); exit(EXIT_FAILURE); } -#line 832 "src/libfsm/parser.c" +#line 815 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-literal */ } @@ -838,14 +820,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: add-edge-epsilon */ { -#line 350 "src/libfsm/parser.act" +#line 345 "src/libfsm/parser.act" if (!fsm_addedge_epsilon(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_epsilon"); exit(EXIT_FAILURE); } -#line 848 "src/libfsm/parser.c" +#line 831 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-epsilon */ } @@ -856,11 +838,11 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: err-expected-trans */ { -#line 363 "src/libfsm/parser.act" +#line 358 "src/libfsm/parser.act" err_expected(lex_state, "transition"); -#line 863 "src/libfsm/parser.c" +#line 846 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-trans */ } @@ -880,7 +862,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -919,11 +901,8 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((*ZIa)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZI45))) { perror("fsm_addstate"); @@ -936,16 +915,16 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 939 "src/libfsm/parser.c" +#line 919 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 312 "src/libfsm/parser.act" free((*ZIa)); -#line 948 "src/libfsm/parser.c" +#line 928 "src/libfsm/parser.c" } /* END OF ACTION: free */ p_61 (fsm, lex_state, act_state); @@ -988,7 +967,7 @@ p_fsm(fsm fsm, lex_state lex_state, act_state act_state) ADVANCE_LEXER; /* BEGINNING OF ACTION: free-statelist */ { -#line 333 "src/libfsm/parser.act" +#line 328 "src/libfsm/parser.act" struct act_statelist *p; struct act_statelist *next; @@ -1003,7 +982,7 @@ p_fsm(fsm fsm, lex_state lex_state, act_state act_state) } } -#line 1006 "src/libfsm/parser.c" +#line 986 "src/libfsm/parser.c" } /* END OF ACTION: free-statelist */ } @@ -1012,12 +991,12 @@ ZL1:; { /* BEGINNING OF ACTION: err-syntax */ { -#line 380 "src/libfsm/parser.act" +#line 375 "src/libfsm/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 1020 "src/libfsm/parser.c" +#line 1000 "src/libfsm/parser.c" } /* END OF ACTION: err-syntax */ } @@ -1053,11 +1032,11 @@ ZL2_xend_C_Cend_Hids:; { /* BEGINNING OF ACTION: err-expected-comma */ { -#line 367 "src/libfsm/parser.act" +#line 362 "src/libfsm/parser.act" err_expected(lex_state, "','"); -#line 1060 "src/libfsm/parser.c" +#line 1040 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-comma */ } @@ -1101,15 +1080,12 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 245 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 1112 "src/libfsm/parser.c" +#line 1089 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -1122,7 +1098,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state) /* END OF INLINE: id */ /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -1161,11 +1137,8 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((ZIn)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIs))) { perror("fsm_addstate"); @@ -1178,25 +1151,25 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 1181 "src/libfsm/parser.c" +#line 1155 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-end */ { -#line 313 "src/libfsm/parser.act" +#line 308 "src/libfsm/parser.act" fsm_setend(fsm, (ZIs), 1); -#line 1190 "src/libfsm/parser.c" +#line 1164 "src/libfsm/parser.c" } /* END OF ACTION: mark-end */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 312 "src/libfsm/parser.act" free((ZIn)); -#line 1199 "src/libfsm/parser.c" +#line 1173 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -1208,7 +1181,7 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 436 "src/libfsm/parser.act" +#line 431 "src/libfsm/parser.act" struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { @@ -1263,6 +1236,6 @@ ZL1:; return new; } -#line 1266 "src/libfsm/parser.c" +#line 1240 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index 181f13d44..70155c504 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 159 "src/libfsm/parser.act" +#line 160 "src/libfsm/parser.act" typedef struct lex_state * lex_state; @@ -26,7 +26,7 @@ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 437 "src/libfsm/parser.act" +#line 432 "src/libfsm/parser.act" #line 32 "src/libfsm/parser.h" diff --git a/src/lx/parser.act b/src/lx/parser.act index f9b0b32f9..27975d9d5 100644 --- a/src/lx/parser.act +++ b/src/lx/parser.act @@ -223,18 +223,10 @@ TOKEN: () -> (s :string) = @{ /* TODO: submatch addressing */ @s = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (@s == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } @}; IDENT: () -> (s :string) = @{ @s = xstrdup(lex_state->buf.a); - if (@s == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } @}; RE: () -> (f :flags) = @{ diff --git a/src/lx/parser.c b/src/lx/parser.c index dfcf36b8f..0933c386f 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 27 "src/lx/parser.act" +#line 128 "src/lx/parser.act" #include @@ -183,7 +183,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 817 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); @@ -205,21 +205,17 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: IDENT */ { -#line 232 "src/lx/parser.act" +#line 229 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 217 "src/lx/parser.c" +#line 213 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-var */ { -#line 286 "src/lx/parser.act" +#line 281 "src/lx/parser.act" struct ast_zone *z; @@ -246,7 +242,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) goto ZL1; } -#line 250 "src/lx/parser.c" +#line 246 "src/lx/parser.c" } /* END OF ACTION: deref-var */ } @@ -257,22 +253,18 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: TOKEN */ { -#line 223 "src/lx/parser.act" +#line 225 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (ZIt == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 270 "src/lx/parser.c" +#line 262 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 313 "src/lx/parser.act" +#line 309 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -329,7 +321,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) fsm_setstart((ZIr), start); -#line 333 "src/lx/parser.c" +#line 325 "src/lx/parser.c" } /* END OF ACTION: deref-token */ } @@ -383,7 +375,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-reverse */ { -#line 685 "src/lx/parser.act" +#line 678 "src/lx/parser.act" assert((ZI210) != NULL); @@ -392,7 +384,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 396 "src/lx/parser.c" +#line 388 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -407,7 +399,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI231) != NULL); @@ -427,7 +419,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 431 "src/lx/parser.c" +#line 423 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI232, &ZI233); @@ -458,7 +450,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complete */ { -#line 676 "src/lx/parser.act" +#line 669 "src/lx/parser.act" assert((ZI210) != NULL); @@ -467,7 +459,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 471 "src/lx/parser.c" +#line 463 "src/lx/parser.c" } /* END OF ACTION: op-complete */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -482,7 +474,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI239) != NULL); @@ -502,7 +494,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 506 "src/lx/parser.c" +#line 498 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI240, &ZI241); @@ -518,15 +510,11 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: IDENT */ { -#line 232 "src/lx/parser.act" +#line 229 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 530 "src/lx/parser.c" +#line 518 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; @@ -576,7 +564,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI248) != NULL); @@ -596,7 +584,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 600 "src/lx/parser.c" +#line 588 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI249, &ZI250); @@ -627,7 +615,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complement */ { -#line 667 "src/lx/parser.act" +#line 660 "src/lx/parser.act" assert((ZI210) != NULL); @@ -636,7 +624,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 640 "src/lx/parser.c" +#line 628 "src/lx/parser.c" } /* END OF ACTION: op-complement */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -651,7 +639,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI223) != NULL); @@ -671,7 +659,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 675 "src/lx/parser.c" +#line 663 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI224, &ZI225); @@ -698,22 +686,18 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: TOKEN */ { -#line 223 "src/lx/parser.act" +#line 225 "src/lx/parser.act" /* TODO: submatch addressing */ ZI253 = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (ZI253 == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 711 "src/lx/parser.c" +#line 695 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 313 "src/lx/parser.act" +#line 309 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -770,7 +754,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z fsm_setstart((ZI254), start); -#line 774 "src/lx/parser.c" +#line 758 "src/lx/parser.c" } /* END OF ACTION: deref-token */ p_197 (lex_state, act_state, ZI254, &ZI210); @@ -786,7 +770,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI259) != NULL); @@ -806,7 +790,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 810 "src/lx/parser.c" +#line 794 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI260, &ZI261); @@ -846,7 +830,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI268) != NULL); @@ -866,7 +850,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 870 "src/lx/parser.c" +#line 854 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI269, &ZI270); @@ -886,11 +870,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-thing */ { -#line 819 "src/lx/parser.act" +#line 813 "src/lx/parser.act" err_expected(lex_state, "mapping, binding or zone"); -#line 894 "src/lx/parser.c" +#line 878 "src/lx/parser.c" } /* END OF ACTION: err-expected-thing */ } @@ -916,11 +900,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-open */ { -#line 807 "src/lx/parser.act" +#line 801 "src/lx/parser.act" err_expected(lex_state, "'{'"); -#line 924 "src/lx/parser.c" +#line 908 "src/lx/parser.c" } /* END OF ACTION: err-expected-open */ } @@ -946,11 +930,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-close */ { -#line 811 "src/lx/parser.act" +#line 805 "src/lx/parser.act" err_expected(lex_state, "'}'"); -#line 954 "src/lx/parser.c" +#line 938 "src/lx/parser.c" } /* END OF ACTION: err-expected-close */ } @@ -972,14 +956,14 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: CHAR */ { -#line 216 "src/lx/parser.act" +#line 220 "src/lx/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 983 "src/lx/parser.c" +#line 967 "src/lx/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -989,7 +973,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: ESC */ { -#line 144 "src/lx/parser.act" +#line 150 "src/lx/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1007,7 +991,7 @@ ZL2_pattern_C_Cbody:; default: ZIc = '\0'; break; /* TODO: handle error */ } -#line 1011 "src/lx/parser.c" +#line 995 "src/lx/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1017,7 +1001,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: HEX */ { -#line 189 "src/lx/parser.act" +#line 213 "src/lx/parser.act" unsigned long u; char *e; @@ -1044,7 +1028,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1048 "src/lx/parser.c" +#line 1032 "src/lx/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1054,7 +1038,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: OCT */ { -#line 162 "src/lx/parser.act" +#line 186 "src/lx/parser.act" unsigned long u; char *e; @@ -1081,7 +1065,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1085 "src/lx/parser.c" +#line 1069 "src/lx/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1094,19 +1078,19 @@ ZL2_pattern_C_Cbody:; /* END OF INLINE: 84 */ /* BEGINNING OF ACTION: pattern-char */ { -#line 256 "src/lx/parser.act" +#line 250 "src/lx/parser.act" /* TODO */ *lex_state->p++ = (ZIc); -#line 1103 "src/lx/parser.c" +#line 1087 "src/lx/parser.c" } /* END OF ACTION: pattern-char */ /* BEGINNING OF INLINE: pattern::body */ goto ZL2_pattern_C_Cbody; /* END OF INLINE: pattern::body */ } - /* UNREACHED */ + /*UNREACHED*/ case (ERROR_TERMINAL): return; default: @@ -1141,7 +1125,7 @@ ZL2_174:; } /* END OF INLINE: 174 */ } - /* UNREACHED */ + /*UNREACHED*/ default: { ZI171 = ZI168; @@ -1228,7 +1212,7 @@ ZL2_180:; } /* BEGINNING OF ACTION: op-alt */ { -#line 733 "src/lx/parser.act" +#line 726 "src/lx/parser.act" assert((ZI177) != NULL); assert((ZIb) != NULL); @@ -1239,7 +1223,7 @@ ZL2_180:; goto ZL1; } -#line 1243 "src/lx/parser.c" +#line 1227 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: 180 */ @@ -1247,7 +1231,7 @@ ZL2_180:; goto ZL2_180; /* END OF INLINE: 180 */ } - /* UNREACHED */ + /*UNREACHED*/ default: { ZI178 = ZI176; @@ -1308,7 +1292,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-reverse */ { -#line 685 "src/lx/parser.act" +#line 678 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1317,7 +1301,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1321 "src/lx/parser.c" +#line 1305 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ } @@ -1332,7 +1316,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complete */ { -#line 676 "src/lx/parser.act" +#line 669 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1341,7 +1325,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1345 "src/lx/parser.c" +#line 1329 "src/lx/parser.c" } /* END OF ACTION: op-complete */ } @@ -1356,7 +1340,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complement */ { -#line 667 "src/lx/parser.act" +#line 660 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1365,7 +1349,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1369 "src/lx/parser.c" +#line 1353 "src/lx/parser.c" } /* END OF ACTION: op-complement */ } @@ -1415,7 +1399,7 @@ ZL2_186:; } /* BEGINNING OF ACTION: op-intersect */ { -#line 722 "src/lx/parser.act" +#line 715 "src/lx/parser.act" assert((ZI183) != NULL); assert((ZIb) != NULL); @@ -1426,7 +1410,7 @@ ZL2_186:; goto ZL1; } -#line 1430 "src/lx/parser.c" +#line 1414 "src/lx/parser.c" } /* END OF ACTION: op-intersect */ /* BEGINNING OF INLINE: 186 */ @@ -1434,7 +1418,7 @@ ZL2_186:; goto ZL2_186; /* END OF INLINE: 186 */ } - /* UNREACHED */ + /*UNREACHED*/ default: { ZI184 = ZI182; @@ -1472,7 +1456,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ } /* BEGINNING OF ACTION: op-alt */ { -#line 733 "src/lx/parser.act" +#line 726 "src/lx/parser.act" assert((ZIold_Hexit) != NULL); assert((ZInew_Hexit) != NULL); @@ -1483,14 +1467,14 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ goto ZL1; } -#line 1487 "src/lx/parser.c" +#line 1471 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-to-mappings::list-of-zone-to-mappings-x */ goto ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_Hzone_Hto_Hmappings_Hx; /* END OF INLINE: list-of-things::zone-thing::list-of-zone-to-mappings::list-of-zone-to-mappings-x */ } - /* UNREACHED */ + /*UNREACHED*/ case (ERROR_TERMINAL): return; default: @@ -1571,7 +1555,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-cross */ { -#line 577 "src/lx/parser.act" +#line 576 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1616,7 +1600,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1620 "src/lx/parser.c" +#line 1604 "src/lx/parser.c" } /* END OF ACTION: op-cross */ /* BEGINNING OF INLINE: 197 */ @@ -1624,13 +1608,13 @@ ZL2_197:; goto ZL2_197; /* END OF INLINE: 197 */ } - /* UNREACHED */ + /*UNREACHED*/ case (TOK_QMARK): { ADVANCE_LEXER; /* BEGINNING OF ACTION: op-qmark */ { -#line 622 "src/lx/parser.act" +#line 621 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1675,7 +1659,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1679 "src/lx/parser.c" +#line 1663 "src/lx/parser.c" } /* END OF ACTION: op-qmark */ /* BEGINNING OF INLINE: 197 */ @@ -1683,13 +1667,13 @@ ZL2_197:; goto ZL2_197; /* END OF INLINE: 197 */ } - /* UNREACHED */ + /*UNREACHED*/ case (TOK_STAR): { ADVANCE_LEXER; /* BEGINNING OF ACTION: op-star */ { -#line 527 "src/lx/parser.act" +#line 526 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1739,7 +1723,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1743 "src/lx/parser.c" +#line 1727 "src/lx/parser.c" } /* END OF ACTION: op-star */ /* BEGINNING OF INLINE: 197 */ @@ -1747,14 +1731,14 @@ ZL2_197:; goto ZL2_197; /* END OF INLINE: 197 */ } - /* UNREACHED */ + /*UNREACHED*/ default: goto ZL1; } } /* END OF INLINE: 272 */ } - /* UNREACHED */ + /*UNREACHED*/ default: { ZI196 = ZI191; @@ -1812,7 +1796,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) } /* BEGINNING OF ACTION: op-subtract */ { -#line 711 "src/lx/parser.act" +#line 704 "src/lx/parser.act" assert((*ZI202) != NULL); assert((ZIb) != NULL); @@ -1823,7 +1807,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) goto ZL1; } -#line 1827 "src/lx/parser.c" +#line 1811 "src/lx/parser.c" } /* END OF ACTION: op-subtract */ } @@ -1887,7 +1871,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) } /* BEGINNING OF ACTION: op-concat */ { -#line 694 "src/lx/parser.act" +#line 687 "src/lx/parser.act" assert((*ZI206) != NULL); assert((ZIb) != NULL); @@ -1898,7 +1882,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) goto ZL1; } -#line 1902 "src/lx/parser.c" +#line 1886 "src/lx/parser.c" } /* END OF ACTION: op-concat */ } @@ -1937,13 +1921,13 @@ p_212(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI210, fsm *ZOq) } /* BEGINNING OF ACTION: op-product */ { -#line 705 "src/lx/parser.act" +#line 699 "src/lx/parser.act" fprintf(stderr, "unimplemented\n"); (ZIq) = NULL; goto ZL1; -#line 1947 "src/lx/parser.c" +#line 1931 "src/lx/parser.c" } /* END OF ACTION: op-product */ } @@ -1977,7 +1961,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* BEGINNING OF EXTRACT: RE */ { -#line 240 "src/lx/parser.act" +#line 237 "src/lx/parser.act" assert(lex_state->buf.a[0] == '/'); @@ -1991,13 +1975,13 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ZIf |= RE_ANCHORED; /* regexps in lx are implicitly anchored */ ZIf |= RE_SINGLE; /* we can't assume we're lexing text */ -#line 1995 "src/lx/parser.c" +#line 1979 "src/lx/parser.c" } /* END OF EXTRACT: RE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 261 "src/lx/parser.act" +#line 264 "src/lx/parser.act" size_t len; @@ -2022,12 +2006,12 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2026 "src/lx/parser.c" +#line 2010 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-regex */ { -#line 385 "src/lx/parser.act" +#line 380 "src/lx/parser.act" struct re_err err; @@ -2042,16 +2026,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2046 "src/lx/parser.c" +#line 2030 "src/lx/parser.c" } /* END OF ACTION: compile-regex */ /* BEGINNING OF ACTION: free-arr */ { -#line 774 "src/lx/parser.act" +#line 767 "src/lx/parser.act" free((ZIa)); -#line 2055 "src/lx/parser.c" +#line 2039 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2063,7 +2047,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 261 "src/lx/parser.act" +#line 264 "src/lx/parser.act" size_t len; @@ -2088,12 +2072,12 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2092 "src/lx/parser.c" +#line 2076 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-literal */ { -#line 370 "src/lx/parser.act" +#line 365 "src/lx/parser.act" struct re_err err; @@ -2108,16 +2092,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2112 "src/lx/parser.c" +#line 2096 "src/lx/parser.c" } /* END OF ACTION: compile-literal */ /* BEGINNING OF ACTION: free-arr */ { -#line 774 "src/lx/parser.act" +#line 767 "src/lx/parser.act" free((ZIa)); -#line 2121 "src/lx/parser.c" +#line 2105 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2181,7 +2165,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2201,12 +2185,12 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } } -#line 2205 "src/lx/parser.c" +#line 2189 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 454 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2239,14 +2223,14 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho nmappings++; } -#line 2243 "src/lx/parser.c" +#line 2227 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-from-mappings::list-of-zone-from-mappings-x */ goto ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Hof_Hzone_Hfrom_Hmappings_Hx; /* END OF INLINE: list-of-things::zone-thing::list-of-zone-from-mappings::list-of-zone-from-mappings-x */ } - /* UNREACHED */ + /*UNREACHED*/ case (ERROR_TERMINAL): return; default: @@ -2273,16 +2257,16 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* BEGINNING OF ACTION: no-zone */ { -#line 523 "src/lx/parser.act" +#line 516 "src/lx/parser.act" (ZIparent) = NULL; -#line 2281 "src/lx/parser.c" +#line 2265 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: make-ast */ { -#line 432 "src/lx/parser.act" +#line 425 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2290,12 +2274,12 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2294 "src/lx/parser.c" +#line 2278 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: make-zone */ { -#line 440 "src/lx/parser.act" +#line 433 "src/lx/parser.act" assert((ZIa) != NULL); @@ -2313,28 +2297,28 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2317 "src/lx/parser.c" +#line 2301 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: no-exit */ { -#line 519 "src/lx/parser.act" +#line 512 "src/lx/parser.act" (ZIexit) = NULL; -#line 2326 "src/lx/parser.c" +#line 2310 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: set-globalzone */ { -#line 508 "src/lx/parser.act" +#line 501 "src/lx/parser.act" assert((ZIa) != NULL); assert((ZIz) != NULL); (ZIa)->global = (ZIz); -#line 2338 "src/lx/parser.c" +#line 2322 "src/lx/parser.c" } /* END OF ACTION: set-globalzone */ p_list_Hof_Hthings (lex_state, act_state, ZIa, ZIz, ZIexit); @@ -2358,11 +2342,11 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 815 "src/lx/parser.act" +#line 809 "src/lx/parser.act" err_expected(lex_state, "EOF"); -#line 2366 "src/lx/parser.c" +#line 2350 "src/lx/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2375,7 +2359,7 @@ ZL1:; { /* BEGINNING OF ACTION: make-ast */ { -#line 432 "src/lx/parser.act" +#line 425 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2383,17 +2367,17 @@ ZL1:; goto ZL4; } -#line 2387 "src/lx/parser.c" +#line 2371 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: err-syntax */ { -#line 782 "src/lx/parser.act" +#line 777 "src/lx/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 2397 "src/lx/parser.c" +#line 2381 "src/lx/parser.c" } /* END OF ACTION: err-syntax */ } @@ -2450,7 +2434,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ } /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 454 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2483,12 +2467,12 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ nmappings++; } -#line 2487 "src/lx/parser.c" +#line 2471 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF ACTION: clone */ { -#line 764 "src/lx/parser.act" +#line 757 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2498,7 +2482,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ goto ZL1; } -#line 2502 "src/lx/parser.c" +#line 2486 "src/lx/parser.c" } /* END OF ACTION: clone */ } @@ -2534,11 +2518,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: err-expected-map */ { -#line 791 "src/lx/parser.act" +#line 785 "src/lx/parser.act" err_expected(lex_state, "'->'"); -#line 2542 "src/lx/parser.c" +#line 2526 "src/lx/parser.c" } /* END OF ACTION: err-expected-map */ } @@ -2549,16 +2533,12 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) case (TOK_TOKEN): /* BEGINNING OF EXTRACT: TOKEN */ { -#line 223 "src/lx/parser.act" +#line 225 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (ZIt == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 2562 "src/lx/parser.c" +#line 2542 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ break; @@ -2572,11 +2552,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: no-token */ { -#line 515 "src/lx/parser.act" +#line 508 "src/lx/parser.act" (ZIt) = NULL; -#line 2580 "src/lx/parser.c" +#line 2560 "src/lx/parser.c" } /* END OF ACTION: no-token */ } @@ -2665,11 +2645,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-semi */ { -#line 799 "src/lx/parser.act" +#line 793 "src/lx/parser.act" err_expected(lex_state, "';'"); -#line 2673 "src/lx/parser.c" +#line 2653 "src/lx/parser.c" } /* END OF ACTION: err-expected-semi */ } @@ -2716,16 +2696,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 523 "src/lx/parser.act" +#line 516 "src/lx/parser.act" (ZIto) = NULL; -#line 2724 "src/lx/parser.c" +#line 2704 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 454 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2758,7 +2738,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2762 "src/lx/parser.c" +#line 2742 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -2770,7 +2750,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: make-zone */ { -#line 440 "src/lx/parser.act" +#line 433 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -2788,12 +2768,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 2792 "src/lx/parser.c" +#line 2772 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 454 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2826,7 +2806,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2830 "src/lx/parser.c" +#line 2810 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ p_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Hof_Hzone_Hfrom_Hmappings_Hx (lex_state, act_state, *ZIa, *ZIz, ZIchild, *ZIexit); @@ -2850,11 +2830,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-to */ { -#line 803 "src/lx/parser.act" +#line 797 "src/lx/parser.act" err_expected(lex_state, "'..'"); -#line 2858 "src/lx/parser.c" +#line 2838 "src/lx/parser.c" } /* END OF ACTION: err-expected-to */ } @@ -2879,25 +2859,25 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 523 "src/lx/parser.act" +#line 516 "src/lx/parser.act" (ZIx) = NULL; -#line 2887 "src/lx/parser.c" +#line 2867 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: no-token */ { -#line 515 "src/lx/parser.act" +#line 508 "src/lx/parser.act" (ZIy) = NULL; -#line 2896 "src/lx/parser.c" +#line 2876 "src/lx/parser.c" } /* END OF ACTION: no-token */ /* BEGINNING OF ACTION: regex-any */ { -#line 401 "src/lx/parser.act" +#line 396 "src/lx/parser.act" fsm_state_t start, end; @@ -2928,12 +2908,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit fsm_setstart((ZIw), start); fsm_setend((ZIw), end, 1); -#line 2932 "src/lx/parser.c" +#line 2912 "src/lx/parser.c" } /* END OF ACTION: regex-any */ /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZIw) != NULL); @@ -2953,12 +2933,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 2957 "src/lx/parser.c" +#line 2937 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 454 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2991,7 +2971,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2995 "src/lx/parser.c" +#line 2975 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -3018,11 +2998,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 817 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3026 "src/lx/parser.c" +#line 3006 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3038,16 +3018,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: no-exit */ { -#line 519 "src/lx/parser.act" +#line 512 "src/lx/parser.act" (ZIr2) = NULL; -#line 3046 "src/lx/parser.c" +#line 3026 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: make-zone */ { -#line 440 "src/lx/parser.act" +#line 433 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -3065,12 +3045,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3069 "src/lx/parser.c" +#line 3049 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 454 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -3103,7 +3083,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 3107 "src/lx/parser.c" +#line 3087 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: 131 */ @@ -3122,11 +3102,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 817 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3130 "src/lx/parser.c" +#line 3110 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3170,11 +3150,11 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-bind */ { -#line 795 "src/lx/parser.act" +#line 789 "src/lx/parser.act" err_expected(lex_state, "'='"); -#line 3178 "src/lx/parser.c" +#line 3158 "src/lx/parser.c" } /* END OF ACTION: err-expected-bind */ } @@ -3189,7 +3169,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: add-binding */ { -#line 491 "src/lx/parser.act" +#line 486 "src/lx/parser.act" struct var *v; @@ -3206,7 +3186,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3210 "src/lx/parser.c" +#line 3190 "src/lx/parser.c" } /* END OF ACTION: add-binding */ } @@ -3232,7 +3212,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: deref-var */ { -#line 286 "src/lx/parser.act" +#line 281 "src/lx/parser.act" struct ast_zone *z; @@ -3259,7 +3239,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3263 "src/lx/parser.c" +#line 3243 "src/lx/parser.c" } /* END OF ACTION: deref-var */ p_197 (lex_state, act_state, ZI273, &ZI210); @@ -3275,7 +3255,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 737 "src/lx/parser.act" assert((ZI278) != NULL); @@ -3295,7 +3275,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 3299 "src/lx/parser.c" +#line 3279 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, ZIa, ZIz, ZIexit, &ZI279, &ZI280); @@ -3318,7 +3298,7 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 827 "src/lx/parser.act" +#line 881 "src/lx/parser.act" struct ast *lx_parse(FILE *f, const struct fsm_options *opt) { @@ -3381,6 +3361,6 @@ ZL1:; return ast; } -#line 3385 "src/lx/parser.c" +#line 3365 "src/lx/parser.c" /* END OF FILE */ diff --git a/src/lx/parser.h b/src/lx/parser.h index bde7c36ab..970fe2b0e 100644 --- a/src/lx/parser.h +++ b/src/lx/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 128 "src/lx/parser.act" +#line 140 "src/lx/parser.act" #include @@ -29,7 +29,7 @@ extern void p_lx(lex_state, act_state, ast *); /* BEGINNING OF TRAILER */ -#line 889 "src/lx/parser.act" +#line 883 "src/lx/parser.act" #line 36 "src/lx/parser.h" diff --git a/src/retest/cvtpcre.c b/src/retest/cvtpcre.c index 4bcd6e63a..633431f7d 100644 --- a/src/retest/cvtpcre.c +++ b/src/retest/cvtpcre.c @@ -263,10 +263,6 @@ parse_modifiers(size_t linenum, const char *s, enum re_flags *mods_p) assert(s != NULL); buf = xstrdup(s); - if (buf == NULL) { - return 0; - } - pos = tok = buf; first = true; From 825a6218314c1afcf7aa63267ca070a37e80bf4b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 31 May 2024 08:09:21 -0700 Subject: [PATCH 124/373] No need to depend on POSIX here. --- src/adt/xalloc.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/adt/xalloc.c b/src/adt/xalloc.c index bdce10a49..91ee82431 100644 --- a/src/adt/xalloc.c +++ b/src/adt/xalloc.c @@ -4,8 +4,6 @@ * See LICENCE for the full copyright terms. */ -#define _POSIX_C_SOURCE 200809L - #include #include #include @@ -29,13 +27,14 @@ xstrdup(const char *s) char * xstrndup(const char *s, size_t n) { - char *new = strndup(s, n); + char *new; + + new = malloc(n); if (new == NULL) { - perror("strndup"); - abort(); + return NULL; } - return new; + return memcpy(new, s, n); } void * From 2afe5637728c5dcbf6d9aa53f5b390290cc636a5 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 7 Jun 2024 15:52:29 +0100 Subject: [PATCH 125/373] Default to printing endids when no .endleaf callback is present. --- src/libfsm/print/dot.c | 54 +++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 25f6ea4a7..ab360af2b 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -37,16 +37,6 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) assert(fsm->opt != NULL); assert(s < fsm->statecount); - if (!fsm->opt->anonymous_states) { - fprintf(f, "\t%sS%-2u [ label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s); - - fprintf(f, "%u", s); - - fprintf(f, "> ];\n"); - } - if (!fsm->opt->consolidate_edges) { { struct state_iter jt; @@ -163,18 +153,24 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) static int print_dotfrag(FILE *f, const struct fsm *fsm) { - fsm_state_t i; + fsm_state_t s; assert(f != NULL); assert(fsm != NULL); assert(fsm->opt != NULL); - for (i = 0; i < fsm->statecount; i++) { - if (fsm_isend(fsm, i)) { + for (s = 0; s < fsm->statecount; s++) { + if (!fsm_isend(fsm, s)) { + if (!fsm->opt->anonymous_states) { + fprintf(f, "\t%sS%-2u [ label = <%u> ];\n", + fsm->opt->prefix != NULL ? fsm->opt->prefix : "", + s, s); + } + } else { enum fsm_getendids_res res; size_t written; struct fsm_end_ids *ids = NULL; - const size_t count = fsm_getendidcount(fsm, i); + const size_t count = fsm_getendidcount(fsm, s); if (count > 0) { ids = f_malloc(fsm->opt->alloc, sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); @@ -183,7 +179,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_getendids(fsm, i, count, + res = fsm_getendids(fsm, s, count, ids->ids, &written); if (res == FSM_GETENDIDS_FOUND) { ids->count = (unsigned)written; @@ -193,13 +189,37 @@ print_dotfrag(FILE *f, const struct fsm *fsm) } fprintf(f, "\t%sS%-2u [ shape = doublecircle", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", i); + fsm->opt->prefix != NULL ? fsm->opt->prefix : "", s); if (fsm->opt->endleaf != NULL) { fprintf(f, ", "); if (-1 == fsm->opt->endleaf(f, ids, fsm->opt->endleaf_opaque)) { return -1; } + } else { + assert(f != NULL); + + fprintf(f, ", "); + + fprintf(f, "label = <"); + + if (!fsm->opt->anonymous_states) { + fprintf(f, "%u", s); + + if (ids->count > 0) { + fprintf(f, "
"); + } + } + + for (size_t i = 0; i < ids->count; i++) { + fprintf(f, "#%u", ids->ids[i]); + + if (i < ids->count - 1) { + fprintf(f, ","); + } + } + + fprintf(f, ">"); } fprintf(f, " ];\n"); @@ -209,7 +229,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) /* TODO: show example here, unless !opt->comments */ - if (-1 == singlestate(f, fsm, i)) { + if (-1 == singlestate(f, fsm, s)) { return -1; } } From ffea1decb956b0f396405c60a514ea6f049eecac Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 7 Jun 2024 16:31:37 +0100 Subject: [PATCH 126/373] Return a set of endids, rather than the state id. --- src/libfsm/print/awk.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index b71ffc307..c7083e6e2 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -91,6 +91,8 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits, const struct ir *ir) { + (void) ir; + if (end_bits == VM_END_FAIL) { fprintf(f, "return -1"); return 0; @@ -101,7 +103,22 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return %td", op->ir_state - ir->states); + size_t i; + + assert(f != NULL); + + /* awk can't return an array */ + fprintf(f, "return \""); + + for (i = 0; i < op->ir_state->end_ids->count; i++) { + fprintf(f, "%u", op->ir_state->end_ids->ids[i]); + + if (i < op->ir_state->end_ids->count - 1) { + fprintf(f, ","); + } + } + + fprintf(f, "\""); } return 0; From f84aa387d96644063d5c429c7819b065e822cca9 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 8 Jun 2024 07:02:14 -0700 Subject: [PATCH 127/373] Naming. I'm keeping "id" free. --- src/libfsm/parser.c | 412 +++++++++++++++++++++--------------------- src/libfsm/parser.sid | 22 +-- 2 files changed, 217 insertions(+), 217 deletions(-) diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 19d5664f3..998e9ee75 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -146,13 +146,13 @@ static void p_label(fsm, lex_state, act_state, char *); static void p_items(fsm, lex_state, act_state); +static void p_xend_C_Cend_Hstates(fsm, lex_state, act_state); static void p_xstart(fsm, lex_state, act_state); static void p_xend(fsm, lex_state, act_state); +static void p_xend_C_Cend_Hstate(fsm, lex_state, act_state); static void p_61(fsm, lex_state, act_state); static void p_63(fsm, lex_state, act_state, string *); extern void p_fsm(fsm, lex_state, act_state); -static void p_xend_C_Cend_Hids(fsm, lex_state, act_state); -static void p_xend_C_Cend_Hid(fsm, lex_state, act_state); /* BEGINNING OF STATIC VARIABLES */ @@ -323,7 +323,7 @@ ZL2_items:; { string ZIa; - /* BEGINNING OF INLINE: id */ + /* BEGINNING OF INLINE: ident */ { { switch (CURRENT_TERMINAL) { @@ -345,7 +345,7 @@ ZL2_items:; ADVANCE_LEXER; } } - /* END OF INLINE: id */ + /* END OF INLINE: ident */ p_63 (fsm, lex_state, act_state, &ZIa); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -370,6 +370,67 @@ ZL1:; return; } +static void +p_xend_C_Cend_Hstates(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } +ZL2_xend_C_Cend_Hstates:; + { + p_xend_C_Cend_Hstate (fsm, lex_state, act_state); + /* BEGINNING OF INLINE: 58 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + { + /* BEGINNING OF INLINE: 59 */ + { + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + break; + default: + goto ZL5; + } + ADVANCE_LEXER; + } + goto ZL4; + ZL5:; + { + /* BEGINNING OF ACTION: err-expected-comma */ + { +#line 362 "src/libfsm/parser.act" + + err_expected(lex_state, "','"); + +#line 408 "src/libfsm/parser.c" + } + /* END OF ACTION: err-expected-comma */ + } + ZL4:; + } + /* END OF INLINE: 59 */ + /* BEGINNING OF INLINE: xend::end-states */ + goto ZL2_xend_C_Cend_Hstates; + /* END OF INLINE: xend::end-states */ + } + /*UNREACHED*/ + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + default: + break; + } + } + /* END OF INLINE: 58 */ + } + return; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +} + static void p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { @@ -399,14 +460,14 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) err_expected(lex_state, "'start:'"); -#line 403 "src/libfsm/parser.c" +#line 464 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-start */ } ZL2:; } /* END OF INLINE: 50 */ - /* BEGINNING OF INLINE: id */ + /* BEGINNING OF INLINE: ident */ { { switch (CURRENT_TERMINAL) { @@ -418,7 +479,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); -#line 422 "src/libfsm/parser.c" +#line 483 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -428,7 +489,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) ADVANCE_LEXER; } } - /* END OF INLINE: id */ + /* END OF INLINE: ident */ p_61 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -489,7 +550,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 493 "src/libfsm/parser.c" +#line 554 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ @@ -498,7 +559,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) fsm_setstart(fsm, (ZIs)); -#line 502 "src/libfsm/parser.c" +#line 563 "src/libfsm/parser.c" } /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ @@ -507,7 +568,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) free((ZIn)); -#line 511 "src/libfsm/parser.c" +#line 572 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -549,14 +610,14 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) err_expected(lex_state, "'end:'"); -#line 553 "src/libfsm/parser.c" +#line 614 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-end */ } ZL2:; } /* END OF INLINE: 60 */ - p_xend_C_Cend_Hids (fsm, lex_state, act_state); + p_xend_C_Cend_Hstates (fsm, lex_state, act_state); p_61 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -575,6 +636,122 @@ ZL1:; return; } +static void +p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + string ZIn; + state ZIs; + + /* BEGINNING OF INLINE: ident */ + { + { + switch (CURRENT_TERMINAL) { + case (TOK_IDENT): + /* BEGINNING OF EXTRACT: IDENT */ + { +#line 245 "src/libfsm/parser.act" + + /* XXX: don't exit in library code */ + ZIn = xstrdup(lex_state->buf.a); + +#line 662 "src/libfsm/parser.c" + } + /* END OF EXTRACT: IDENT */ + break; + default: + goto ZL1; + } + ADVANCE_LEXER; + } + } + /* END OF INLINE: ident */ + /* BEGINNING OF ACTION: add-state */ + { +#line 252 "src/libfsm/parser.act" + + struct act_statelist *p; + const unsigned hash = hash_of_id((ZIn)); + const unsigned mask = act_state->states.bucket_count - 1; + size_t chain_length = 0; + unsigned bucket_id = hash & mask; + + assert((ZIn) != NULL); + + if (act_state->states.longest_chain_length > MAX_CHAIN_LENGTH) { + grow_state_hash_table(&act_state->states); + bucket_id = hash & (act_state->states.bucket_count - 1); + } + + for (p = act_state->states.buckets[bucket_id]; p != NULL; p = p->next) { + assert(p->id != NULL); + + chain_length++; + + if (0 == strcmp(p->id, (ZIn))) { + (ZIs) = p->state; + break; + } + } + + if (chain_length > act_state->states.longest_chain_length) { + act_state->states.longest_chain_length = chain_length; + } + + if (p == NULL) { + struct act_statelist *new; + + new = malloc(sizeof *new); + if (new == NULL) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + /* XXX: don't exit in library code */ + new->id = xstrdup((ZIn)); + + if (!fsm_addstate(fsm, &(ZIs))) { + perror("fsm_addstate"); + exit(EXIT_FAILURE); + } + + new->state = (ZIs); + + new->next = act_state->states.buckets[bucket_id]; + act_state->states.buckets[bucket_id] = new; + } + +#line 728 "src/libfsm/parser.c" + } + /* END OF ACTION: add-state */ + /* BEGINNING OF ACTION: mark-end */ + { +#line 308 "src/libfsm/parser.act" + + fsm_setend(fsm, (ZIs), 1); + +#line 737 "src/libfsm/parser.c" + } + /* END OF ACTION: mark-end */ + /* BEGINNING OF ACTION: free */ + { +#line 312 "src/libfsm/parser.act" + + free((ZIn)); + +#line 746 "src/libfsm/parser.c" + } + /* END OF ACTION: free */ + } + return; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +} + static void p_61(fsm fsm, lex_state lex_state, act_state act_state) { @@ -599,7 +776,7 @@ ZL1:; err_expected(lex_state, "';'"); -#line 603 "src/libfsm/parser.c" +#line 780 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-sep */ } @@ -616,7 +793,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) state ZIy; ADVANCE_LEXER; - /* BEGINNING OF INLINE: id */ + /* BEGINNING OF INLINE: ident */ { { switch (CURRENT_TERMINAL) { @@ -628,7 +805,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); -#line 632 "src/libfsm/parser.c" +#line 809 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -638,7 +815,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; } } - /* END OF INLINE: id */ + /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { #line 252 "src/libfsm/parser.act" @@ -694,7 +871,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 698 "src/libfsm/parser.c" +#line 875 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ @@ -752,7 +929,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 756 "src/libfsm/parser.c" +#line 933 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ @@ -761,7 +938,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) free((*ZIa)); -#line 765 "src/libfsm/parser.c" +#line 942 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ @@ -770,7 +947,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) free((ZIb)); -#line 774 "src/libfsm/parser.c" +#line 951 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF INLINE: 42 */ @@ -788,7 +965,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } -#line 792 "src/libfsm/parser.c" +#line 969 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-any */ } @@ -811,7 +988,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } -#line 815 "src/libfsm/parser.c" +#line 992 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-literal */ } @@ -827,7 +1004,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } -#line 831 "src/libfsm/parser.c" +#line 1008 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-epsilon */ } @@ -842,7 +1019,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) err_expected(lex_state, "transition"); -#line 846 "src/libfsm/parser.c" +#line 1023 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-trans */ } @@ -915,7 +1092,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 919 "src/libfsm/parser.c" +#line 1096 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ @@ -924,7 +1101,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) free((*ZIa)); -#line 928 "src/libfsm/parser.c" +#line 1105 "src/libfsm/parser.c" } /* END OF ACTION: free */ p_61 (fsm, lex_state, act_state); @@ -982,7 +1159,7 @@ p_fsm(fsm fsm, lex_state lex_state, act_state act_state) } } -#line 986 "src/libfsm/parser.c" +#line 1163 "src/libfsm/parser.c" } /* END OF ACTION: free-statelist */ } @@ -996,189 +1173,12 @@ ZL1:; err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 1000 "src/libfsm/parser.c" +#line 1177 "src/libfsm/parser.c" } /* END OF ACTION: err-syntax */ } } -static void -p_xend_C_Cend_Hids(fsm fsm, lex_state lex_state, act_state act_state) -{ - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } -ZL2_xend_C_Cend_Hids:; - { - p_xend_C_Cend_Hid (fsm, lex_state, act_state); - /* BEGINNING OF INLINE: 58 */ - { - switch (CURRENT_TERMINAL) { - case (TOK_COMMA): - { - /* BEGINNING OF INLINE: 59 */ - { - { - switch (CURRENT_TERMINAL) { - case (TOK_COMMA): - break; - default: - goto ZL5; - } - ADVANCE_LEXER; - } - goto ZL4; - ZL5:; - { - /* BEGINNING OF ACTION: err-expected-comma */ - { -#line 362 "src/libfsm/parser.act" - - err_expected(lex_state, "','"); - -#line 1040 "src/libfsm/parser.c" - } - /* END OF ACTION: err-expected-comma */ - } - ZL4:; - } - /* END OF INLINE: 59 */ - /* BEGINNING OF INLINE: xend::end-ids */ - goto ZL2_xend_C_Cend_Hids; - /* END OF INLINE: xend::end-ids */ - } - /*UNREACHED*/ - case (ERROR_TERMINAL): - RESTORE_LEXER; - goto ZL1; - default: - break; - } - } - /* END OF INLINE: 58 */ - } - return; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -} - -static void -p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state) -{ - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - string ZIn; - state ZIs; - - /* BEGINNING OF INLINE: id */ - { - { - switch (CURRENT_TERMINAL) { - case (TOK_IDENT): - /* BEGINNING OF EXTRACT: IDENT */ - { -#line 245 "src/libfsm/parser.act" - - /* XXX: don't exit in library code */ - ZIn = xstrdup(lex_state->buf.a); - -#line 1089 "src/libfsm/parser.c" - } - /* END OF EXTRACT: IDENT */ - break; - default: - goto ZL1; - } - ADVANCE_LEXER; - } - } - /* END OF INLINE: id */ - /* BEGINNING OF ACTION: add-state */ - { -#line 252 "src/libfsm/parser.act" - - struct act_statelist *p; - const unsigned hash = hash_of_id((ZIn)); - const unsigned mask = act_state->states.bucket_count - 1; - size_t chain_length = 0; - unsigned bucket_id = hash & mask; - - assert((ZIn) != NULL); - - if (act_state->states.longest_chain_length > MAX_CHAIN_LENGTH) { - grow_state_hash_table(&act_state->states); - bucket_id = hash & (act_state->states.bucket_count - 1); - } - - for (p = act_state->states.buckets[bucket_id]; p != NULL; p = p->next) { - assert(p->id != NULL); - - chain_length++; - - if (0 == strcmp(p->id, (ZIn))) { - (ZIs) = p->state; - break; - } - } - - if (chain_length > act_state->states.longest_chain_length) { - act_state->states.longest_chain_length = chain_length; - } - - if (p == NULL) { - struct act_statelist *new; - - new = malloc(sizeof *new); - if (new == NULL) { - perror("malloc"); - exit(EXIT_FAILURE); - } - - /* XXX: don't exit in library code */ - new->id = xstrdup((ZIn)); - - if (!fsm_addstate(fsm, &(ZIs))) { - perror("fsm_addstate"); - exit(EXIT_FAILURE); - } - - new->state = (ZIs); - - new->next = act_state->states.buckets[bucket_id]; - act_state->states.buckets[bucket_id] = new; - } - -#line 1155 "src/libfsm/parser.c" - } - /* END OF ACTION: add-state */ - /* BEGINNING OF ACTION: mark-end */ - { -#line 308 "src/libfsm/parser.act" - - fsm_setend(fsm, (ZIs), 1); - -#line 1164 "src/libfsm/parser.c" - } - /* END OF ACTION: mark-end */ - /* BEGINNING OF ACTION: free */ - { -#line 312 "src/libfsm/parser.act" - - free((ZIn)); - -#line 1173 "src/libfsm/parser.c" - } - /* END OF ACTION: free */ - } - return; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -} - /* BEGINNING OF TRAILER */ #line 431 "src/libfsm/parser.act" diff --git a/src/libfsm/parser.sid b/src/libfsm/parser.sid index d1da43b30..32a370312 100644 --- a/src/libfsm/parser.sid +++ b/src/libfsm/parser.sid @@ -64,14 +64,14 @@ LABEL; }; - id: () -> (n :string) = { + ident: () -> (n :string) = { n = IDENT; }; edge = { - a = id; + a = ident; TO; - b = id; + b = ident; x = (a); y = (b); @@ -106,7 +106,7 @@ * its own output. */ decl = { - a = id; + a = ident; ! = (a); @@ -137,7 +137,7 @@ ; }; - n = id; + n = ident; { SEP; @@ -155,8 +155,8 @@ xend [ - end-id = { - n = id; + end-state = { + n = ident; s = (n); (s); @@ -164,8 +164,8 @@ (n); }; - end-ids = { - end-id; + end-states = { + end-state; { { @@ -174,7 +174,7 @@ ; }; - end-ids; + end-states; || $; }; @@ -187,7 +187,7 @@ ; }; - end-ids; + end-states; { SEP; From a7d2e505d5c3dc997f7fb45579c145e2a5c130b0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 10 Jun 2024 01:46:16 -0700 Subject: [PATCH 128/373] Add syntax for endids in .fsm format. These are optional. I'd expected this to be per state at file scope, but I don't like that the syntax would then allow attempting to attach an id to a non-accepting state. Since we have a block for `end:` already, adding the end ids here means they can't be mixed up. This looks like: ``` ; ./build/bin/re -r pcre -bpl fsm 'ab?c' 'abc*' 0 -> 1 "a"; # e.g. "a" 1 -> 2 "b"; # e.g. "ab" 1 -> 3 "c"; # e.g. "ac" 2 -> 4 "c"; # e.g. "abc" 4 -> 5 "c"; # e.g. "abcc" 5 -> 5 "c"; # e.g. "abcc" start: 0; end: 2 = [0], 3 = [0], 4 = [0, 1], 5 = [0]; ``` --- src/libfsm/lexer.c | 539 ++++++++++++++++++++++++++++++----------- src/libfsm/lexer.h | 6 +- src/libfsm/lexer.lx | 11 + src/libfsm/parser.act | 43 +++- src/libfsm/parser.c | 471 +++++++++++++++++++++++------------ src/libfsm/parser.h | 4 +- src/libfsm/parser.sid | 45 +++- src/libfsm/print/fsm.c | 66 ++++- 8 files changed, 876 insertions(+), 309 deletions(-) diff --git a/src/libfsm/lexer.c b/src/libfsm/lexer.c index 7213bb142..8bd374cec 100644 --- a/src/libfsm/lexer.c +++ b/src/libfsm/lexer.c @@ -12,6 +12,8 @@ static enum lx_token z0(struct lx *lx); static enum lx_token z1(struct lx *lx); static enum lx_token z2(struct lx *lx); static enum lx_token z3(struct lx *lx); +static enum lx_token z4(struct lx *lx); +static enum lx_token z5(struct lx *lx); #if __STDC_VERSION__ >= 199901L inline @@ -180,25 +182,35 @@ z0(struct lx *lx) switch (state) { case S0: /* start */ switch ((unsigned char) c) { - case '\'': state = S2; break; + case '\n': state = S2; break; default: state = S1; break; } break; case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + lx_ungetc(lx, c); return lx->z(lx); - case S2: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z3, TOK_LABEL; + case S2: /* e.g. "" */ + lx_ungetc(lx, c); return lx->z = z1, lx->z(lx); default: ; /* unreached */ } - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { - return TOK_ERROR; + switch (state) { + case S0: + case S1: + case S2: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } } + break; + } } @@ -206,8 +218,8 @@ z0(struct lx *lx) switch (state) { case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_LABEL; + case S1: return TOK_EOF; + case S2: return TOK_EOF; default: errno = EINVAL; return TOK_ERROR; } } @@ -218,7 +230,7 @@ z1(struct lx *lx) int c; enum { - S0, S1, S2, S3, S4, S5, S6, S7, NONE + S0, S1, S2, S3, S4, S5, NONE } state; assert(lx != NULL); @@ -239,27 +251,29 @@ z1(struct lx *lx) switch (state) { case S0: /* start */ switch ((unsigned char) c) { - case '"': state = S2; break; - case '\\': state = S3; break; - default: state = S1; break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': state = S1; break; + case ',': state = S2; break; + case '#': state = S3; break; + case '\t': + case '\n': + case '\r': + case ' ': state = S4; break; + case ']': state = S5; break; + default: lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; - - case S2: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z3, TOK_LABEL; - - case S3: /* e.g. "\\" */ + case S1: /* e.g. "0" */ switch ((unsigned char) c) { - case '"': - case '\\': - case 'f': - case 'n': - case 'r': - case 't': - case 'v': state = S4; break; case '0': case '1': case '2': @@ -267,17 +281,160 @@ z1(struct lx *lx) case '4': case '5': case '6': - case '7': state = S5; break; - case 'x': state = S6; break; - default: lx_ungetc(lx, c); return TOK_CHAR; + case '7': + case '8': + case '9': break; + default: lx_ungetc(lx, c); return TOK_ENDID; } break; - case S4: /* e.g. "\\f" */ - lx_ungetc(lx, c); return TOK_ESC; + case S2: /* e.g. "," */ + lx_ungetc(lx, c); return TOK_COMMA; - case S5: /* e.g. "\\0" */ + case S3: /* e.g. "#" */ + lx_ungetc(lx, c); return lx->z = z0, lx->z(lx); + + case S4: /* e.g. "\\x09" */ + switch ((unsigned char) c) { + case '\t': + case '\n': + case '\r': + case ' ': break; + default: lx_ungetc(lx, c); return lx->z(lx); + } + break; + + case S5: /* e.g. "]" */ + lx_ungetc(lx, c); return lx->z = z5, TOK_CLOSEENDIDS; + + default: + ; /* unreached */ + } + + switch (state) { + case S3: + case S4: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + break; + + } + } + + lx->lgetc = NULL; + + switch (state) { + case NONE: return TOK_EOF; + case S1: return TOK_ENDID; + case S2: return TOK_COMMA; + case S3: return TOK_EOF; + case S4: return TOK_EOF; + case S5: return TOK_CLOSEENDIDS; + default: errno = EINVAL; return TOK_ERROR; + } +} + +static enum lx_token +z2(struct lx *lx) +{ + int c; + + enum { + S0, S1, S2, NONE + } state; + + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + state = NONE; + + lx->start = lx->end; + + while (c = lx_getc(lx), c != EOF) { + if (state == NONE) { + state = S0; + } + + switch (state) { + case S0: /* start */ switch ((unsigned char) c) { + case '\'': state = S2; break; + default: state = S1; break; + } + break; + + case S1: /* e.g. "a" */ + lx_ungetc(lx, c); return TOK_CHAR; + + case S2: /* e.g. "'" */ + lx_ungetc(lx, c); return lx->z = z5, TOK_LABEL; + + default: + ; /* unreached */ + } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + } + + lx->lgetc = NULL; + + switch (state) { + case NONE: return TOK_EOF; + case S1: return TOK_CHAR; + case S2: return TOK_LABEL; + default: errno = EINVAL; return TOK_ERROR; + } +} + +static enum lx_token +z3(struct lx *lx) +{ + int c; + + enum { + S0, S1, S2, S3, S4, S5, S6, S7, NONE + } state; + + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + state = NONE; + + lx->start = lx->end; + + while (c = lx_getc(lx), c != EOF) { + if (state == NONE) { + state = S0; + } + + switch (state) { + case S0: /* start */ + switch ((unsigned char) c) { + case '\\': state = S1; break; + case '"': state = S3; break; + default: state = S2; break; + } + break; + + case S1: /* e.g. "\\" */ + switch ((unsigned char) c) { + case 'x': state = S4; break; case '0': case '1': case '2': @@ -285,12 +442,25 @@ z1(struct lx *lx) case '4': case '5': case '6': - case '7': break; - default: lx_ungetc(lx, c); return TOK_OCT; + case '7': state = S5; break; + case '"': + case '\\': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': state = S6; break; + default: lx_ungetc(lx, c); return TOK_CHAR; } break; - case S6: /* e.g. "\\x" */ + case S2: /* e.g. "a" */ + lx_ungetc(lx, c); return TOK_CHAR; + + case S3: /* e.g. "\"" */ + lx_ungetc(lx, c); return lx->z = z5, TOK_LABEL; + + case S4: /* e.g. "\\x" */ switch ((unsigned char) c) { case '0': case '1': @@ -318,6 +488,23 @@ z1(struct lx *lx) } break; + case S5: /* e.g. "\\0" */ + switch ((unsigned char) c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': break; + default: lx_ungetc(lx, c); return TOK_OCT; + } + break; + + case S6: /* e.g. "\\f" */ + lx_ungetc(lx, c); return TOK_ESC; + case S7: /* e.g. "\\xa" */ switch ((unsigned char) c) { case '0': @@ -351,7 +538,7 @@ z1(struct lx *lx) } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } @@ -362,17 +549,17 @@ z1(struct lx *lx) switch (state) { case NONE: return TOK_EOF; case S1: return TOK_CHAR; - case S2: return TOK_LABEL; - case S3: return TOK_CHAR; - case S4: return TOK_ESC; + case S2: return TOK_CHAR; + case S3: return TOK_LABEL; case S5: return TOK_OCT; + case S6: return TOK_ESC; case S7: return TOK_HEX; default: errno = EINVAL; return TOK_ERROR; } } static enum lx_token -z2(struct lx *lx) +z4(struct lx *lx) { int c; @@ -407,7 +594,7 @@ z2(struct lx *lx) lx_ungetc(lx, c); return lx->z(lx); case S2: /* e.g. "" */ - lx_ungetc(lx, c); return lx->z = z3, lx->z(lx); + lx_ungetc(lx, c); return lx->z = z5, lx->z(lx); default: ; /* unreached */ @@ -421,7 +608,7 @@ z2(struct lx *lx) default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } @@ -441,14 +628,14 @@ z2(struct lx *lx) } static enum lx_token -z3(struct lx *lx) +z5(struct lx *lx) { int c; enum { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, NONE + S20, S21, S22, NONE } state; assert(lx != NULL); @@ -469,15 +656,14 @@ z3(struct lx *lx) switch (state) { case S0: /* start */ switch ((unsigned char) c) { - case '\t': - case '\n': - case '\r': - case ' ': state = S1; break; - case '"': state = S2; break; - case '#': state = S3; break; - case '\'': state = S4; break; - case ',': state = S5; break; - case '-': state = S6; break; + case ',': state = S1; break; + case ';': state = S2; break; + case '?': state = S3; break; + case '-': state = S4; break; + case '[': state = S5; break; + case '=': state = S6; break; + case 'e': state = S7; break; + case 's': state = S8; break; case '0': case '1': case '2': @@ -538,45 +724,41 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case ';': state = S8; break; - case '?': state = S9; break; - case 'e': state = S10; break; - case 's': state = S11; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; - - case S1: /* e.g. "\\x09" */ - switch ((unsigned char) c) { + case 'z': state = S9; break; + case '\'': state = S10; break; + case '"': state = S11; break; + case '#': state = S12; break; case '\t': case '\n': case '\r': - case ' ': break; - default: lx_ungetc(lx, c); return lx->z(lx); + case ' ': state = S13; break; + default: lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S2: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z1, lx->z(lx); - - case S3: /* e.g. "#" */ - lx_ungetc(lx, c); return lx->z = z2, lx->z(lx); + case S1: /* e.g. "," */ + lx_ungetc(lx, c); return TOK_COMMA; - case S4: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z0, lx->z(lx); + case S2: /* e.g. ";" */ + lx_ungetc(lx, c); return TOK_SEP; - case S5: /* e.g. "," */ - lx_ungetc(lx, c); return TOK_COMMA; + case S3: /* e.g. "?" */ + lx_ungetc(lx, c); return TOK_ANY; - case S6: /* e.g. "-" */ + case S4: /* e.g. "-" */ switch ((unsigned char) c) { - case '>': state = S20; break; + case '>': state = S22; break; default: lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S7: /* e.g. "a" */ + case S5: /* e.g. "[" */ + lx_ungetc(lx, c); return lx->z = z1, TOK_OPENENDIDS; + + case S6: /* e.g. "=" */ + lx_ungetc(lx, c); return TOK_EQUALS; + + case S7: /* e.g. "e" */ switch ((unsigned char) c) { case '0': case '1': @@ -628,7 +810,6 @@ z3(struct lx *lx) case 'k': case 'l': case 'm': - case 'n': case 'o': case 'p': case 'q': @@ -640,18 +821,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': break; + case 'z': state = S9; break; + case 'n': state = S19; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S8: /* e.g. ";" */ - lx_ungetc(lx, c); return TOK_SEP; - - case S9: /* e.g. "?" */ - lx_ungetc(lx, c); return TOK_ANY; - - case S10: /* e.g. "e" */ + case S8: /* e.g. "s" */ switch ((unsigned char) c) { case '0': case '1': @@ -703,24 +879,24 @@ z3(struct lx *lx) case 'k': case 'l': case 'm': + case 'n': case 'o': case 'p': case 'q': case 'r': case 's': - case 't': case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'n': state = S17; break; + case 'z': state = S9; break; + case 't': state = S14; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S11: /* e.g. "s" */ + case S9: /* e.g. "a" */ switch ((unsigned char) c) { case '0': case '1': @@ -778,18 +954,37 @@ z3(struct lx *lx) case 'q': case 'r': case 's': + case 't': case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 't': state = S12; break; + case 'z': break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S12: /* e.g. "st" */ + case S10: /* e.g. "'" */ + lx_ungetc(lx, c); return lx->z = z2, lx->z(lx); + + case S11: /* e.g. "\"" */ + lx_ungetc(lx, c); return lx->z = z3, lx->z(lx); + + case S12: /* e.g. "#" */ + lx_ungetc(lx, c); return lx->z = z4, lx->z(lx); + + case S13: /* e.g. "\\x09" */ + switch ((unsigned char) c) { + case '\t': + case '\n': + case '\r': + case ' ': break; + default: lx_ungetc(lx, c); return lx->z(lx); + } + break; + + case S14: /* e.g. "st" */ switch ((unsigned char) c) { case '0': case '1': @@ -852,13 +1047,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'a': state = S13; break; + case 'z': state = S9; break; + case 'a': state = S15; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S13: /* e.g. "sta" */ + case S15: /* e.g. "sta" */ switch ((unsigned char) c) { case '0': case '1': @@ -921,13 +1116,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'r': state = S14; break; + case 'z': state = S9; break; + case 'r': state = S16; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S14: /* e.g. "star" */ + case S16: /* e.g. "star" */ switch ((unsigned char) c) { case '0': case '1': @@ -990,13 +1185,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 't': state = S15; break; + case 'z': state = S9; break; + case 't': state = S17; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S15: /* e.g. "start" */ + case S17: /* e.g. "start" */ switch ((unsigned char) c) { case '0': case '1': @@ -1060,16 +1255,16 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case ':': state = S16; break; + case 'z': state = S9; break; + case ':': state = S18; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S16: /* e.g. "start:" */ + case S18: /* e.g. "start:" */ lx_ungetc(lx, c); return TOK_START; - case S17: /* e.g. "en" */ + case S19: /* e.g. "en" */ switch ((unsigned char) c) { case '0': case '1': @@ -1132,13 +1327,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'd': state = S18; break; + case 'z': state = S9; break; + case 'd': state = S20; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S18: /* e.g. "end" */ + case S20: /* e.g. "end" */ switch ((unsigned char) c) { case '0': case '1': @@ -1202,16 +1397,16 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case ':': state = S19; break; + case 'z': state = S9; break; + case ':': state = S21; break; default: lx_ungetc(lx, c); return TOK_IDENT; } break; - case S19: /* e.g. "end:" */ + case S21: /* e.g. "end:" */ lx_ungetc(lx, c); return TOK_END; - case S20: /* e.g. "->" */ + case S22: /* e.g. "->" */ lx_ungetc(lx, c); return TOK_TO; default: @@ -1219,15 +1414,15 @@ z3(struct lx *lx) } switch (state) { - case S1: - case S2: - case S3: - case S4: + case S10: + case S11: + case S12: + case S13: break; default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } @@ -1240,25 +1435,27 @@ z3(struct lx *lx) switch (state) { case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - case S3: return TOK_EOF; - case S4: return TOK_EOF; - case S5: return TOK_COMMA; + case S1: return TOK_COMMA; + case S2: return TOK_SEP; + case S3: return TOK_ANY; + case S5: return TOK_OPENENDIDS; + case S6: return TOK_EQUALS; case S7: return TOK_IDENT; - case S8: return TOK_SEP; - case S9: return TOK_ANY; - case S10: return TOK_IDENT; - case S11: return TOK_IDENT; - case S12: return TOK_IDENT; - case S13: return TOK_IDENT; + case S8: return TOK_IDENT; + case S9: return TOK_IDENT; + case S10: return TOK_EOF; + case S11: return TOK_EOF; + case S12: return TOK_EOF; + case S13: return TOK_EOF; case S14: return TOK_IDENT; case S15: return TOK_IDENT; - case S16: return TOK_START; + case S16: return TOK_IDENT; case S17: return TOK_IDENT; - case S18: return TOK_IDENT; - case S19: return TOK_END; - case S20: return TOK_TO; + case S18: return TOK_START; + case S19: return TOK_IDENT; + case S20: return TOK_IDENT; + case S21: return TOK_END; + case S22: return TOK_TO; default: errno = EINVAL; return TOK_ERROR; } } @@ -1267,10 +1464,14 @@ const char * lx_name(enum lx_token t) { switch (t) { - case TOK_COMMA: return "COMMA"; case TOK_SEP: return "SEP"; case TOK_ANY: return "ANY"; case TOK_TO: return "TO"; + case TOK_ENDID: return "ENDID"; + case TOK_COMMA: return "COMMA"; + case TOK_CLOSEENDIDS: return "CLOSEENDIDS"; + case TOK_OPENENDIDS: return "OPENENDIDS"; + case TOK_EQUALS: return "EQUALS"; case TOK_IDENT: return "IDENT"; case TOK_END: return "END"; case TOK_START: return "START"; @@ -1293,10 +1494,14 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) if (z == z0) { switch (t) { - case TOK_COMMA: return ""; case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1310,10 +1515,14 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) } else if (z == z1) { switch (t) { - case TOK_COMMA: return ""; case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1327,10 +1536,14 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) } else if (z == z2) { switch (t) { - case TOK_COMMA: return ""; case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1344,10 +1557,56 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) } else if (z == z3) { switch (t) { + case TOK_SEP: return ""; + case TOK_ANY: return ""; + case TOK_TO: return ""; + case TOK_ENDID: return ""; case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; + case TOK_IDENT: return ""; + case TOK_END: return ""; + case TOK_START: return ""; + case TOK_CHAR: return ""; + case TOK_HEX: return ""; + case TOK_OCT: return ""; + case TOK_ESC: return ""; + case TOK_LABEL: return ""; + default: goto error; + } + } else + if (z == z4) { + switch (t) { case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; + case TOK_IDENT: return ""; + case TOK_END: return ""; + case TOK_START: return ""; + case TOK_CHAR: return ""; + case TOK_HEX: return ""; + case TOK_OCT: return ""; + case TOK_ESC: return ""; + case TOK_LABEL: return ""; + default: goto error; + } + } else + if (z == z5) { + switch (t) { + case TOK_SEP: return ""; + case TOK_ANY: return ""; + case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1376,7 +1635,7 @@ lx_init(struct lx *lx) *lx = lx_default; lx->c = EOF; - lx->z = z3; + lx->z = z5; lx->end.byte = 0; lx->end.line = 1; diff --git a/src/libfsm/lexer.h b/src/libfsm/lexer.h index 7848879fc..68e55b612 100644 --- a/src/libfsm/lexer.h +++ b/src/libfsm/lexer.h @@ -4,10 +4,14 @@ #define LX_H enum lx_token { - TOK_COMMA, TOK_SEP, TOK_ANY, TOK_TO, + TOK_ENDID, + TOK_COMMA, + TOK_CLOSEENDIDS, + TOK_OPENENDIDS, + TOK_EQUALS, TOK_IDENT, TOK_END, TOK_START, diff --git a/src/libfsm/lexer.lx b/src/libfsm/lexer.lx index 7aa8cf5bf..0288e1877 100644 --- a/src/libfsm/lexer.lx +++ b/src/libfsm/lexer.lx @@ -31,6 +31,17 @@ /[a-z0-9_]+/i -> $ident; +'=' -> $equals; +'[' -> $openendids .. ']' -> $closeendids { + /[\r\n\t ]+/; + "#" .. "\n"; + + ',' -> $comma; + + # using a lexical zone because $endid overlaps $ident + /[0-9]+/ -> $endid; +} + '->' -> $to; '?' -> $any; ';' -> $sep; diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index 80ace7fdf..ffae3402a 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -22,6 +22,7 @@ char -> char; string -> string; state -> state; + endid -> endid; %header% @{ @@ -41,8 +42,9 @@ #include "lexer.h" #include "parser.h" - typedef char * string; - typedef fsm_state_t state; + typedef char * string; + typedef fsm_state_t state; + typedef fsm_end_id_t endid; struct act_statelist { char *id; @@ -194,12 +196,14 @@ if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "octal escape %s out of range: expected \\0..\\%o inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\0..\\%o inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } @@ -221,12 +225,14 @@ if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "hex escape %s out of range: expected \\x0..\\x%x inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\x0..\\x%x inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } @@ -245,6 +251,31 @@ @s = xstrdup(lex_state->buf.a); @}; + ENDID: () -> (id :endid) = @{ + unsigned long u; + char *e; + + errno = 0; + + u = strtoul(lex_state->buf.a, &e, 16); + + if ((u == ULONG_MAX && errno == ERANGE) || u > FSM_END_ID_MAX) { + err(lex_state, "end id %s out of range: expected 0..%u inclusive", + lex_state->buf.a, FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); + } + + if ((u == ULONG_MAX && errno != 0) || *e != '\0') { + err(lex_state, "%s: %s: expected 0..%u inclusive", + lex_state->buf.a, strerror(errno), FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); + } + + @id = (fsm_end_id_t) u; + @}; + %actions% : (n :string) -> (s :state) = @{ @@ -308,6 +339,12 @@ fsm_setend(fsm, @s, 1); @}; + : (s :state, id :endid) -> () = @{ + if (!fsm_setendidstate(fsm, @s, @id)) { + @!; + } + @}; + : (s :string) -> () = @{ free(@s); @}; diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 998e9ee75..40e692db4 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 151 "src/libfsm/parser.act" +#line 153 "src/libfsm/parser.act" #include @@ -28,8 +28,9 @@ #include "lexer.h" #include "parser.h" - typedef char * string; - typedef fsm_state_t state; + typedef char * string; + typedef fsm_state_t state; + typedef fsm_end_id_t endid; struct act_statelist { char *id; @@ -135,7 +136,7 @@ htab->longest_chain_length = max_chain_length; } -#line 139 "src/libfsm/parser.c" +#line 140 "src/libfsm/parser.c" #ifndef ERROR_TERMINAL @@ -149,10 +150,13 @@ static void p_items(fsm, lex_state, act_state); static void p_xend_C_Cend_Hstates(fsm, lex_state, act_state); static void p_xstart(fsm, lex_state, act_state); static void p_xend(fsm, lex_state, act_state); -static void p_xend_C_Cend_Hstate(fsm, lex_state, act_state); -static void p_61(fsm, lex_state, act_state); -static void p_63(fsm, lex_state, act_state, string *); +static void p_xend_C_Cend_Hstate(fsm, lex_state, act_state, state *); extern void p_fsm(fsm, lex_state, act_state); +static void p_73(fsm, lex_state, act_state); +static void p_75(fsm, lex_state, act_state); +static void p_77(fsm, lex_state, act_state, string *); +static void p_xend_C_Cend_Hids(fsm, lex_state, act_state, state); +static void p_xend_C_Cend_Hid(fsm, lex_state, act_state, state); /* BEGINNING OF STATIC VARIABLES */ @@ -168,21 +172,21 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) return; } { - /* BEGINNING OF INLINE: 34 */ + /* BEGINNING OF INLINE: 40 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF EXTRACT: CHAR */ { -#line 240 "src/libfsm/parser.act" +#line 246 "src/libfsm/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 186 "src/libfsm/parser.c" +#line 190 "src/libfsm/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -192,7 +196,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: ESC */ { -#line 168 "src/libfsm/parser.act" +#line 170 "src/libfsm/parser.act" assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(2 == strlen(lex_state->buf.a)); @@ -210,7 +214,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) default: break; } -#line 214 "src/libfsm/parser.c" +#line 218 "src/libfsm/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -220,7 +224,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: HEX */ { -#line 233 "src/libfsm/parser.act" +#line 239 "src/libfsm/parser.act" unsigned long u; char *e; @@ -236,18 +240,20 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "hex escape %s out of range: expected \\x0..\\x%x inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\x0..\\x%x inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } ZIc = (char) (unsigned char) u; -#line 251 "src/libfsm/parser.c" +#line 257 "src/libfsm/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -257,7 +263,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: OCT */ { -#line 206 "src/libfsm/parser.act" +#line 210 "src/libfsm/parser.act" unsigned long u; char *e; @@ -273,18 +279,20 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "octal escape %s out of range: expected \\0..\\%o inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\0..\\%o inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } ZIc = (char) (unsigned char) u; -#line 288 "src/libfsm/parser.c" +#line 296 "src/libfsm/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -294,7 +302,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) goto ZL1; } } - /* END OF INLINE: 34 */ + /* END OF INLINE: 40 */ switch (CURRENT_TERMINAL) { case (TOK_LABEL): break; @@ -318,7 +326,7 @@ ZL2_items:; switch (CURRENT_TERMINAL) { case (TOK_IDENT): { - /* BEGINNING OF INLINE: 48 */ + /* BEGINNING OF INLINE: 54 */ { { string ZIa; @@ -330,12 +338,12 @@ ZL2_items:; case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 245 "src/libfsm/parser.act" +#line 251 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIa = xstrdup(lex_state->buf.a); -#line 339 "src/libfsm/parser.c" +#line 347 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -346,14 +354,14 @@ ZL2_items:; } } /* END OF INLINE: ident */ - p_63 (fsm, lex_state, act_state, &ZIa); + p_77 (fsm, lex_state, act_state, &ZIa); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } } } - /* END OF INLINE: 48 */ + /* END OF INLINE: 54 */ /* BEGINNING OF INLINE: items */ goto ZL2_items; /* END OF INLINE: items */ @@ -378,44 +386,35 @@ p_xend_C_Cend_Hstates(fsm fsm, lex_state lex_state, act_state act_state) } ZL2_xend_C_Cend_Hstates:; { - p_xend_C_Cend_Hstate (fsm, lex_state, act_state); - /* BEGINNING OF INLINE: 58 */ + state ZIs; + + p_xend_C_Cend_Hstate (fsm, lex_state, act_state, &ZIs); + /* BEGINNING OF INLINE: 71 */ { switch (CURRENT_TERMINAL) { - case (TOK_COMMA): + case (TOK_EQUALS): { - /* BEGINNING OF INLINE: 59 */ - { - { - switch (CURRENT_TERMINAL) { - case (TOK_COMMA): - break; - default: - goto ZL5; - } - ADVANCE_LEXER; - } - goto ZL4; - ZL5:; - { - /* BEGINNING OF ACTION: err-expected-comma */ - { -#line 362 "src/libfsm/parser.act" - - err_expected(lex_state, "','"); - -#line 408 "src/libfsm/parser.c" - } - /* END OF ACTION: err-expected-comma */ - } - ZL4:; + ADVANCE_LEXER; + switch (CURRENT_TERMINAL) { + case (TOK_OPENENDIDS): + break; + default: + goto ZL1; } - /* END OF INLINE: 59 */ - /* BEGINNING OF INLINE: xend::end-states */ - goto ZL2_xend_C_Cend_Hstates; - /* END OF INLINE: xend::end-states */ + ADVANCE_LEXER; + p_xend_C_Cend_Hids (fsm, lex_state, act_state, ZIs); + switch (CURRENT_TERMINAL) { + case (TOK_CLOSEENDIDS): + break; + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + default: + goto ZL1; + } + ADVANCE_LEXER; } - /*UNREACHED*/ + break; case (ERROR_TERMINAL): RESTORE_LEXER; goto ZL1; @@ -423,7 +422,28 @@ ZL2_xend_C_Cend_Hstates:; break; } } - /* END OF INLINE: 58 */ + /* END OF INLINE: 71 */ + /* BEGINNING OF INLINE: 72 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + { + p_73 (fsm, lex_state, act_state); + /* BEGINNING OF INLINE: xend::end-states */ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } else { + goto ZL2_xend_C_Cend_Hstates; + } + /* END OF INLINE: xend::end-states */ + } + /*UNREACHED*/ + default: + break; + } + } + /* END OF INLINE: 72 */ } return; ZL1:; @@ -440,7 +460,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) string ZIn; state ZIs; - /* BEGINNING OF INLINE: 50 */ + /* BEGINNING OF INLINE: 56 */ { { switch (CURRENT_TERMINAL) { @@ -456,17 +476,17 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-start */ { -#line 366 "src/libfsm/parser.act" +#line 403 "src/libfsm/parser.act" err_expected(lex_state, "'start:'"); -#line 464 "src/libfsm/parser.c" +#line 484 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-start */ } ZL2:; } - /* END OF INLINE: 50 */ + /* END OF INLINE: 56 */ /* BEGINNING OF INLINE: ident */ { { @@ -474,12 +494,12 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 245 "src/libfsm/parser.act" +#line 251 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); -#line 483 "src/libfsm/parser.c" +#line 503 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -490,14 +510,14 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) } } /* END OF INLINE: ident */ - p_61 (fsm, lex_state, act_state); + p_75 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: add-state */ { -#line 252 "src/libfsm/parser.act" +#line 283 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -550,25 +570,25 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 554 "src/libfsm/parser.c" +#line 574 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ { -#line 304 "src/libfsm/parser.act" +#line 335 "src/libfsm/parser.act" fsm_setstart(fsm, (ZIs)); -#line 563 "src/libfsm/parser.c" +#line 583 "src/libfsm/parser.c" } /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ { -#line 312 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIn)); -#line 572 "src/libfsm/parser.c" +#line 592 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -590,7 +610,7 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) switch (CURRENT_TERMINAL) { case (TOK_END): { - /* BEGINNING OF INLINE: 60 */ + /* BEGINNING OF INLINE: 74 */ { { switch (CURRENT_TERMINAL) { @@ -606,19 +626,19 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-end */ { -#line 370 "src/libfsm/parser.act" +#line 407 "src/libfsm/parser.act" err_expected(lex_state, "'end:'"); -#line 614 "src/libfsm/parser.c" +#line 634 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-end */ } ZL2:; } - /* END OF INLINE: 60 */ + /* END OF INLINE: 74 */ p_xend_C_Cend_Hstates (fsm, lex_state, act_state); - p_61 (fsm, lex_state, act_state); + p_75 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -637,14 +657,15 @@ ZL1:; } static void -p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state) +p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *ZOs) { + state ZIs; + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } { string ZIn; - state ZIs; /* BEGINNING OF INLINE: ident */ { @@ -653,12 +674,12 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 245 "src/libfsm/parser.act" +#line 251 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); -#line 662 "src/libfsm/parser.c" +#line 683 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -671,7 +692,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state) /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 252 "src/libfsm/parser.act" +#line 283 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -724,36 +745,125 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 728 "src/libfsm/parser.c" +#line 749 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-end */ { -#line 308 "src/libfsm/parser.act" +#line 339 "src/libfsm/parser.act" fsm_setend(fsm, (ZIs), 1); -#line 737 "src/libfsm/parser.c" +#line 758 "src/libfsm/parser.c" } /* END OF ACTION: mark-end */ /* BEGINNING OF ACTION: free */ { -#line 312 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIn)); -#line 746 "src/libfsm/parser.c" +#line 767 "src/libfsm/parser.c" } /* END OF ACTION: free */ } - return; + goto ZL0; ZL1:; SAVE_LEXER ((ERROR_TERMINAL)); return; +ZL0:; + *ZOs = ZIs; +} + +void +p_fsm(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + p_items (fsm, lex_state, act_state); + p_xstart (fsm, lex_state, act_state); + p_xend (fsm, lex_state, act_state); + switch (CURRENT_TERMINAL) { + case (TOK_EOF): + break; + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + default: + goto ZL1; + } + ADVANCE_LEXER; + /* BEGINNING OF ACTION: free-statelist */ + { +#line 365 "src/libfsm/parser.act" + + struct act_statelist *p; + struct act_statelist *next; + unsigned b_i; /* bucket */ + + for (b_i = 0; b_i < act_state->states.bucket_count; b_i++) { + for (p = act_state->states.buckets[b_i]; p != NULL; p = next) { + next = p->next; + assert(p->id != NULL); + free(p->id); + free(p); + } + } + +#line 816 "src/libfsm/parser.c" + } + /* END OF ACTION: free-statelist */ + } + return; +ZL1:; + { + /* BEGINNING OF ACTION: err-syntax */ + { +#line 412 "src/libfsm/parser.act" + + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + +#line 830 "src/libfsm/parser.c" + } + /* END OF ACTION: err-syntax */ + } } static void -p_61(fsm fsm, lex_state lex_state, act_state act_state) +p_73(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + break; + default: + goto ZL1; + } + ADVANCE_LEXER; + } + return; +ZL1:; + { + /* BEGINNING OF ACTION: err-expected-comma */ + { +#line 399 "src/libfsm/parser.act" + + err_expected(lex_state, "','"); + +#line 860 "src/libfsm/parser.c" + } + /* END OF ACTION: err-expected-comma */ + } +} + +static void +p_75(fsm fsm, lex_state lex_state, act_state act_state) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; @@ -772,18 +882,18 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-sep */ { -#line 354 "src/libfsm/parser.act" +#line 391 "src/libfsm/parser.act" err_expected(lex_state, "';'"); -#line 780 "src/libfsm/parser.c" +#line 890 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-sep */ } } static void -p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) +p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { switch (CURRENT_TERMINAL) { case (TOK_TO): @@ -800,12 +910,12 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 245 "src/libfsm/parser.act" +#line 251 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); -#line 809 "src/libfsm/parser.c" +#line 919 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -818,7 +928,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 252 "src/libfsm/parser.act" +#line 283 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -871,12 +981,12 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 875 "src/libfsm/parser.c" +#line 985 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ { -#line 252 "src/libfsm/parser.act" +#line 283 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIb)); @@ -929,28 +1039,28 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 933 "src/libfsm/parser.c" +#line 1043 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 312 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((*ZIa)); -#line 942 "src/libfsm/parser.c" +#line 1052 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ { -#line 312 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIb)); -#line 951 "src/libfsm/parser.c" +#line 1061 "src/libfsm/parser.c" } /* END OF ACTION: free */ - /* BEGINNING OF INLINE: 42 */ + /* BEGINNING OF INLINE: 48 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): @@ -958,14 +1068,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; /* BEGINNING OF ACTION: add-edge-any */ { -#line 338 "src/libfsm/parser.act" +#line 375 "src/libfsm/parser.act" if (!fsm_addedge_any(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_any"); exit(EXIT_FAILURE); } -#line 969 "src/libfsm/parser.c" +#line 1079 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-any */ } @@ -981,14 +1091,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) } /* BEGINNING OF ACTION: add-edge-literal */ { -#line 331 "src/libfsm/parser.act" +#line 368 "src/libfsm/parser.act" if (!fsm_addedge_literal(fsm, (ZIx), (ZIy), (ZIc))) { perror("fsm_addedge_literal"); exit(EXIT_FAILURE); } -#line 992 "src/libfsm/parser.c" +#line 1102 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-literal */ } @@ -997,14 +1107,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: add-edge-epsilon */ { -#line 345 "src/libfsm/parser.act" +#line 382 "src/libfsm/parser.act" if (!fsm_addedge_epsilon(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_epsilon"); exit(EXIT_FAILURE); } -#line 1008 "src/libfsm/parser.c" +#line 1118 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-epsilon */ } @@ -1015,18 +1125,18 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: err-expected-trans */ { -#line 358 "src/libfsm/parser.act" +#line 395 "src/libfsm/parser.act" err_expected(lex_state, "transition"); -#line 1023 "src/libfsm/parser.c" +#line 1133 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-trans */ } ZL3:; } - /* END OF INLINE: 42 */ - p_61 (fsm, lex_state, act_state); + /* END OF INLINE: 48 */ + p_75 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1035,11 +1145,11 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) break; case (TOK_SEP): { - state ZI45; + state ZI51; /* BEGINNING OF ACTION: add-state */ { -#line 252 "src/libfsm/parser.act" +#line 283 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -1060,7 +1170,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) chain_length++; if (0 == strcmp(p->id, (*ZIa))) { - (ZI45) = p->state; + (ZI51) = p->state; break; } } @@ -1081,30 +1191,30 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* XXX: don't exit in library code */ new->id = xstrdup((*ZIa)); - if (!fsm_addstate(fsm, &(ZI45))) { + if (!fsm_addstate(fsm, &(ZI51))) { perror("fsm_addstate"); exit(EXIT_FAILURE); } - new->state = (ZI45); + new->state = (ZI51); new->next = act_state->states.buckets[bucket_id]; act_state->states.buckets[bucket_id] = new; } -#line 1096 "src/libfsm/parser.c" +#line 1206 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 312 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((*ZIa)); -#line 1105 "src/libfsm/parser.c" +#line 1215 "src/libfsm/parser.c" } /* END OF ACTION: free */ - p_61 (fsm, lex_state, act_state); + p_75 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1122,66 +1232,113 @@ ZL1:; return; } -void -p_fsm(fsm fsm, lex_state lex_state, act_state act_state) +static void +p_xend_C_Cend_Hids(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } +ZL2_xend_C_Cend_Hids:; { - p_items (fsm, lex_state, act_state); - p_xstart (fsm, lex_state, act_state); - p_xend (fsm, lex_state, act_state); + p_xend_C_Cend_Hid (fsm, lex_state, act_state, ZIs); + /* BEGINNING OF INLINE: 65 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + { + p_73 (fsm, lex_state, act_state); + /* BEGINNING OF INLINE: xend::end-ids */ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } else { + goto ZL2_xend_C_Cend_Hids; + } + /* END OF INLINE: xend::end-ids */ + } + /*UNREACHED*/ + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + default: + break; + } + } + /* END OF INLINE: 65 */ + } + return; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +} + +static void +p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + endid ZIid; + switch (CURRENT_TERMINAL) { - case (TOK_EOF): + case (TOK_ENDID): + /* BEGINNING OF EXTRACT: ENDID */ + { +#line 276 "src/libfsm/parser.act" + + unsigned long u; + char *e; + + errno = 0; + + u = strtoul(lex_state->buf.a, &e, 16); + + if ((u == ULONG_MAX && errno == ERANGE) || u > FSM_END_ID_MAX) { + err(lex_state, "end id %s out of range: expected 0..%u inclusive", + lex_state->buf.a, FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); + } + + if ((u == ULONG_MAX && errno != 0) || *e != '\0') { + err(lex_state, "%s: %s: expected 0..%u inclusive", + lex_state->buf.a, strerror(errno), FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); + } + + ZIid = (fsm_end_id_t) u; + +#line 1314 "src/libfsm/parser.c" + } + /* END OF EXTRACT: ENDID */ break; - case (ERROR_TERMINAL): - RESTORE_LEXER; - goto ZL1; default: goto ZL1; } ADVANCE_LEXER; - /* BEGINNING OF ACTION: free-statelist */ + /* BEGINNING OF ACTION: mark-end-id */ { -#line 328 "src/libfsm/parser.act" +#line 343 "src/libfsm/parser.act" - struct act_statelist *p; - struct act_statelist *next; - unsigned b_i; /* bucket */ - - for (b_i = 0; b_i < act_state->states.bucket_count; b_i++) { - for (p = act_state->states.buckets[b_i]; p != NULL; p = next) { - next = p->next; - assert(p->id != NULL); - free(p->id); - free(p); - } + if (!fsm_setendidstate(fsm, (ZIs), (ZIid))) { + goto ZL1; } -#line 1163 "src/libfsm/parser.c" +#line 1330 "src/libfsm/parser.c" } - /* END OF ACTION: free-statelist */ + /* END OF ACTION: mark-end-id */ } return; ZL1:; - { - /* BEGINNING OF ACTION: err-syntax */ - { -#line 375 "src/libfsm/parser.act" - - err(lex_state, "Syntax error"); - exit(EXIT_FAILURE); - -#line 1177 "src/libfsm/parser.c" - } - /* END OF ACTION: err-syntax */ - } + SAVE_LEXER ((ERROR_TERMINAL)); + return; } /* BEGINNING OF TRAILER */ -#line 431 "src/libfsm/parser.act" +#line 468 "src/libfsm/parser.act" struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { @@ -1236,6 +1393,6 @@ ZL1:; return new; } -#line 1240 "src/libfsm/parser.c" +#line 1397 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index 70155c504..b4aaba2b8 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 160 "src/libfsm/parser.act" +#line 162 "src/libfsm/parser.act" typedef struct lex_state * lex_state; @@ -26,7 +26,7 @@ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 432 "src/libfsm/parser.act" +#line 469 "src/libfsm/parser.act" #line 32 "src/libfsm/parser.h" diff --git a/src/libfsm/parser.sid b/src/libfsm/parser.sid index 32a370312..f8d8e78cd 100644 --- a/src/libfsm/parser.sid +++ b/src/libfsm/parser.sid @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -9,10 +9,12 @@ char; string; state; + endid; %terminals% IDENT: () -> (:string); + ENDID: () -> (:endid); ESC: () -> (:char); OCT: () -> (:char); @@ -23,6 +25,10 @@ START; END; + OPENENDIDS; + CLOSEENDIDS; + + EQUALS; TO; ANY; SEP; @@ -36,6 +42,7 @@ : (:string) -> (:state); : (:state) -> (); : (:state) -> (); + : (:state, :endid) -> (); : (:string) -> (); ; @@ -155,7 +162,29 @@ xend [ - end-state = { + end-id: (s :state) -> () = { + id = ENDID; + + (s, id); + }; + + end-ids: (s :state) -> () = { + end-id(s); + + { + { + COMMA; + ## + ; + }; + + end-ids(s); + || + $; + }; + }; + + end-state: () -> (s :state) = { n = ident; s = (n); @@ -165,7 +194,17 @@ }; end-states = { - end-state; + s = end-state; + + { + EQUALS; + + OPENENDIDS; + end-ids(s); + CLOSEENDIDS; + || + $; + }; { { diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index ea41ea757..ae3163abd 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -24,6 +24,19 @@ #include #include +/* TODO: centralise */ +static int +comp_end_id(const void *a, const void *b) +{ + assert(a != NULL); + assert(b != NULL); + + if (* (fsm_end_id_t *) a < * (fsm_end_id_t *) b) { return -1; } + if (* (fsm_end_id_t *) a > * (fsm_end_id_t *) b) { return +1; } + + return 0; +} + /* TODO: centralise */ static int findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) @@ -278,11 +291,58 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) fprintf(f, "end: "); for (s = 0; s < fsm->statecount; s++) { - if (fsm_isend(fsm, s)) { - end--; + size_t count; + + if (!fsm_isend(fsm, s)) { + continue; + } + + end--; + + fprintf(f, "%u", s); + + count = fsm_getendidcount(fsm, s); + if (count > 0) { + enum fsm_getendids_res res; + fsm_end_id_t *ids; + size_t written; + + ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; + } + + res = fsm_getendids(fsm, s, count, ids, &written); + switch (res) { + case FSM_GETENDIDS_NOT_FOUND: + case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: + assert(!"unreached"); + abort(); + + case FSM_GETENDIDS_FOUND: + break; + } + + assert(written == count); + + qsort(ids, count, sizeof *ids, comp_end_id); + + fprintf(f, " = ["); - fprintf(f, "%u%s", s, end > 0 ? ", " : ";\n"); + for (size_t id = 0; id < count; id++) { + fprintf(f, "%zu", id); + + if (id + 1 < count) { + fprintf(f, ", "); + } + } + + fprintf(f, "]"); + + f_free(fsm->opt->alloc, ids); } + + fprintf(f, "%s", end > 0 ? ", " : ";\n"); } done: From 9ce1fb0210784adfea93a3a99e91737e10ee94e4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 10 Jun 2024 12:45:35 -0700 Subject: [PATCH 129/373] Missing error handling for the initial token. --- src/libfsm/parser.act | 14 +++++++++++++- src/libfsm/parser.c | 18 +++++++++++++++--- src/libfsm/parser.h | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index ffae3402a..30caa03bd 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -456,7 +456,19 @@ return NULL; } - ADVANCE_LEXER; /* XXX: what if the first token is unrecognised? */ + ADVANCE_LEXER; + switch (CURRENT_TERMINAL) { + case TOK_UNKNOWN: + /* fallthrough */ + case TOK_ERROR: + /* XXX: don't exit in library code */ + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + + default: + break; + } + p_fsm(new, lex_state, act_state); free(act_state_s.states.buckets); diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 40e692db4..73d525d25 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -1338,7 +1338,7 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 468 "src/libfsm/parser.act" +#line 480 "src/libfsm/parser.act" struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { @@ -1384,7 +1384,19 @@ ZL1:; return NULL; } - ADVANCE_LEXER; /* XXX: what if the first token is unrecognised? */ + ADVANCE_LEXER; + switch (CURRENT_TERMINAL) { + case TOK_UNKNOWN: + /* fallthrough */ + case TOK_ERROR: + /* XXX: don't exit in library code */ + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + + default: + break; + } + p_fsm(new, lex_state, act_state); free(act_state_s.states.buckets); @@ -1393,6 +1405,6 @@ ZL1:; return new; } -#line 1397 "src/libfsm/parser.c" +#line 1409 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index b4aaba2b8..be8f288ec 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -26,7 +26,7 @@ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 469 "src/libfsm/parser.act" +#line 481 "src/libfsm/parser.act" #line 32 "src/libfsm/parser.h" From 0447463aabb9f8c7e90f33df40fef0afd47ca19a Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 10 Jun 2024 22:49:29 -0700 Subject: [PATCH 130/373] Allow an empty list of end-ids. --- src/libfsm/parser.c | 111 +++++++++++++++++++++++------------------- src/libfsm/parser.sid | 2 +- 2 files changed, 63 insertions(+), 50 deletions(-) diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 73d525d25..0ae7ee763 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -152,9 +152,9 @@ static void p_xstart(fsm, lex_state, act_state); static void p_xend(fsm, lex_state, act_state); static void p_xend_C_Cend_Hstate(fsm, lex_state, act_state, state *); extern void p_fsm(fsm, lex_state, act_state); -static void p_73(fsm, lex_state, act_state); -static void p_75(fsm, lex_state, act_state); -static void p_77(fsm, lex_state, act_state, string *); +static void p_74(fsm, lex_state, act_state); +static void p_76(fsm, lex_state, act_state); +static void p_78(fsm, lex_state, act_state, string *); static void p_xend_C_Cend_Hids(fsm, lex_state, act_state, state); static void p_xend_C_Cend_Hid(fsm, lex_state, act_state, state); @@ -354,7 +354,7 @@ ZL2_items:; } } /* END OF INLINE: ident */ - p_77 (fsm, lex_state, act_state, &ZIa); + p_78 (fsm, lex_state, act_state, &ZIa); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -402,13 +402,26 @@ ZL2_xend_C_Cend_Hstates:; goto ZL1; } ADVANCE_LEXER; - p_xend_C_Cend_Hids (fsm, lex_state, act_state, ZIs); + /* BEGINNING OF INLINE: 72 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_ENDID): + { + p_xend_C_Cend_Hids (fsm, lex_state, act_state, ZIs); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + default: + break; + } + } + /* END OF INLINE: 72 */ switch (CURRENT_TERMINAL) { case (TOK_CLOSEENDIDS): break; - case (ERROR_TERMINAL): - RESTORE_LEXER; - goto ZL1; default: goto ZL1; } @@ -423,12 +436,12 @@ ZL2_xend_C_Cend_Hstates:; } } /* END OF INLINE: 71 */ - /* BEGINNING OF INLINE: 72 */ + /* BEGINNING OF INLINE: 73 */ { switch (CURRENT_TERMINAL) { case (TOK_COMMA): { - p_73 (fsm, lex_state, act_state); + p_74 (fsm, lex_state, act_state); /* BEGINNING OF INLINE: xend::end-states */ if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -443,7 +456,7 @@ ZL2_xend_C_Cend_Hstates:; break; } } - /* END OF INLINE: 72 */ + /* END OF INLINE: 73 */ } return; ZL1:; @@ -480,7 +493,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) err_expected(lex_state, "'start:'"); -#line 484 "src/libfsm/parser.c" +#line 497 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-start */ } @@ -499,7 +512,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); -#line 503 "src/libfsm/parser.c" +#line 516 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -510,7 +523,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) } } /* END OF INLINE: ident */ - p_75 (fsm, lex_state, act_state); + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -570,7 +583,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 574 "src/libfsm/parser.c" +#line 587 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ @@ -579,7 +592,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) fsm_setstart(fsm, (ZIs)); -#line 583 "src/libfsm/parser.c" +#line 596 "src/libfsm/parser.c" } /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ @@ -588,7 +601,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) free((ZIn)); -#line 592 "src/libfsm/parser.c" +#line 605 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -610,7 +623,7 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) switch (CURRENT_TERMINAL) { case (TOK_END): { - /* BEGINNING OF INLINE: 74 */ + /* BEGINNING OF INLINE: 75 */ { { switch (CURRENT_TERMINAL) { @@ -630,15 +643,15 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) err_expected(lex_state, "'end:'"); -#line 634 "src/libfsm/parser.c" +#line 647 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-end */ } ZL2:; } - /* END OF INLINE: 74 */ + /* END OF INLINE: 75 */ p_xend_C_Cend_Hstates (fsm, lex_state, act_state); - p_75 (fsm, lex_state, act_state); + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -679,7 +692,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); -#line 683 "src/libfsm/parser.c" +#line 696 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -745,7 +758,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z act_state->states.buckets[bucket_id] = new; } -#line 749 "src/libfsm/parser.c" +#line 762 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-end */ @@ -754,7 +767,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z fsm_setend(fsm, (ZIs), 1); -#line 758 "src/libfsm/parser.c" +#line 771 "src/libfsm/parser.c" } /* END OF ACTION: mark-end */ /* BEGINNING OF ACTION: free */ @@ -763,7 +776,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z free((ZIn)); -#line 767 "src/libfsm/parser.c" +#line 780 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -812,7 +825,7 @@ p_fsm(fsm fsm, lex_state lex_state, act_state act_state) } } -#line 816 "src/libfsm/parser.c" +#line 829 "src/libfsm/parser.c" } /* END OF ACTION: free-statelist */ } @@ -826,14 +839,14 @@ ZL1:; err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 830 "src/libfsm/parser.c" +#line 843 "src/libfsm/parser.c" } /* END OF ACTION: err-syntax */ } } static void -p_73(fsm fsm, lex_state lex_state, act_state act_state) +p_74(fsm fsm, lex_state lex_state, act_state act_state) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; @@ -856,14 +869,14 @@ ZL1:; err_expected(lex_state, "','"); -#line 860 "src/libfsm/parser.c" +#line 873 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-comma */ } } static void -p_75(fsm fsm, lex_state lex_state, act_state act_state) +p_76(fsm fsm, lex_state lex_state, act_state act_state) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; @@ -886,14 +899,14 @@ ZL1:; err_expected(lex_state, "';'"); -#line 890 "src/libfsm/parser.c" +#line 903 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-sep */ } } static void -p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) +p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { switch (CURRENT_TERMINAL) { case (TOK_TO): @@ -915,7 +928,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); -#line 919 "src/libfsm/parser.c" +#line 932 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -981,7 +994,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 985 "src/libfsm/parser.c" +#line 998 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ @@ -1039,7 +1052,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 1043 "src/libfsm/parser.c" +#line 1056 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ @@ -1048,7 +1061,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) free((*ZIa)); -#line 1052 "src/libfsm/parser.c" +#line 1065 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ @@ -1057,7 +1070,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) free((ZIb)); -#line 1061 "src/libfsm/parser.c" +#line 1074 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF INLINE: 48 */ @@ -1075,7 +1088,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } -#line 1079 "src/libfsm/parser.c" +#line 1092 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-any */ } @@ -1098,7 +1111,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } -#line 1102 "src/libfsm/parser.c" +#line 1115 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-literal */ } @@ -1114,7 +1127,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } -#line 1118 "src/libfsm/parser.c" +#line 1131 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-epsilon */ } @@ -1129,14 +1142,14 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) err_expected(lex_state, "transition"); -#line 1133 "src/libfsm/parser.c" +#line 1146 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-trans */ } ZL3:; } /* END OF INLINE: 48 */ - p_75 (fsm, lex_state, act_state); + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1202,7 +1215,7 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 1206 "src/libfsm/parser.c" +#line 1219 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ @@ -1211,10 +1224,10 @@ p_77(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) free((*ZIa)); -#line 1215 "src/libfsm/parser.c" +#line 1228 "src/libfsm/parser.c" } /* END OF ACTION: free */ - p_75 (fsm, lex_state, act_state); + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1246,7 +1259,7 @@ ZL2_xend_C_Cend_Hids:; switch (CURRENT_TERMINAL) { case (TOK_COMMA): { - p_73 (fsm, lex_state, act_state); + p_74 (fsm, lex_state, act_state); /* BEGINNING OF INLINE: xend::end-ids */ if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1310,7 +1323,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) ZIid = (fsm_end_id_t) u; -#line 1314 "src/libfsm/parser.c" +#line 1327 "src/libfsm/parser.c" } /* END OF EXTRACT: ENDID */ break; @@ -1326,7 +1339,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) goto ZL1; } -#line 1330 "src/libfsm/parser.c" +#line 1343 "src/libfsm/parser.c" } /* END OF ACTION: mark-end-id */ } @@ -1405,6 +1418,6 @@ ZL1:; return new; } -#line 1409 "src/libfsm/parser.c" +#line 1422 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.sid b/src/libfsm/parser.sid index f8d8e78cd..7d22c7420 100644 --- a/src/libfsm/parser.sid +++ b/src/libfsm/parser.sid @@ -200,7 +200,7 @@ EQUALS; OPENENDIDS; - end-ids(s); + { end-ids(s); || $; }; CLOSEENDIDS; || $; From 1d30a1250ee44fa7dc802065c6374e42c415932b Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 10 Jun 2024 23:40:23 -0700 Subject: [PATCH 131/373] Bugfix for argv handling for fsm_exec cli usage. Two things here: Firstly I've reworked all this stuff such that we handle filenames, and then any remaining arguments are used to match with fsm_exec. This is what I'd originally intended, but I think it got lost at some point, probably when introducing operators with arity 2. Secondly matching text with fsm_exec now runs regarless of the operation, not just on OP_IDENTITY. I see no reason to limit that. --- src/fsm/main.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/fsm/main.c b/src/fsm/main.c index d6cd1e83c..a255ae8ee 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -477,17 +477,14 @@ main(int argc, char *argv[]) struct fsm *q; if ((op & OP_ARITY) == 1) { - if (argc > 1) { - usage(); - exit(EXIT_FAILURE); - } + /* argc < 1 is okay */ q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), &opt); if (q == NULL) { exit(EXIT_FAILURE); } } else { - if (argc != 2) { + if (argc < 2) { usage(); exit(EXIT_FAILURE); } @@ -610,6 +607,17 @@ main(int argc, char *argv[]) printf("=> total %g ms (avg %g ms)\n", elapsed, elapsed / iterations); } + /* we're done consuming filenames, remaining argv is text to match */ + if ((op & OP_ARITY) == 1) { + if (argc > 0) { + argc -= 1; + argv += 1; + } + } else { + argc -= 2; + argv += 2; + } + /* henceforth, r is $?-convention (0 for success) */ if (fsm == NULL) { @@ -661,14 +669,15 @@ main(int argc, char *argv[]) } } - /* TODO: optional -- to delimit texts as opposed to .fsm filenames */ - if (op == OP_IDENTITY && argc > 0) { + /* match text */ + if (argc > 0) { int i; /* TODO: option to print input texts which match. like grep(1) does. * This is not the same as printing patterns which match (by associating * a pattern to the end state), like lx(1) does */ + /* TODO: optional -- to delimit texts as opposed to .fsm filenames */ for (i = 0; i < argc; i++) { fsm_state_t state; int e; @@ -694,7 +703,7 @@ main(int argc, char *argv[]) continue; } - /* TODO: option to print state number? */ + /* TODO: option to print matching end-ids */ } } From ebdbbbd6b2fb5a9436f0e9538c1940a053223d3b Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 00:24:51 -0700 Subject: [PATCH 132/373] Tests for .fsm syntax. The file format is complex enough that I want to cover these explicitly now, rather than relying on coverage through other tests. --- Makefile | 1 + tests/fsm/Makefile | 22 ++++++++++++++++++++++ tests/fsm/in0.fsm | 0 tests/fsm/in1.fsm | 1 + tests/fsm/in10.fsm | 10 ++++++++++ tests/fsm/in11.fsm | 9 +++++++++ tests/fsm/in12.fsm | 9 +++++++++ tests/fsm/in13.fsm | 9 +++++++++ tests/fsm/in14.fsm | 12 ++++++++++++ tests/fsm/in15.fsm | 11 +++++++++++ tests/fsm/in16.fsm | 11 +++++++++++ tests/fsm/in17.fsm | 11 +++++++++++ tests/fsm/in18.fsm | 11 +++++++++++ tests/fsm/in19.fsm | 11 +++++++++++ tests/fsm/in2.fsm | 13 +++++++++++++ tests/fsm/in20.fsm | 11 +++++++++++ tests/fsm/in21.fsm | 11 +++++++++++ tests/fsm/in22.fsm | 11 +++++++++++ tests/fsm/in3.fsm | 9 +++++++++ tests/fsm/in4.fsm | 9 +++++++++ tests/fsm/in5.fsm | 9 +++++++++ tests/fsm/in6.fsm | 8 ++++++++ tests/fsm/in7.fsm | 9 +++++++++ tests/fsm/in8.fsm | 12 ++++++++++++ tests/fsm/in9.fsm | 9 +++++++++ tests/fsm/out0.fsm | 0 tests/fsm/out1.fsm | 0 tests/fsm/out10.err | 1 + tests/fsm/out11.err | 1 + tests/fsm/out12.err | 1 + tests/fsm/out13.err | 1 + tests/fsm/out14.err | 1 + tests/fsm/out15.err | 1 + tests/fsm/out16.fsm | 8 ++++++++ tests/fsm/out17.err | 1 + tests/fsm/out18.err | 1 + tests/fsm/out19.fsm | 8 ++++++++ tests/fsm/out2.fsm | 7 +++++++ tests/fsm/out20.fsm | 8 ++++++++ tests/fsm/out21.err | 1 + tests/fsm/out22.fsm | 9 +++++++++ tests/fsm/out3.err | 1 + tests/fsm/out4.err | 1 + tests/fsm/out5.err | 1 + tests/fsm/out6.fsm | 7 +++++++ tests/fsm/out7.fsm | 7 +++++++ tests/fsm/out8.fsm | 10 ++++++++++ tests/fsm/out9.err | 1 + 48 files changed, 316 insertions(+) create mode 100755 tests/fsm/Makefile create mode 100644 tests/fsm/in0.fsm create mode 100644 tests/fsm/in1.fsm create mode 100644 tests/fsm/in10.fsm create mode 100644 tests/fsm/in11.fsm create mode 100644 tests/fsm/in12.fsm create mode 100644 tests/fsm/in13.fsm create mode 100644 tests/fsm/in14.fsm create mode 100644 tests/fsm/in15.fsm create mode 100644 tests/fsm/in16.fsm create mode 100644 tests/fsm/in17.fsm create mode 100644 tests/fsm/in18.fsm create mode 100644 tests/fsm/in19.fsm create mode 100644 tests/fsm/in2.fsm create mode 100644 tests/fsm/in20.fsm create mode 100644 tests/fsm/in21.fsm create mode 100644 tests/fsm/in22.fsm create mode 100644 tests/fsm/in3.fsm create mode 100644 tests/fsm/in4.fsm create mode 100644 tests/fsm/in5.fsm create mode 100644 tests/fsm/in6.fsm create mode 100644 tests/fsm/in7.fsm create mode 100644 tests/fsm/in8.fsm create mode 100644 tests/fsm/in9.fsm create mode 100644 tests/fsm/out0.fsm create mode 100644 tests/fsm/out1.fsm create mode 100644 tests/fsm/out10.err create mode 100644 tests/fsm/out11.err create mode 100644 tests/fsm/out12.err create mode 100644 tests/fsm/out13.err create mode 100644 tests/fsm/out14.err create mode 100644 tests/fsm/out15.err create mode 100644 tests/fsm/out16.fsm create mode 100644 tests/fsm/out17.err create mode 100644 tests/fsm/out18.err create mode 100644 tests/fsm/out19.fsm create mode 100644 tests/fsm/out2.fsm create mode 100644 tests/fsm/out20.fsm create mode 100644 tests/fsm/out21.err create mode 100644 tests/fsm/out22.fsm create mode 100644 tests/fsm/out3.err create mode 100644 tests/fsm/out4.err create mode 100644 tests/fsm/out5.err create mode 100644 tests/fsm/out6.fsm create mode 100644 tests/fsm/out7.fsm create mode 100644 tests/fsm/out8.fsm create mode 100644 tests/fsm/out9.err diff --git a/Makefile b/Makefile index 499239fd9..3613bc941 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,7 @@ SUBDIR += tests/subtract SUBDIR += tests/determinise SUBDIR += tests/endids SUBDIR += tests/epsilons +SUBDIR += tests/fsm SUBDIR += tests/glob SUBDIR += tests/like SUBDIR += tests/literal diff --git a/tests/fsm/Makefile b/tests/fsm/Makefile new file mode 100755 index 000000000..6ac0a2646 --- /dev/null +++ b/tests/fsm/Makefile @@ -0,0 +1,22 @@ +.include "../../share/mk/top.mk" + +TEST.tests/fsm != ls -1 tests/fsm/out*.fsm +TEST_SRCDIR.tests/fsm = tests/fsm +TEST_OUTDIR.tests/fsm = ${BUILD}/tests/fsm + +FSM=${BUILD}/bin/fsm + +.for n in ${TEST.tests/fsm:T:R:C/^out//} + +${TEST_OUTDIR.tests/fsm}/got${n}.fsm: ${TEST_SRCDIR.tests/fsm}/in${n}.fsm + ${FSM} -p ${.ALLSRC:M*.fsm} \ + > $@ + +${TEST_OUTDIR.tests/fsm}/res${n}: \ + ${TEST_SRCDIR.tests/fsm}/out${n}.fsm \ + ${TEST_OUTDIR.tests/fsm}/got${n}.fsm + +FSMTEST_RESULT += ${TEST_OUTDIR.tests/fsm}/res${n} + +.endfor + diff --git a/tests/fsm/in0.fsm b/tests/fsm/in0.fsm new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fsm/in1.fsm b/tests/fsm/in1.fsm new file mode 100644 index 000000000..88d443fec --- /dev/null +++ b/tests/fsm/in1.fsm @@ -0,0 +1 @@ +# just a comment diff --git a/tests/fsm/in10.fsm b/tests/fsm/in10.fsm new file mode 100644 index 000000000..8b044d3bb --- /dev/null +++ b/tests/fsm/in10.fsm @@ -0,0 +1,10 @@ +# +# Copyright 2008-2017 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing seperator +start: a +end: b + diff --git a/tests/fsm/in11.fsm b/tests/fsm/in11.fsm new file mode 100644 index 000000000..bc058b5e2 --- /dev/null +++ b/tests/fsm/in11.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing comma +end: 1 2 3; + diff --git a/tests/fsm/in12.fsm b/tests/fsm/in12.fsm new file mode 100644 index 000000000..973f98c86 --- /dev/null +++ b/tests/fsm/in12.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +start: b; + diff --git a/tests/fsm/in13.fsm b/tests/fsm/in13.fsm new file mode 100644 index 000000000..0ce5b760a --- /dev/null +++ b/tests/fsm/in13.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2017 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +end: a; +end: a; + diff --git a/tests/fsm/in14.fsm b/tests/fsm/in14.fsm new file mode 100644 index 000000000..82bd7b25c --- /dev/null +++ b/tests/fsm/in14.fsm @@ -0,0 +1,12 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing semicolon +0 1; + +start: 0; +end: 1; + diff --git a/tests/fsm/in15.fsm b/tests/fsm/in15.fsm new file mode 100644 index 000000000..f0622bf0f --- /dev/null +++ b/tests/fsm/in15.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# missing ] +end: a = [ ; + diff --git a/tests/fsm/in16.fsm b/tests/fsm/in16.fsm new file mode 100644 index 000000000..0e7fcec60 --- /dev/null +++ b/tests/fsm/in16.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# an empty list is okay +end: a = [ ]; + diff --git a/tests/fsm/in17.fsm b/tests/fsm/in17.fsm new file mode 100644 index 000000000..42df382d5 --- /dev/null +++ b/tests/fsm/in17.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# end-id must be numeric +end: a = [ abc ]; + diff --git a/tests/fsm/in18.fsm b/tests/fsm/in18.fsm new file mode 100644 index 000000000..2cccfc2e5 --- /dev/null +++ b/tests/fsm/in18.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# missing end-id +end: a = [ , ]; + diff --git a/tests/fsm/in19.fsm b/tests/fsm/in19.fsm new file mode 100644 index 000000000..f0875ad1f --- /dev/null +++ b/tests/fsm/in19.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# out of order is okay +end: a = [ 999 ]; + diff --git a/tests/fsm/in2.fsm b/tests/fsm/in2.fsm new file mode 100644 index 000000000..345ab6cc7 --- /dev/null +++ b/tests/fsm/in2.fsm @@ -0,0 +1,13 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# states are named by identifiers +abc; +0; +_x; +_; +abc123; + diff --git a/tests/fsm/in20.fsm b/tests/fsm/in20.fsm new file mode 100644 index 000000000..d025dc47e --- /dev/null +++ b/tests/fsm/in20.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# out of order is okay +end: a = [ 7, 53, 999 ]; + diff --git a/tests/fsm/in21.fsm b/tests/fsm/in21.fsm new file mode 100644 index 000000000..f53082d14 --- /dev/null +++ b/tests/fsm/in21.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# positive integers only +end: a = [ -1 ]; + diff --git a/tests/fsm/in22.fsm b/tests/fsm/in22.fsm new file mode 100644 index 000000000..c376a9d28 --- /dev/null +++ b/tests/fsm/in22.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +a -> b 'a'; + +start: a; +end: b = [ 7 ]; + diff --git a/tests/fsm/in3.fsm b/tests/fsm/in3.fsm new file mode 100644 index 000000000..b3f86c37b --- /dev/null +++ b/tests/fsm/in3.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# not a valid identifier ++; + diff --git a/tests/fsm/in4.fsm b/tests/fsm/in4.fsm new file mode 100644 index 000000000..e1edb77b8 --- /dev/null +++ b/tests/fsm/in4.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# syntax error +abc -> + diff --git a/tests/fsm/in5.fsm b/tests/fsm/in5.fsm new file mode 100644 index 000000000..8550f0566 --- /dev/null +++ b/tests/fsm/in5.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# syntax error +-> def; + diff --git a/tests/fsm/in6.fsm b/tests/fsm/in6.fsm new file mode 100644 index 000000000..9f6c3c3e8 --- /dev/null +++ b/tests/fsm/in6.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +x -> y; + diff --git a/tests/fsm/in7.fsm b/tests/fsm/in7.fsm new file mode 100644 index 000000000..09d508a27 --- /dev/null +++ b/tests/fsm/in7.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# implicit state +start: x; + diff --git a/tests/fsm/in8.fsm b/tests/fsm/in8.fsm new file mode 100644 index 000000000..7760b6e3c --- /dev/null +++ b/tests/fsm/in8.fsm @@ -0,0 +1,12 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# explicitly created states +x; +y; + +start: x; + diff --git a/tests/fsm/in9.fsm b/tests/fsm/in9.fsm new file mode 100644 index 000000000..e7cbbcc0d --- /dev/null +++ b/tests/fsm/in9.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing separator +a b c + diff --git a/tests/fsm/out0.fsm b/tests/fsm/out0.fsm new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fsm/out1.fsm b/tests/fsm/out1.fsm new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fsm/out10.err b/tests/fsm/out10.err new file mode 100644 index 000000000..412679b24 --- /dev/null +++ b/tests/fsm/out10.err @@ -0,0 +1 @@ +9:1: Syntax error: expected ';' diff --git a/tests/fsm/out11.err b/tests/fsm/out11.err new file mode 100644 index 000000000..d5b538786 --- /dev/null +++ b/tests/fsm/out11.err @@ -0,0 +1 @@ +8:8: Syntax error: expected ',' diff --git a/tests/fsm/out12.err b/tests/fsm/out12.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out12.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out13.err b/tests/fsm/out13.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out13.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out14.err b/tests/fsm/out14.err new file mode 100644 index 000000000..f7e4f861b --- /dev/null +++ b/tests/fsm/out14.err @@ -0,0 +1 @@ +8:3: Syntax error diff --git a/tests/fsm/out15.err b/tests/fsm/out15.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out15.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out16.fsm b/tests/fsm/out16.fsm new file mode 100644 index 000000000..49f89e1ca --- /dev/null +++ b/tests/fsm/out16.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +end: a; diff --git a/tests/fsm/out17.err b/tests/fsm/out17.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out17.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out18.err b/tests/fsm/out18.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out18.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out19.fsm b/tests/fsm/out19.fsm new file mode 100644 index 000000000..11c504879 --- /dev/null +++ b/tests/fsm/out19.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +end: a = [ 0 ]; diff --git a/tests/fsm/out2.fsm b/tests/fsm/out2.fsm new file mode 100644 index 000000000..5e711912e --- /dev/null +++ b/tests/fsm/out2.fsm @@ -0,0 +1,7 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0; 1; 2; 3; 4; diff --git a/tests/fsm/out20.fsm b/tests/fsm/out20.fsm new file mode 100644 index 000000000..71556c552 --- /dev/null +++ b/tests/fsm/out20.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +end: a = [ 0, 1, 2 ]; diff --git a/tests/fsm/out21.err b/tests/fsm/out21.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out21.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out22.fsm b/tests/fsm/out22.fsm new file mode 100644 index 000000000..50f1bb868 --- /dev/null +++ b/tests/fsm/out22.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0 -> 1 'a'; +start: 0; +end: 1 = [ 0 ]; diff --git a/tests/fsm/out3.err b/tests/fsm/out3.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out3.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out4.err b/tests/fsm/out4.err new file mode 100644 index 000000000..31fa45171 --- /dev/null +++ b/tests/fsm/out4.err @@ -0,0 +1 @@ +8:7: Syntax error diff --git a/tests/fsm/out5.err b/tests/fsm/out5.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out5.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out6.fsm b/tests/fsm/out6.fsm new file mode 100644 index 000000000..b673566f0 --- /dev/null +++ b/tests/fsm/out6.fsm @@ -0,0 +1,7 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0 -> 1; diff --git a/tests/fsm/out7.fsm b/tests/fsm/out7.fsm new file mode 100644 index 000000000..3d4653a70 --- /dev/null +++ b/tests/fsm/out7.fsm @@ -0,0 +1,7 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: 0; diff --git a/tests/fsm/out8.fsm b/tests/fsm/out8.fsm new file mode 100644 index 000000000..86110f4ba --- /dev/null +++ b/tests/fsm/out8.fsm @@ -0,0 +1,10 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0; +1; + +start: 0; diff --git a/tests/fsm/out9.err b/tests/fsm/out9.err new file mode 100644 index 000000000..f7e4f861b --- /dev/null +++ b/tests/fsm/out9.err @@ -0,0 +1 @@ +8:3: Syntax error From db27a74564b3ab6d522a2acddb53a844fb70e906 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 04:40:38 -0700 Subject: [PATCH 133/373] Wrong type. This should've been done when updating to fsm_state_t. --- src/libfsm/print/api.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 38e5dfc27..16e787a05 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -93,7 +93,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\n"); fprintf(f, "int\n"); - fprintf(f, "%sfsm(struct fsm *fsm, struct fsm_state *x, struct fsm_state *y)\n", + fprintf(f, "%sfsm(struct fsm *fsm, fsm_state_t *x, fsm_state_t *y)\n", fsm->opt->prefix != NULL ? fsm->opt->prefix : ""); fprintf(f, "{\n"); @@ -103,16 +103,20 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\tsize_t i;\n"); fprintf(f, "\n"); + fprintf(f, "\tassert(x != NULL);\n"); + fprintf(f, "\tassert(y != NULL);\n"); + fprintf(f, "\n"); + fprintf(f, "\tfor (i = 0; i < %zu; i++) {\n", fsm->statecount); fprintf(f, "\t\tif (i == %u) {\n", start); - fprintf(f, "\t\t\ts[%u] = x;\n", start); + fprintf(f, "\t\t\ts[%u] = *x;\n", start); fprintf(f, "\t\t\tcontinue;\n"); fprintf(f, "\t\t}\n"); fprintf(f, "\n"); fprintf(f, "\t\tif (i == %u) {\n", end); - fprintf(f, "\t\t\ts[%u] = y;\n", end); + fprintf(f, "\t\t\ts[%u] = *y;\n", end); fprintf(f, "\t\t\tcontinue;\n"); fprintf(f, "\t\t}\n"); fprintf(f, "\n"); From 26b9d25333193a3ee23a67063d24d5352c33e4da Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 07:43:17 -0700 Subject: [PATCH 134/373] Printing for endids. This is never actually reached, because fsm_collate() doesn't handle end IDs. --- src/libfsm/print/api.c | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 16e787a05..33da6191a 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -27,6 +27,19 @@ #include #include +/* TODO: centralise */ +static int +comp_end_id(const void *a, const void *b) +{ + assert(a != NULL); + assert(b != NULL); + + if (* (fsm_end_id_t *) a < * (fsm_end_id_t *) b) { return -1; } + if (* (fsm_end_id_t *) a > * (fsm_end_id_t *) b) { return +1; } + + return 0; +} + static int rangeclass(unsigned char x, unsigned char y) { @@ -90,6 +103,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\n"); fprintf(f, "#include \n"); + fprintf(f, "#include \n"); fprintf(f, "\n"); fprintf(f, "int\n"); @@ -128,6 +142,47 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\t}\n"); fprintf(f, "\n"); + { + size_t count; + + count = fsm_getendidcount(fsm, end); + if (count > 0) { + enum fsm_getendids_res res; + fsm_end_id_t *ids; + size_t written; + + ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + if (ids == NULL) { + /* XXX */ + goto error; + } + + res = fsm_getendids(fsm, end, count, ids, &written); + switch (res) { + case FSM_GETENDIDS_NOT_FOUND: + case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: + assert(!"unreached"); + abort(); + + case FSM_GETENDIDS_FOUND: + break; + } + + assert(written == count); + + qsort(ids, count, sizeof *ids, comp_end_id); + + fprintf(f, "\tif (fsm_isend(fsm, s[%u])) {\n", end); + for (size_t id = 0; id < count; id++) { + fprintf(f, "\t\tfsm_setendidstate(fsm, s[%u], %zu);\n", end, id); + } + fprintf(f, "\t}\n"); + fprintf(f, "\n"); + + f_free(fsm->opt->alloc, ids); + } + } + a = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *a); if (a == NULL) { /* XXX */ From 20ac21b674b7faafbaf835d65369386adcf99b37 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 11:54:53 -0700 Subject: [PATCH 135/373] Printing for endids. --- src/libfsm/print/irdot.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 206a42e5e..d388aa88e 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -191,6 +191,20 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\t S%u%s\n", ir_indexof(ir, cs), strategy_name(cs->strategy)); + if (cs->isend && cs->end_ids != NULL) { + fprintf(f, "\t\t end id"); + + for (size_t i = 0; i < cs->end_ids->count; i++) { + fprintf(f, "#%u", cs->end_ids->ids[i]); + + if (i < cs->end_ids->count - 1) { + fprintf(f, " "); + } + } + + fprintf(f, "\n"); + } + if (cs->example != NULL) { fprintf(f, "\t\t example"); escputs(f, opt, dot_escputc_html, cs->example); From 8edcc074d50ee977b3a23c2163c650ae387b48f5 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 12:01:00 -0700 Subject: [PATCH 136/373] Printing for endids. --- src/libfsm/print/irjson.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 173bef0df..a1e154942 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -123,6 +123,18 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\t{\n"); fprintf(f, "\t\t\t\"end\": %s,\n", cs->isend ? "true" : "false"); + if (cs->isend && cs->end_ids != NULL) { + fprintf(f, "\t\t\t\"end_id\": ["); + for (size_t i = 0; i < cs->end_ids->count; i++) { + fprintf(f, "%u", cs->end_ids->ids[i]); + + if (i < cs->end_ids->count - 1) { + fprintf(f, ", "); + } + } + fprintf(f, "],\n"); + } + fprintf(f, "\t\t\t\"strategy\": \"%s\"", strategy_name(cs->strategy)); if (cs->example != NULL || cs->strategy != IR_NONE) { fprintf(f, ","); From 69efe14ad293efca25cb2106ad7585c3160daa1b Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 12:39:52 -0700 Subject: [PATCH 137/373] Missing free. --- src/libfsm/print/json.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 5c3235237..9a878825c 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -184,6 +184,8 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) print_edge_bitmap(f, notfirst, fsm->opt, s, e.state, &bm); } + state_set_free(unique); + /* * Special edges are not consolidated above */ From 2b4821e7e1933e06818c1d9b154162c981b5a98e Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 13:56:25 -0700 Subject: [PATCH 138/373] Printing for endids. --- src/libfsm/print/vmdot.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 101934f17..775714798 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -97,6 +97,18 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } } else { fprintf(f, "ret %td", op->ir_state - ir->states); + + if (op->ir_state->end_ids != NULL) { + fprintf(f, " / "); + + for (size_t i = 0; i < op->ir_state->end_ids->count; i++) { + fprintf(f, "#%u", op->ir_state->end_ids->ids[i]); + + if (i < op->ir_state->end_ids->count - 1) { + fprintf(f, " "); + } + } + } } return 0; From e07f66e9f4a322efbb9069e87cc2e026fdadc7b4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 11 Jun 2024 16:45:32 -0700 Subject: [PATCH 139/373] No need for the wrapper functions here. --- include/fsm/fsm.h | 4 +-- src/libfsm/clone.c | 4 +-- src/libfsm/endids.c | 22 ++------------ src/libfsm/endids.h | 9 ------ src/libfsm/epsilons.c | 4 +-- src/libfsm/libfsm.syms | 4 +-- src/libfsm/minimise.c | 4 +-- src/libfsm/minimise_test_oracle.c | 6 ++-- src/libfsm/print/api.c | 4 +-- src/libfsm/print/dot.c | 4 +-- src/libfsm/print/fsm.c | 4 +-- src/libfsm/print/ir.c | 4 +-- src/libfsm/print/json.c | 4 +-- src/libfsm/walk2.c | 8 ++--- src/lx/ast.c | 4 +-- src/re/main.c | 2 +- tests/capture/captest.c | 8 ++--- tests/capture/capture4.c | 2 +- tests/endids/endids0.c | 4 +-- tests/endids/endids0_many_endids.c | 8 ++--- tests/endids/endids1_determinise.c | 4 +-- .../endids/endids1_determinise_and_minimise.c | 4 +-- tests/endids/endids2_union.c | 16 +++++----- tests/endids/endids2_union_many_endids.c | 22 +++++++------- tests/endids/endids4.c | 18 +++++------ tests/endids/endids5.c | 20 ++++++------- tests/endids/endids6.c | 30 +++++++++---------- tests/endids/endids7.c | 26 ++++++++-------- tests/endids/endids7_with_duplicates.c | 26 ++++++++-------- tests/endids/endids8.c | 14 ++++----- tests/endids/endids9.c | 14 ++++----- tests/endids/utils.c | 16 +++++----- tests/gen/gtest.c | 4 +-- tests/re_strings/testutil.c | 2 +- 34 files changed, 149 insertions(+), 180 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 2fea117cd..81c472d3e 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -238,13 +238,13 @@ enum fsm_getendids_res { FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE = -1 }; enum fsm_getendids_res -fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, +fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, size_t id_buf_count, fsm_end_id_t *id_buf, size_t *ids_written); /* Get the number of end IDs associated with an end state. */ size_t -fsm_getendidcount(const struct fsm *fsm, fsm_state_t end_state); +fsm_endid_count(const struct fsm *fsm, fsm_state_t end_state); /* Callback function to remap the end ids of a state. This function can * remap to fewer end ids, but cannot add additional end ids, and cannot diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 78511b12c..587976e94 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -127,15 +127,13 @@ static int copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) { struct copy_end_ids_env *env = opaque; - enum fsm_endid_set_res sres; assert(env->tag == 'c'); #if LOG_CLONE_ENDIDS fprintf(stderr, "clone[%d] <- %d\n", state, id); #endif - sres = fsm_endid_set(env->dst, state, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_setendidstate(env->dst, state, id)) { env->ok = 0; return 0; } diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 1fc98ae68..b2c1eeadf 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -69,13 +69,11 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) /* for every end state */ for (i = 0; i < fsm->statecount; i++) { if (fsm_isend(fsm, i)) { - enum fsm_endid_set_res sres; #if LOG_ENDIDS > 3 fprintf(stderr, "fsm_setendid: setting id %u on state %d\n", id, i); #endif - sres = fsm_endid_set(fsm, i, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_setendidstate(fsm, i, id)) { return 0; } } @@ -94,21 +92,6 @@ fsm_setendidstate(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id) return 1; } -enum fsm_getendids_res -fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written) -{ - return fsm_endid_get(fsm, end_state, - id_buf_count, id_buf, ids_written); -} - -size_t -fsm_getendidcount(const struct fsm *fsm, fsm_state_t end_state) -{ - return fsm_endid_count(fsm, end_state); -} - int fsm_endid_init(struct fsm *fsm) { @@ -796,8 +779,7 @@ carry_iter_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) (void)state; - sres = fsm_endid_set(env->dst, env->dst_state, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_setendidstate(env->dst, env->dst_state, id)) { env->ok = 0; return 0; } diff --git a/src/libfsm/endids.h b/src/libfsm/endids.h index 6c46567b3..c4b3184b5 100644 --- a/src/libfsm/endids.h +++ b/src/libfsm/endids.h @@ -45,15 +45,6 @@ enum fsm_endid_set_res fsm_endid_set_bulk(struct fsm *fsm, fsm_state_t state, size_t num_ids, const fsm_end_id_t *ids, enum fsm_endid_bulk_op op); -size_t -fsm_endid_count(const struct fsm *fsm, - fsm_state_t state); - -enum fsm_getendids_res -fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written); - int fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, struct fsm *dst_fsm, fsm_state_t dst_state); diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index e87d9d974..82dd6cfa4 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -405,9 +405,7 @@ carry_endids(struct fsm *fsm, struct state_set *states, /* add them */ for (i = 0; i < env.count; i++) { - enum fsm_endid_set_res sres; - sres = fsm_endid_set(fsm, dst_state, env.ids[i]); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_setendidstate(fsm, dst_state, env.ids[i])) { env.ok = 0; goto cleanup; } diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 6e1f72dbe..0e40ae884 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -86,8 +86,8 @@ fsm_setstart fsm_getstart fsm_setend -fsm_getendidcount -fsm_getendids +fsm_endid_count +fsm_endid_get fsm_setendid fsm_mapendids fsm_increndids diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 94dfdf80b..294615292 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -951,7 +951,7 @@ static int collect_end_ids(const struct fsm *fsm, fsm_state_t s, struct end_metadata_end *e) { - e->count = fsm_getendidcount(fsm, s); + e->count = fsm_endid_count(fsm, s); if (e->count > 0) { e->ids = f_malloc(fsm->opt->alloc, @@ -961,7 +961,7 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, } size_t written; - enum fsm_getendids_res res = fsm_getendids(fsm, s, + enum fsm_getendids_res res = fsm_endid_get(fsm, s, e->count, e->ids, &written); assert(res == FSM_GETENDIDS_FOUND); assert(written == e->count); diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 9045ce67b..388cc3634 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -143,7 +143,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) * This includes the dead state. */ for (size_t i = 0; i < table_states; i++) { if (i < state_count && fsm_isend(fsm, i)) { - const size_t count = fsm_getendidcount(fsm, i); + const size_t count = fsm_endid_count(fsm, i); if (count > max_endid_count) { max_endid_count = count; } @@ -177,7 +177,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) for (size_t i = 0; i < state_count; i++) { if (fsm_isend(fsm, i)) { size_t written_a; - enum fsm_getendids_res eres = fsm_getendids(fsm, i, + enum fsm_getendids_res eres = fsm_endid_get(fsm, i, max_endid_count, endid_buf_a, &written_a); assert(eres != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); if (eres == FSM_GETENDIDS_NOT_FOUND) { @@ -188,7 +188,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) /* note: skipping eg 0 here since that's the empty set */ for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { size_t written_b; - eres = fsm_getendids(fsm, endid_group_leaders[eg_i], + eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], max_endid_count, endid_buf_b, &written_b); assert(eres != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); if (written_b != written_a) { continue; } diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 33da6191a..dab9cc4df 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -145,7 +145,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) { size_t count; - count = fsm_getendidcount(fsm, end); + count = fsm_endid_count(fsm, end); if (count > 0) { enum fsm_getendids_res res; fsm_end_id_t *ids; @@ -157,7 +157,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) goto error; } - res = fsm_getendids(fsm, end, count, ids, &written); + res = fsm_endid_get(fsm, end, count, ids, &written); switch (res) { case FSM_GETENDIDS_NOT_FOUND: case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index ab360af2b..518009dd3 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -170,7 +170,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) enum fsm_getendids_res res; size_t written; struct fsm_end_ids *ids = NULL; - const size_t count = fsm_getendidcount(fsm, s); + const size_t count = fsm_endid_count(fsm, s); if (count > 0) { ids = f_malloc(fsm->opt->alloc, sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); @@ -179,7 +179,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_getendids(fsm, s, count, + res = fsm_endid_get(fsm, s, count, ids->ids, &written); if (res == FSM_GETENDIDS_FOUND) { ids->count = (unsigned)written; diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index ae3163abd..928c6f83a 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -301,7 +301,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) fprintf(f, "%u", s); - count = fsm_getendidcount(fsm, s); + count = fsm_endid_count(fsm, s); if (count > 0) { enum fsm_getendids_res res; fsm_end_id_t *ids; @@ -312,7 +312,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_getendids(fsm, s, count, ids, &written); + res = fsm_endid_get(fsm, s, count, ids, &written); switch (res) { case FSM_GETENDIDS_NOT_FOUND: case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 3a7aee222..7c53e3846 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -471,14 +471,14 @@ make_ir(const struct fsm *fsm) if (fsm_isend(fsm, i)) { enum fsm_getendids_res res; size_t written; - const size_t count = fsm_getendidcount(fsm, i); + const size_t count = fsm_endid_count(fsm, i); struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, sizeof(*ids) + (count == 0 ? 0 : (count - 1) * sizeof(ids->ids[0]))); if (ids == NULL) { goto error; } - res = fsm_getendids(fsm, i, count, + res = fsm_endid_get(fsm, i, count, ids->ids, &written); if (res == FSM_GETENDIDS_FOUND) { ids->count = (unsigned)written; diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 9a878825c..69615d287 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -249,12 +249,12 @@ fsm_print_json(FILE *f, const struct fsm *fsm) if (fsm_isend(fsm, i)) { enum fsm_getendids_res res; size_t written; - const size_t count = fsm_getendidcount(fsm, i); + const size_t count = fsm_endid_count(fsm, i); struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, sizeof(*ids) + ((count - 1) * sizeof(ids->ids[0]))); assert(ids != NULL); - res = fsm_getendids(fsm, i, count, + res = fsm_endid_get(fsm, i, count, ids->ids, &written); if (res == FSM_GETENDIDS_FOUND) { ids->count = (unsigned)written; diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index c321db678..b21883abd 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -266,11 +266,11 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, size_t num_a_endids = 0, num_b_endids = 0, total_num_endids; if (fsm_a != NULL && fsm_isend(fsm_a,a)) { - num_a_endids = fsm_getendidcount(fsm_a, a); + num_a_endids = fsm_endid_count(fsm_a, a); } if (fsm_b != NULL && fsm_isend(fsm_b,b)) { - num_b_endids = fsm_getendidcount(fsm_b, b); + num_b_endids = fsm_endid_count(fsm_b, b); } total_num_endids = num_a_endids + num_b_endids; @@ -286,7 +286,7 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, if (num_a_endids > 0) { size_t nwritten = 0; - ret = fsm_getendids(fsm_a, a, num_a_endids, &endids[0], &nwritten); + ret = fsm_endid_get(fsm_a, a, num_a_endids, &endids[0], &nwritten); if (ret != FSM_GETENDIDS_FOUND || nwritten != num_a_endids) { free(endids); @@ -297,7 +297,7 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, if (num_b_endids > 0) { size_t nwritten = 0; - ret = fsm_getendids(fsm_b, b, num_b_endids, &endids[num_a_endids], &nwritten); + ret = fsm_endid_get(fsm_b, b, num_b_endids, &endids[num_a_endids], &nwritten); if (ret != FSM_GETENDIDS_FOUND || nwritten != num_b_endids) { free(endids); diff --git a/src/lx/ast.c b/src/lx/ast.c index 75754f58d..4d088cd75 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -241,7 +241,7 @@ ast_getendmapping(const struct fsm *fsm, fsm_state_t s) size_t written; struct ast_mapping *m; - id_count = fsm_getendidcount(fsm, s); + id_count = fsm_endid_count(fsm, s); if (id_count > ID_STACK_BUF_COUNT) { id_buf_dynamic = malloc(id_count * sizeof(id_buf_dynamic[0])); if (id_buf_dynamic == NULL) { @@ -249,7 +249,7 @@ ast_getendmapping(const struct fsm *fsm, fsm_state_t s) } } - res = fsm_getendids(fsm, + res = fsm_endid_get(fsm, s, id_count, id_buf_dynamic == NULL ? id_buf : id_buf_dynamic, &written); diff --git a/src/re/main.c b/src/re/main.c index 77f147518..592e368c7 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -294,7 +294,7 @@ find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) return NULL; } - res = fsm_getendids(dfa, s, MAX_END_IDS, + res = fsm_endid_get(dfa, s, MAX_END_IDS, end_id_buf, &end_ids_written); if (res == FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE) { fprintf(stderr, "Error: Multiple end IDs\n"); diff --git a/tests/capture/captest.c b/tests/capture/captest.c index dc66f81d4..b2bc1243f 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -92,11 +92,11 @@ captest_run_single(const struct captest_single_fsm_test_info *info) fsm_end_id_t id_buf[1] = { ~0 }; enum fsm_getendids_res gres; size_t written; - if (1 != fsm_getendidcount(fsm, end)) { + if (1 != fsm_endid_count(fsm, end)) { FAIL("did not have exactly one end ID"); } - gres = fsm_getendids(fsm, end, 1, id_buf, &written); + gres = fsm_endid_get(fsm, end, 1, id_buf, &written); if (gres != FSM_GETENDIDS_FOUND) { FAIL("failed to get end IDs"); } @@ -190,12 +190,12 @@ captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, msg = &unused; } - if (1 != fsm_getendidcount(fsm, end_state)) { + if (1 != fsm_endid_count(fsm, end_state)) { *msg = "did not have exactly one end ID"; return 0; } - gres = fsm_getendids(fsm, end_state, 1, id_buf, &written); + gres = fsm_endid_get(fsm, end_state, 1, id_buf, &written); if (gres != FSM_GETENDIDS_FOUND) { *msg = "failed to get end IDs"; return 0; diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 170cbe8b0..1759afb05 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -223,7 +223,7 @@ check(const struct fsm *fsm, const char *string, enum fsm_getendids_res gres; fsm_end_id_t id_buf[2]; size_t written; - gres = fsm_getendids(fsm, end, 2, id_buf, &written); + gres = fsm_endid_get(fsm, end, 2, id_buf, &written); if (gres != FSM_GETENDIDS_FOUND) { assert(!"fsm_getendids failed"); } diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index f101f0cc6..4dd4b4ebe 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -47,10 +47,10 @@ int main(void) size_t nwritten; enum fsm_getendids_res ret; - assert( fsm_getendidcount(fsm, state_ind) == 1); + assert( fsm_endid_count(fsm, state_ind) == 1); nwritten = 0; - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, 1, diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index cd5230855..834702bdc 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -69,7 +69,7 @@ int main(void) assert(end_state < nstates); - assert(fsm_getendidcount(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); for (i=0; i < sizeof all_end_ids / sizeof all_end_ids[0]; i++) { /* add duplicate end ids */ ret = fsm_setendid(fsm, all_end_ids[i]); @@ -78,7 +78,7 @@ int main(void) assert(ret == 1); /* but it shouldn't add a duplicate id */ - assert(fsm_getendidcount(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); } nend = 0; @@ -90,10 +90,10 @@ int main(void) memset(&endids[0], 0, sizeof endids); - assert(fsm_getendidcount(fsm, state_ind) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, state_ind) == sizeof all_end_ids / sizeof all_end_ids[0]); nwritten = 0; - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids/sizeof endids[0], diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 956e0ee33..75a89db0d 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -43,10 +43,10 @@ int main(void) size_t nwritten; enum fsm_getendids_res ret; - assert( fsm_getendidcount(fsm, state_ind) == 1); + assert( fsm_endid_count(fsm, state_ind) == 1); nwritten = 0; - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, 1, diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index 4619af4a8..d8df93543 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -48,10 +48,10 @@ int main(void) size_t nwritten; enum fsm_getendids_res ret; - assert( fsm_getendidcount(fsm, state_ind) == 1); + assert( fsm_endid_count(fsm, state_ind) == 1); nwritten = 0; - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, 1, diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index d65ef17e8..a81bce01c 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -56,16 +56,16 @@ int main(void) if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); - // fprintf(stderr, "state %u, num_endids = %zu\n", state_ind, num_endids); + count = fsm_endid_count(comb, state_ind); + // fprintf(stderr, "state %u, count = %zu\n", state_ind, count); - assert( num_endids > 0 && num_endids <= 2); + assert( count > 0 && count <= 2); - ret = fsm_getendids( + ret = fsm_endid_get( comb, state_ind, 2, @@ -73,7 +73,7 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); if (nwritten == 1) { assert(endids[0] == 1 || endids[0] == 2); @@ -111,7 +111,7 @@ int main(void) static const struct { const char *s; int should_match; - size_t num_endids; + size_t count; fsm_end_id_t endid[2]; } matches[] = { { "abc" , 1, 1, { 1, 0 } }, @@ -160,7 +160,7 @@ int main(void) assert( ret == 1 ); assert( end_ids != NULL ); - assert( num_end_ids == matches[i].num_endids ); + assert( num_end_ids == matches[i].count ); qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index 46f2a189e..d7beb8777 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -183,17 +183,17 @@ int main(void) if (fsm_isend(fsm, state_ind)) { int tested_pattern[NUM_PATTERNS]; fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, num_endids, j; + size_t nwritten, count, j; enum fsm_getendids_res ret; memset(&endids[0], 0, sizeof endids); nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert(num_endids > 0 && num_endids <= sizeof endids/sizeof endids[0]); + assert(count > 0 && count <= sizeof endids/sizeof endids[0]); - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids/sizeof endids[0], @@ -201,11 +201,11 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); memset(&tested_pattern[0], 0, sizeof tested_pattern); - for (j=0; j < num_endids; j++) { + for (j=0; j < count; j++) { size_t k, pattern_index; pattern_index = (endids[j] - 1)/NUM_ENDIDS_PER_PATTERN; @@ -250,17 +250,17 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, num_endids; + size_t nwritten, count; enum fsm_getendids_res ret; memset(&endids[0], 0, sizeof endids); nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert(num_endids <= NUM_ENDIDS_TOTAL); + assert(count <= NUM_ENDIDS_TOTAL); - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids/sizeof endids[0], @@ -268,7 +268,7 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); } } diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index c8f312663..c4b7da823 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -68,17 +68,17 @@ int main(void) if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); + count = fsm_endid_count(comb, state_ind); - printf("state %u num_endids = %zu\n", state_ind, num_endids); + printf("state %u count = %zu\n", state_ind, count); - assert(num_endids == 2); + assert(count == 2); - ret = fsm_getendids( + ret = fsm_endid_get( comb, state_ind, 2, @@ -86,9 +86,9 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); - qsort(&endids[0], num_endids, sizeof endids[0], cmp_endids); + qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2); } } @@ -102,7 +102,7 @@ int main(void) static const struct { const char *s; int should_match; - size_t num_endids; + size_t count; fsm_end_id_t endid[2]; } matches[] = { { "abc" , 0, 0, { 0, 0 } }, @@ -151,7 +151,7 @@ int main(void) assert( ret == 1 ); assert( end_ids != NULL ); - assert( num_end_ids == matches[i].num_endids ); + assert( num_end_ids == matches[i].count ); qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index f6220115a..91e39b5eb 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -69,17 +69,17 @@ int main(void) if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); + count = fsm_endid_count(comb, state_ind); - printf("state %u num_endids = %zu\n", state_ind, num_endids); + printf("state %u count = %zu\n", state_ind, count); - assert(num_endids == 1); + assert(count == 1); - ret = fsm_getendids( + ret = fsm_endid_get( comb, state_ind, 2, @@ -87,7 +87,7 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); assert(endids[0] == 1); } @@ -102,7 +102,7 @@ int main(void) static const struct { const char *s; int should_match; - size_t num_endids; + size_t count; fsm_end_id_t endid[2]; } matches[] = { { "abc" , 1, 1, { 1, 0 } }, @@ -145,8 +145,8 @@ int main(void) num_end_ids = 0; ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); - printf("match %zu, \"%s\", should_match=%d, num_endids=%zu, endids={ %u, %u }\n", - i, matches[i].s, matches[i].should_match, matches[i].num_endids, + printf("match %zu, \"%s\", should_match=%d, count=%zu, endids={ %u, %u }\n", + i, matches[i].s, matches[i].should_match, matches[i].count, (unsigned)matches[i].endid[0], (unsigned)matches[i].endid[1]); if (matches[i].should_match) { @@ -155,7 +155,7 @@ int main(void) assert( ret == 1 ); assert( end_ids != NULL ); - assert( num_end_ids == matches[i].num_endids ); + assert( num_end_ids == matches[i].count ); qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index ca089d0f1..2179d0a03 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -19,7 +19,7 @@ struct state_info { fsm_state_t state; - unsigned num_endids; + unsigned count; fsm_end_id_t endids[2]; }; @@ -93,15 +93,15 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert( num_endids > 0 && num_endids <= 2); + assert( count > 0 && count <= 2); - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids / sizeof endids[0], @@ -109,10 +109,10 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; + info[ninfo].count = nwritten; if (nwritten == 1) { assert(endids[0] == 1 || endids[0] == 2); @@ -137,18 +137,18 @@ int main(void) for (state_ind = 0, info_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten, num_endids, ind; + size_t nwritten, count, ind; enum fsm_getendids_res ret; assert( state_ind == info[info_ind].state ); nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert(num_endids > 0 && num_endids <= 2); + assert(count > 0 && count <= 2); - assert( num_endids == info[info_ind].num_endids ); - ret = fsm_getendids( + assert( count == info[info_ind].count ); + ret = fsm_endid_get( fsm, state_ind, sizeof endids / sizeof endids[0], @@ -156,13 +156,13 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); if (nwritten > 1) { - qsort(&endids[0], num_endids, sizeof endids[0], cmp_endids); + qsort(&endids[0], count, sizeof endids[0], cmp_endids); } - for (ind=0; ind < num_endids; ind++) { + for (ind=0; ind < count; ind++) { fsm_end_id_t expected = info[info_ind].endids[ind]; switch (info[info_ind].endids[ind]) { case 1: expected = 512; break; diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index cc68d02c0..c695136cc 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -19,7 +19,7 @@ struct state_info { fsm_state_t state; - unsigned num_endids; + unsigned count; fsm_end_id_t endids[2]; }; @@ -105,15 +105,15 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert( num_endids > 0 && num_endids <= 2); + assert( count > 0 && count <= 2); - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids / sizeof endids[0], @@ -121,10 +121,10 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; + info[ninfo].count = nwritten; if (nwritten == 1) { assert(endids[0] == 1 || endids[0] == 2); @@ -150,16 +150,16 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; fsm_end_id_t expected; - size_t nwritten, num_endids; + size_t nwritten, count; enum fsm_getendids_res ret; assert( state_ind == info[info_ind].state ); nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert(num_endids == 1); - ret = fsm_getendids( + assert(count == 1); + ret = fsm_endid_get( fsm, state_ind, 1, @@ -167,9 +167,9 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); - if (info[info_ind].num_endids == 2) { + if (info[info_ind].count == 2) { expected = 3; } else if (info[info_ind].endids[0] == 1) { expected = 7; diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index b554f7cb6..3da80d9bc 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -19,7 +19,7 @@ struct state_info { fsm_state_t state; - unsigned num_endids; + unsigned count; fsm_end_id_t endids[2]; }; @@ -109,15 +109,15 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert( num_endids > 0 && num_endids <= 2); + assert( count > 0 && count <= 2); - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids / sizeof endids[0], @@ -125,10 +125,10 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; + info[ninfo].count = nwritten; if (nwritten == 1) { assert(endids[0] == 1 || endids[0] == 2); @@ -154,16 +154,16 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; fsm_end_id_t expected; - size_t nwritten, num_endids; + size_t nwritten, count; enum fsm_getendids_res ret; assert( state_ind == info[info_ind].state ); nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert(num_endids == 1); - ret = fsm_getendids( + assert(count == 1); + ret = fsm_endid_get( fsm, state_ind, 1, @@ -171,9 +171,9 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); - if (info[info_ind].num_endids == 2) { + if (info[info_ind].count == 2) { expected = 3; } else if (info[info_ind].endids[0] == 1) { expected = 7; diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index e413e71e7..0be5c244d 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -74,17 +74,17 @@ int main(void) if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[3] = {0,0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); + count = fsm_endid_count(comb, state_ind); - printf("state %u num_endids = %zu\n", state_ind, num_endids); + printf("state %u count = %zu\n", state_ind, count); - assert(num_endids == 3); + assert(count == 3); - ret = fsm_getendids( + ret = fsm_endid_get( comb, state_ind, sizeof endids / sizeof endids[0], @@ -92,9 +92,9 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); - qsort(&endids[0], num_endids, sizeof endids[0], cmp_endids); + qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2 && endids[2] == 4); } } diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index de3f28644..c6c467328 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -19,7 +19,7 @@ struct state_info { fsm_state_t state; - unsigned num_endids; + unsigned count; fsm_end_id_t endids[2]; }; @@ -73,15 +73,15 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten; - size_t num_endids; + size_t count; enum fsm_getendids_res ret; nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); - assert( num_endids > 0 && num_endids <= 2); + assert( count > 0 && count <= 2); - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, sizeof endids / sizeof endids[0], @@ -89,10 +89,10 @@ int main(void) &nwritten); assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; + info[ninfo].count = nwritten; if (nwritten == 1) { assert(endids[0] == 11 || endids[0] == 12); diff --git a/tests/endids/utils.c b/tests/endids/utils.c index 79777e825..b6dd23042 100644 --- a/tests/endids/utils.c +++ b/tests/endids/utils.c @@ -4,30 +4,30 @@ #include int -match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **endids_ptr, size_t *num_endids_ptr) +match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **endids_ptr, size_t *count_ptr) { fsm_state_t end = 0; int ret; ret = fsm_exec(fsm, fsm_sgetc, &s, &end, NULL); if (ret == 1) { - size_t num_endids; + size_t count; if (end_ptr != NULL) { *end_ptr = end; } - num_endids = fsm_getendidcount(fsm, end); - if (num_endids > 0) { + count = fsm_endid_count(fsm, end); + if (count > 0) { enum fsm_getendids_res ret; size_t nwritten = 0; - fsm_end_id_t *endids = calloc(num_endids, sizeof *endids); + fsm_end_id_t *endids = calloc(count, sizeof *endids); if (endids == NULL) { return -1; } - ret = fsm_getendids(fsm, end, num_endids, endids, &nwritten); + ret = fsm_endid_get(fsm, end, count, endids, &nwritten); if (ret != FSM_GETENDIDS_FOUND) { free(endids); errno = (ret == FSM_GETENDIDS_NOT_FOUND) ? EINVAL : ENOMEM; @@ -36,8 +36,8 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end if (endids_ptr != NULL) { *endids_ptr = endids; - if (num_endids_ptr != NULL) { - *num_endids_ptr = num_endids; + if (count_ptr != NULL) { + *count_ptr = count; } } else { free(endids); diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index 140407943..c99b2eeac 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -60,12 +60,12 @@ gtest_matches_cb(const struct fsm *fsm, #define ID_BUF_COUNT 1 fsm_end_id_t id_buf[ID_BUF_COUNT]; size_t written; - enum fsm_getendids_res gres = fsm_getendids(fsm, + enum fsm_getendids_res gres = fsm_endid_get(fsm, end_state, ID_BUF_COUNT, id_buf, &written); if (gres != FSM_GETENDIDS_FOUND) { fprintf(stderr, - "ERROR: fsm_getendids: returned %d\n", + "ERROR: fsm_endid_get: returned %d\n", gres); return FSM_GENERATE_MATCHES_CB_RES_HALT; } diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 4f4678b0d..5b366c0ce 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -48,7 +48,7 @@ run_test(const char **strings) assert(res > 0); /* match */ size_t written; - enum fsm_getendids_res eres = fsm_getendids(fsm, end, + enum fsm_getendids_res eres = fsm_endid_get(fsm, end, MAX_INPUTS, id_buf, &written); assert(eres == FSM_GETENDIDS_FOUND); bool found = false; From ed4709c74e55590b5c25943aabb3dede39308447 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 12 Jun 2024 02:42:48 -0700 Subject: [PATCH 140/373] No need for the tri-state enum here. I don't see any reason callers of fsm_endid_get() would want to fetch into a fixed-size buffer of fewer elements than the number of endids an end state has. In all cases the caller can fsm_endid_count() to find out. --- include/fsm/fsm.h | 9 +--- src/libfsm/endids.c | 11 +++-- src/libfsm/minimise.c | 41 ++++++++--------- src/libfsm/minimise_test_oracle.c | 7 ++- src/libfsm/print/api.c | 13 +----- src/libfsm/print/dot.c | 10 ++--- src/libfsm/print/fsm.c | 13 +----- src/libfsm/print/ir.c | 5 +-- src/libfsm/print/json.c | 5 +-- src/libfsm/walk2.c | 24 ++++------ src/lx/ast.c | 12 +---- src/re/main.c | 44 +++++++++---------- tests/capture/captest.c | 8 ++-- tests/capture/capture4.c | 5 ++- tests/endids/endids0.c | 10 ++--- tests/endids/endids0_many_endids.c | 10 ++--- tests/endids/endids1_determinise.c | 8 ++-- .../endids/endids1_determinise_and_minimise.c | 12 ++--- tests/endids/endids2_union.c | 4 +- tests/endids/endids2_union_many_endids.c | 8 ++-- tests/endids/endids4.c | 8 ++-- tests/endids/endids5.c | 6 +-- tests/endids/endids6.c | 8 ++-- tests/endids/endids7.c | 8 ++-- tests/endids/endids7_with_duplicates.c | 8 ++-- tests/endids/endids8.c | 8 ++-- tests/endids/endids9.c | 4 +- tests/endids/utils.c | 6 +-- tests/gen/gtest.c | 4 +- tests/re_strings/testutil.c | 4 +- 30 files changed, 139 insertions(+), 184 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 81c472d3e..484541735 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -228,16 +228,11 @@ fsm_setendidstate(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); * If id_buf has enough cells to store all the end IDs (according * to id_buf_count) then they are written into id_buf[] and * *ids_written is set to the number of IDs. The end IDs in the - * buffer may appear in any order, but should not have duplicates. + * buffer may appear in any order, but will not have duplicates. * * Returns 0 if there is not enough space in id_buf for the * end IDs, or 1 if zero or more end IDs were returned. */ -enum fsm_getendids_res { - FSM_GETENDIDS_NOT_FOUND, - FSM_GETENDIDS_FOUND, - FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE = -1 -}; -enum fsm_getendids_res +int fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, size_t id_buf_count, fsm_end_id_t *id_buf, size_t *ids_written); diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index b2c1eeadf..80fcbfd8d 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -691,7 +691,7 @@ fsm_endid_count(const struct fsm *fsm, return 0; } -enum fsm_getendids_res +int fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, size_t id_buf_count, fsm_end_id_t *id_buf, size_t *ids_written) @@ -732,14 +732,14 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, fprintf(stderr, "fsm_endid_get: not found\n"); #endif *ids_written = 0; /* not found */ - return FSM_GETENDIDS_NOT_FOUND; + return 0; } else if (b->state == end_state) { size_t id_i; if (b->ids->count > id_buf_count) { #if LOG_ENDIDS > 2 fprintf(stderr, "fsm_endid_get: insufficient space\n"); #endif - return FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE; + return 0; /* insufficient space */ } for (id_i = 0; id_i < b->ids->count; id_i++) { #if LOG_ENDIDS > 2 @@ -750,7 +750,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, /* todo: could sort them here, if it matters. */ *ids_written = b->ids->count; - return FSM_GETENDIDS_FOUND; + return 1; } else { /* collision */ #if LOG_ENDIDS > 4 fprintf(stderr, "fsm_endid_get: collision\n"); @@ -760,7 +760,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, } assert(!"unreachable"); - return FSM_GETENDIDS_NOT_FOUND; + return 0; } struct carry_env { @@ -773,7 +773,6 @@ struct carry_env { static int carry_iter_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) { - enum fsm_endid_set_res sres; struct carry_env *env = opaque; assert(env->tag == 'C'); diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 294615292..a55de5d61 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -952,32 +952,33 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, struct end_metadata_end *e) { e->count = fsm_endid_count(fsm, s); + if (e->count == 0) { + return 1; + } - if (e->count > 0) { - e->ids = f_malloc(fsm->opt->alloc, - e->count * sizeof(e->ids[0])); - if (e->ids == NULL) { - return 0; - } + e->ids = f_malloc(fsm->opt->alloc, + e->count * sizeof(e->ids[0])); + if (e->ids == NULL) { + return 0; + } - size_t written; - enum fsm_getendids_res res = fsm_endid_get(fsm, s, - e->count, e->ids, &written); - assert(res == FSM_GETENDIDS_FOUND); - assert(written == e->count); + size_t written; + int res = fsm_endid_get(fsm, s, e->count, e->ids, &written); + assert(res == 1); + assert(written == e->count); - /* sort, to make comparison easier later */ - qsort(e->ids, e->count, - sizeof(e->ids[0]), cmp_end_ids); + /* sort, to make comparison easier later */ + qsort(e->ids, e->count, + sizeof(e->ids[0]), cmp_end_ids); #if LOG_ECS - fprintf(stderr, "%d:", s); - for (size_t i = 0; i < written; i++) { - fprintf(stderr, " %u", e->ids[i]); - } - fprintf(stderr, "\n"); -#endif + fprintf(stderr, "%d:", s); + for (size_t i = 0; i < written; i++) { + fprintf(stderr, " %u", e->ids[i]); } + fprintf(stderr, "\n"); +#endif + return 1; } diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 388cc3634..70209f865 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -177,10 +177,9 @@ fsm_minimise_test_oracle(const struct fsm *fsm) for (size_t i = 0; i < state_count; i++) { if (fsm_isend(fsm, i)) { size_t written_a; - enum fsm_getendids_res eres = fsm_endid_get(fsm, i, + int eres = fsm_endid_get(fsm, i, max_endid_count, endid_buf_a, &written_a); - assert(eres != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); - if (eres == FSM_GETENDIDS_NOT_FOUND) { + if (eres == 0) { assert(written_a == 0); } else { assert(written_a > 0); @@ -190,7 +189,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) size_t written_b; eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], max_endid_count, endid_buf_b, &written_b); - assert(eres != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); + assert(eres != 0); if (written_b != written_a) { continue; } if (0 == memcmp(endid_buf_a, endid_buf_b, written_a * sizeof(endid_buf_a[0]))) { found = true; diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index dab9cc4df..69bf76eed 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -147,9 +147,9 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) count = fsm_endid_count(fsm, end); if (count > 0) { - enum fsm_getendids_res res; fsm_end_id_t *ids; size_t written; + int res; ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); if (ids == NULL) { @@ -158,16 +158,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) } res = fsm_endid_get(fsm, end, count, ids, &written); - switch (res) { - case FSM_GETENDIDS_NOT_FOUND: - case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: - assert(!"unreached"); - abort(); - - case FSM_GETENDIDS_FOUND: - break; - } - + assert(res == 1); assert(written == count); qsort(ids, count, sizeof *ids, comp_end_id); diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 518009dd3..1f070b382 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -167,11 +167,12 @@ print_dotfrag(FILE *f, const struct fsm *fsm) s, s); } } else { - enum fsm_getendids_res res; size_t written; struct fsm_end_ids *ids = NULL; const size_t count = fsm_endid_count(fsm, s); if (count > 0) { + int res; + ids = f_malloc(fsm->opt->alloc, sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); assert(ids != NULL); @@ -179,12 +180,9 @@ print_dotfrag(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_endid_get(fsm, s, count, - ids->ids, &written); - if (res == FSM_GETENDIDS_FOUND) { + res = fsm_endid_get(fsm, s, count, ids->ids, &written); + if (res == 1) { ids->count = (unsigned)written; - } else { - assert(res == FSM_GETENDIDS_NOT_FOUND); } } diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 928c6f83a..762bb1c30 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -303,9 +303,9 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) count = fsm_endid_count(fsm, s); if (count > 0) { - enum fsm_getendids_res res; fsm_end_id_t *ids; size_t written; + int res; ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); if (ids == NULL) { @@ -313,16 +313,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) } res = fsm_endid_get(fsm, s, count, ids, &written); - switch (res) { - case FSM_GETENDIDS_NOT_FOUND: - case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: - assert(!"unreached"); - abort(); - - case FSM_GETENDIDS_FOUND: - break; - } - + assert(res == 1); assert(written == count); qsort(ids, count, sizeof *ids, comp_end_id); diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 7c53e3846..96e2f304e 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -469,7 +469,7 @@ make_ir(const struct fsm *fsm) ir->states[i].isend = fsm_isend(fsm, i); ir->states[i].end_ids = NULL; if (fsm_isend(fsm, i)) { - enum fsm_getendids_res res; + int res; size_t written; const size_t count = fsm_endid_count(fsm, i); struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, @@ -480,11 +480,10 @@ make_ir(const struct fsm *fsm) res = fsm_endid_get(fsm, i, count, ids->ids, &written); - if (res == FSM_GETENDIDS_FOUND) { + if (res == 1) { ids->count = (unsigned)written; ir->states[i].end_ids = ids; } else { - assert(res == FSM_GETENDIDS_NOT_FOUND); f_free(fsm->opt->alloc, ids); } } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 69615d287..e76da56e3 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -247,7 +247,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " \"endleaf\": [ "); for (i = 0; i < fsm->statecount; i++) { if (fsm_isend(fsm, i)) { - enum fsm_getendids_res res; + int res; size_t written; const size_t count = fsm_endid_count(fsm, i); struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, @@ -256,10 +256,9 @@ fsm_print_json(FILE *f, const struct fsm *fsm) res = fsm_endid_get(fsm, i, count, ids->ids, &written); - if (res == FSM_GETENDIDS_FOUND) { + if (res == 1) { ids->count = (unsigned)written; } else { - assert(res == FSM_GETENDIDS_NOT_FOUND); ids->count = 0; } diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index b21883abd..d57407d23 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -276,8 +276,8 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, total_num_endids = num_a_endids + num_b_endids; if (total_num_endids > 0) { - fsm_end_id_t *endids= NULL; - enum fsm_getendids_res ret; + fsm_end_id_t *endids; + int ret; endids = calloc(total_num_endids, sizeof endids[0]); if (endids == NULL) { @@ -286,24 +286,16 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, if (num_a_endids > 0) { size_t nwritten = 0; - ret = fsm_endid_get(fsm_a, a, num_a_endids, &endids[0], &nwritten); - - if (ret != FSM_GETENDIDS_FOUND || nwritten != num_a_endids) { - free(endids); - errno = (ret != FSM_GETENDIDS_FOUND) ? ENOENT : EINVAL; - return NULL; - } + ret = fsm_endid_get(fsm_a, a, num_a_endids, endids, &nwritten); + assert(ret == 1); + assert(nwritten == num_a_endids); } if (num_b_endids > 0) { size_t nwritten = 0; - ret = fsm_endid_get(fsm_b, b, num_b_endids, &endids[num_a_endids], &nwritten); - - if (ret != FSM_GETENDIDS_FOUND || nwritten != num_b_endids) { - free(endids); - errno = (ret != FSM_GETENDIDS_FOUND) ? ENOENT : EINVAL; - return NULL; - } + ret = fsm_endid_get(fsm_b, b, num_b_endids, endids + num_a_endids, &nwritten); + assert(ret == 1); + assert(nwritten == num_b_endids); } { diff --git a/src/lx/ast.c b/src/lx/ast.c index 4d088cd75..50d780855 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -237,9 +237,9 @@ ast_getendmapping(const struct fsm *fsm, fsm_state_t s) size_t id_count; fsm_end_id_t *id_buf_dynamic = NULL; fsm_end_id_t id_buf[ID_STACK_BUF_COUNT]; - enum fsm_getendids_res res; size_t written; struct ast_mapping *m; + int res; id_count = fsm_endid_count(fsm, s); if (id_count > ID_STACK_BUF_COUNT) { @@ -253,18 +253,10 @@ ast_getendmapping(const struct fsm *fsm, fsm_state_t s) s, id_count, id_buf_dynamic == NULL ? id_buf : id_buf_dynamic, &written); - if (res == FSM_GETENDIDS_NOT_FOUND) { - if (id_buf_dynamic != NULL) { - free(id_buf_dynamic); - } - return NULL; - } /* Should always have an appropriately sized buffer, * or fail to allocate above */ - assert(res != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); - - assert(res == FSM_GETENDIDS_FOUND); + assert(res == 1); assert(written == id_count); assert(written > 0); diff --git a/src/re/main.c b/src/re/main.c index 592e368c7..895b832ea 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -30,6 +30,8 @@ #include +#include + #include "libfsm/internal.h" /* XXX */ #include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ @@ -285,28 +287,31 @@ find_match_with_id(fsm_end_id_t id) static struct match * find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) { -#define MAX_END_IDS 8 /* FIXME: what is reasonable here? */ - fsm_end_id_t end_id_buf[MAX_END_IDS]; - size_t end_ids_written; - enum fsm_getendids_res res; + fsm_end_id_t *ids, id; + size_t count; + size_t nwritten; + int res; if (!fsm_isend(dfa, s)) { return NULL; } - res = fsm_endid_get(dfa, s, MAX_END_IDS, - end_id_buf, &end_ids_written); - if (res == FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE) { - fprintf(stderr, "Error: Multiple end IDs\n"); - return NULL; - } else if (res == FSM_GETENDIDS_NOT_FOUND) { + count = fsm_endid_count(dfa, s); + if (count == 0) { return NULL; - } else { - assert(res == FSM_GETENDIDS_FOUND); - /* continues below */ } - return find_match_with_id(end_id_buf[0]); + ids = xmalloc(count * sizeof *ids); + + res = fsm_endid_get(dfa, s, count, ids, &nwritten); + assert(res == 1); + assert(nwritten == count); + + id = ids[0]; + + free(ids); + + return find_match_with_id(id); } static struct match * @@ -337,10 +342,7 @@ addmatch(struct match **head, int i, const char *s) } } - new = malloc(sizeof *new); - if (new == NULL) { - return NULL; - } + new = xmalloc(sizeof *new); new->i = i; new->s = s; @@ -360,11 +362,7 @@ add_matches_list(struct match *head) return; } - new = malloc(sizeof *new); - if (new == NULL) { - perror("allocating matches list"); - abort(); - } + new = xmalloc(sizeof *new); new->next = all_matches; new->head = head; diff --git a/tests/capture/captest.c b/tests/capture/captest.c index b2bc1243f..5a6384686 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -90,14 +90,14 @@ captest_run_single(const struct captest_single_fsm_test_info *info) { fsm_end_id_t id_buf[1] = { ~0 }; - enum fsm_getendids_res gres; + int gres; size_t written; if (1 != fsm_endid_count(fsm, end)) { FAIL("did not have exactly one end ID"); } gres = fsm_endid_get(fsm, end, 1, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { + if (gres != 1) { FAIL("failed to get end IDs"); } @@ -182,7 +182,7 @@ captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, unsigned expected_end_id, const char **msg) { fsm_end_id_t id_buf[1] = { ~0 }; - enum fsm_getendids_res gres; + int gres; size_t written; const char *unused; @@ -196,7 +196,7 @@ captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, } gres = fsm_endid_get(fsm, end_state, 1, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { + if (gres != 1) { *msg = "failed to get end IDs"; return 0; } diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 1759afb05..4394b637f 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -220,11 +220,12 @@ check(const struct fsm *fsm, const char *string, assert(captures[cb_b].pos[1] == pb_1); { - enum fsm_getendids_res gres; + int gres; fsm_end_id_t id_buf[2]; size_t written; + gres = fsm_endid_get(fsm, end, 2, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { + if (gres != 1) { assert(!"fsm_getendids failed"); } diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index 4dd4b4ebe..69df62775 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -45,11 +45,11 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; size_t nwritten; - enum fsm_getendids_res ret; + int ret; assert( fsm_endid_count(fsm, state_ind) == 1); - nwritten = 0; + nwritten = 0; ret = fsm_endid_get( fsm, state_ind, @@ -57,9 +57,9 @@ int main(void) &endid, &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == 1); - assert( endid == 1 ); + assert(ret == 1); + assert(nwritten == 1); + assert(endid == 1); nend++; } diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index 834702bdc..f451b9acc 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -86,7 +86,7 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[sizeof all_end_ids / sizeof all_end_ids[0]]; size_t i, nwritten; - enum fsm_getendids_res ret; + int ret; memset(&endids[0], 0, sizeof endids); @@ -100,11 +100,11 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(ret == 1); + assert(nwritten == sizeof all_end_ids / sizeof all_end_ids[0]); - /* sort endids and compare */ - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); + /* sort endids and compare */ + qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); for (i=0; i < nwritten; i++) { assert(endids[i] == sorted_all_end_ids[i]); } diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 75a89db0d..5d61e1db4 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -41,7 +41,7 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; size_t nwritten; - enum fsm_getendids_res ret; + int ret; assert( fsm_endid_count(fsm, state_ind) == 1); @@ -53,9 +53,9 @@ int main(void) &endid, &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == 1); - assert( endid == 1 ); + assert(ret == 1); + assert(nwritten == 1); + assert( endid == 1 ); } } diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index d8df93543..91adf3f44 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -46,11 +46,11 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; size_t nwritten; - enum fsm_getendids_res ret; + int ret; - assert( fsm_endid_count(fsm, state_ind) == 1); + assert(fsm_endid_count(fsm, state_ind) == 1); - nwritten = 0; + nwritten = 0; ret = fsm_endid_get( fsm, state_ind, @@ -58,9 +58,9 @@ int main(void) &endid, &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == 1); - assert( endid == 1 ); + assert(ret == 1); + assert(nwritten == 1); + assert( endid == 1 ); } } diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index a81bce01c..905c4b324 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -57,7 +57,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(comb, state_ind); @@ -72,7 +72,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); if (nwritten == 1) { diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index d7beb8777..e292be47b 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -184,7 +184,7 @@ int main(void) int tested_pattern[NUM_PATTERNS]; fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; size_t nwritten, count, j; - enum fsm_getendids_res ret; + int ret; memset(&endids[0], 0, sizeof endids); @@ -200,7 +200,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); memset(&tested_pattern[0], 0, sizeof tested_pattern); @@ -251,7 +251,7 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; size_t nwritten, count; - enum fsm_getendids_res ret; + int ret; memset(&endids[0], 0, sizeof endids); @@ -267,7 +267,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); } } diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index c4b7da823..1b022aad4 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -69,7 +69,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(comb, state_ind); @@ -85,11 +85,11 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); + qsort(&endids[0], count, sizeof endids[0], cmp_endids); + assert(endids[0] == 1 && endids[1] == 2); } } diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index 91e39b5eb..3639ab0ac 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -70,7 +70,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(comb, state_ind); @@ -86,10 +86,10 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); - assert(endids[0] == 1); + assert(endids[0] == 1); } } diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index 2179d0a03..657a314dc 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -94,7 +94,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(fsm, state_ind); @@ -108,7 +108,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); info[ninfo].state = state_ind; @@ -138,7 +138,7 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; size_t nwritten, count, ind; - enum fsm_getendids_res ret; + int ret; assert( state_ind == info[info_ind].state ); @@ -155,7 +155,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); if (nwritten > 1) { diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index c695136cc..488e1e3e0 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -106,7 +106,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(fsm, state_ind); @@ -120,7 +120,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); info[ninfo].state = state_ind; @@ -151,7 +151,7 @@ int main(void) fsm_end_id_t endid = 0; fsm_end_id_t expected; size_t nwritten, count; - enum fsm_getendids_res ret; + int ret; assert( state_ind == info[info_ind].state ); @@ -166,7 +166,7 @@ int main(void) &endid, &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); if (info[info_ind].count == 2) { diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index 3da80d9bc..b4d0701d3 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -110,7 +110,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(fsm, state_ind); @@ -124,7 +124,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); info[ninfo].state = state_ind; @@ -155,7 +155,7 @@ int main(void) fsm_end_id_t endid = 0; fsm_end_id_t expected; size_t nwritten, count; - enum fsm_getendids_res ret; + int ret; assert( state_ind == info[info_ind].state ); @@ -170,7 +170,7 @@ int main(void) &endid, &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); if (info[info_ind].count == 2) { diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index 0be5c244d..0ef6e024b 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -75,7 +75,7 @@ int main(void) fsm_end_id_t endids[3] = {0,0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(comb, state_ind); @@ -91,11 +91,11 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2 && endids[2] == 4); + qsort(&endids[0], count, sizeof endids[0], cmp_endids); + assert(endids[0] == 1 && endids[1] == 2 && endids[2] == 4); } } diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index c6c467328..ef15efe13 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -74,7 +74,7 @@ int main(void) fsm_end_id_t endids[2] = {0,0}; size_t nwritten; size_t count; - enum fsm_getendids_res ret; + int ret; nwritten = 0; count = fsm_endid_count(fsm, state_ind); @@ -88,7 +88,7 @@ int main(void) &endids[0], &nwritten); - assert(ret == FSM_GETENDIDS_FOUND); + assert(ret == 1); assert(nwritten == count); info[ninfo].state = state_ind; diff --git a/tests/endids/utils.c b/tests/endids/utils.c index b6dd23042..7457fc70e 100644 --- a/tests/endids/utils.c +++ b/tests/endids/utils.c @@ -19,7 +19,7 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end count = fsm_endid_count(fsm, end); if (count > 0) { - enum fsm_getendids_res ret; + int ret; size_t nwritten = 0; fsm_end_id_t *endids = calloc(count, sizeof *endids); @@ -28,9 +28,9 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end } ret = fsm_endid_get(fsm, end, count, endids, &nwritten); - if (ret != FSM_GETENDIDS_FOUND) { + if (ret == 0) { free(endids); - errno = (ret == FSM_GETENDIDS_NOT_FOUND) ? EINVAL : ENOMEM; + errno = EINVAL; return -1; } diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index c99b2eeac..aff46ba3a 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -60,10 +60,10 @@ gtest_matches_cb(const struct fsm *fsm, #define ID_BUF_COUNT 1 fsm_end_id_t id_buf[ID_BUF_COUNT]; size_t written; - enum fsm_getendids_res gres = fsm_endid_get(fsm, + int gres = fsm_endid_get(fsm, end_state, ID_BUF_COUNT, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { + if (gres != 1) { fprintf(stderr, "ERROR: fsm_endid_get: returned %d\n", gres); diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 5b366c0ce..197415492 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -48,9 +48,9 @@ run_test(const char **strings) assert(res > 0); /* match */ size_t written; - enum fsm_getendids_res eres = fsm_endid_get(fsm, end, + int eres = fsm_endid_get(fsm, end, MAX_INPUTS, id_buf, &written); - assert(eres == FSM_GETENDIDS_FOUND); + assert(eres == 1); bool found = false; for (size_t i = 0; i < written; i++) { if (id_buf[i] == id) { From c27a1e44dedbb346e6e26565dc41347304f40de5 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 12 Jun 2024 05:56:33 -0700 Subject: [PATCH 141/373] Missing error handling. --- src/libfsm/print/api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 69bf76eed..4c5b31cc8 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -165,7 +165,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\tif (fsm_isend(fsm, s[%u])) {\n", end); for (size_t id = 0; id < count; id++) { - fprintf(f, "\t\tfsm_setendidstate(fsm, s[%u], %zu);\n", end, id); + fprintf(f, "\t\tif (!fsm_setendidstate(fsm, s[%u], %zu)) { return 0; }\n", end, id); } fprintf(f, "\t}\n"); fprintf(f, "\n"); From d981fdd0cc35e805894b13edbe32c1974228ecef Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 12 Jun 2024 06:28:58 -0700 Subject: [PATCH 142/373] No need for the tri-state enum here. --- include/fsm/fsm.h | 2 +- src/libfsm/clone.c | 2 +- src/libfsm/consolidate.c | 9 ++------ src/libfsm/endids.c | 44 ++++++++++++++++------------------------ src/libfsm/endids.h | 14 ++----------- src/libfsm/epsilons.c | 2 +- src/libfsm/libfsm.syms | 2 +- src/libfsm/merge.c | 9 ++------ src/libfsm/parser.act | 2 +- src/libfsm/parser.c | 2 +- src/libfsm/print/api.c | 2 +- src/libfsm/walk2.c | 28 +++++++++++-------------- src/libre/ac.c | 3 +-- 13 files changed, 43 insertions(+), 78 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 484541735..02f04c951 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -222,7 +222,7 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id); * Returns 1 on success, 0 on error. * */ int -fsm_setendidstate(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); +fsm_endid_set(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); /* Get the end IDs associated with an end state, if any. * If id_buf has enough cells to store all the end IDs (according diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 587976e94..69aedaa63 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -133,7 +133,7 @@ copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) fprintf(stderr, "clone[%d] <- %d\n", state, id); #endif - if (!fsm_setendidstate(env->dst, state, id)) { + if (!fsm_endid_set(env->dst, state, id)) { env->ok = 0; return 0; } diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 4518d3926..a000a4c7b 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -231,19 +231,14 @@ static int consolidate_end_ids_cb(fsm_state_t state, const fsm_end_id_t *ids, size_t num_ids, void *opaque) { struct consolidate_end_ids_env *env = opaque; - enum fsm_endid_set_res sres; fsm_state_t s; assert(env->tag == 'C'); assert(state < env->mapping_count); s = env->mapping[state]; - sres = fsm_endid_set_bulk(env->dst, s, num_ids, ids, FSM_ENDID_BULK_APPEND); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - return 0; - } - - return 1; + return fsm_endid_set_bulk(env->dst, s, + num_ids, ids, FSM_ENDID_BULK_APPEND); } static int diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 80fcbfd8d..fb63baf66 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -73,7 +73,7 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) fprintf(stderr, "fsm_setendid: setting id %u on state %d\n", id, i); #endif - if (!fsm_setendidstate(fsm, i, id)) { + if (!fsm_endid_set(fsm, i, id)) { return 0; } } @@ -82,16 +82,6 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) return 1; } -int -fsm_setendidstate(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id) -{ - enum fsm_endid_set_res sres = fsm_endid_set(fsm, end_state, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - return 0; - } - return 1; -} - int fsm_endid_init(struct fsm *fsm) { @@ -377,7 +367,7 @@ allocate_ids(const struct fsm *fsm, struct end_info_ids *prev, size_t n) return f_realloc(fsm->opt->alloc, prev, id_alloc_size); } -enum fsm_endid_set_res +int fsm_endid_set(struct fsm *fsm, fsm_state_t state, fsm_end_id_t id) { @@ -389,7 +379,7 @@ fsm_endid_set(struct fsm *fsm, struct endid_info_bucket *b = endid_find_bucket(fsm, state); if (b == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } LOG_2("fsm_endid_set: state %d, bucket %p\n", state, (void *)b); @@ -399,7 +389,7 @@ fsm_endid_set(struct fsm *fsm, ids = allocate_ids(fsm, NULL, DEF_BUCKET_ID_COUNT); if (ids == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } ids->ids[0] = id; @@ -410,7 +400,7 @@ fsm_endid_set(struct fsm *fsm, b->ids = ids; ei->buckets_used++; - return FSM_ENDID_SET_ADDED; + return 1; } else if (b->state == state) { size_t ind; @@ -427,7 +417,7 @@ fsm_endid_set(struct fsm *fsm, nids = allocate_ids(fsm, b->ids, nceil); if (nids == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; /* alloc fail */ + return 0; /* alloc fail */ } nids->ceil = nceil; b->ids = nids; @@ -448,7 +438,7 @@ fsm_endid_set(struct fsm *fsm, } else if (b->ids->ids[ind] == id) { /* already present, our work is done! */ LOG_2("fsm_endid_set: already present, skipping\n"); - return FSM_ENDID_SET_ALREADY_PRESENT; + return 1; } else { /* need to shift items up to make room for id */ memmove(&b->ids->ids[ind+1], &b->ids->ids[ind], @@ -464,10 +454,10 @@ fsm_endid_set(struct fsm *fsm, (void)dump_buckets; DBG_3(dump_buckets("set_dump", ei)); - return FSM_ENDID_SET_ADDED; + return 1; } else { assert(!"unreachable: endid_find_bucket failed"); - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } } @@ -488,7 +478,7 @@ cmp_endids(const void *pa, const void *pb) return 0; } -enum fsm_endid_set_res +int fsm_endid_set_bulk(struct fsm *fsm, fsm_state_t state, size_t num_ids, const fsm_end_id_t *ids, enum fsm_endid_bulk_op op) { @@ -502,7 +492,7 @@ fsm_endid_set_bulk(struct fsm *fsm, b = endid_find_bucket(fsm, state); if (b == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } if (b->state == state) { @@ -525,12 +515,12 @@ fsm_endid_set_bulk(struct fsm *fsm, if (new_ceil < total_count) { /* num_ids is too large? */ - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } new_ids = allocate_ids(fsm, b->ids, new_ceil); if (new_ids == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } b->ids = new_ids; @@ -560,12 +550,12 @@ fsm_endid_set_bulk(struct fsm *fsm, if (n < num_ids) { /* num_ids is too large? */ - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } new_ids = allocate_ids(fsm, NULL, n); if (new_ids == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } memcpy(&new_ids->ids[0], &ids[0], num_ids * sizeof ids[0]); @@ -602,7 +592,7 @@ fsm_endid_set_bulk(struct fsm *fsm, b->ids->count = j; } - return FSM_ENDID_SET_ADDED; + return 1; } int @@ -778,7 +768,7 @@ carry_iter_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) (void)state; - if (!fsm_setendidstate(env->dst, env->dst_state, id)) { + if (!fsm_endid_set(env->dst, env->dst_state, id)) { env->ok = 0; return 0; } diff --git a/src/libfsm/endids.h b/src/libfsm/endids.h index c4b3184b5..a5697f10b 100644 --- a/src/libfsm/endids.h +++ b/src/libfsm/endids.h @@ -10,15 +10,6 @@ fsm_endid_init(struct fsm *fsm); void fsm_endid_free(struct fsm *fsm); -enum fsm_endid_set_res { - FSM_ENDID_SET_ADDED, - FSM_ENDID_SET_ALREADY_PRESENT, - FSM_ENDID_SET_ERROR_ALLOC_FAIL = -1 -}; -enum fsm_endid_set_res -fsm_endid_set(struct fsm *fsm, - fsm_state_t state, fsm_end_id_t id); - /* Sets end ids in a bulk operation. * * Repeatedly calling fsm_endid_set will end up with quadratic behavior. This @@ -34,14 +25,13 @@ fsm_endid_set(struct fsm *fsm, * * The caller maintains ownership of ids, and must free it if needed. * - * Only FSM_ENDID_SET_ADDED and FSM_ENDID_SET_ERROR_ALLOC_FAIL - * are returned. + * Returns 1 on success, 0 on failure. */ enum fsm_endid_bulk_op { FSM_ENDID_BULK_REPLACE = 0, FSM_ENDID_BULK_APPEND = 1, }; -enum fsm_endid_set_res +int fsm_endid_set_bulk(struct fsm *fsm, fsm_state_t state, size_t num_ids, const fsm_end_id_t *ids, enum fsm_endid_bulk_op op); diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 82dd6cfa4..f8bb1d9d3 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -405,7 +405,7 @@ carry_endids(struct fsm *fsm, struct state_set *states, /* add them */ for (i = 0; i < env.count; i++) { - if (!fsm_setendidstate(fsm, dst_state, env.ids[i])) { + if (!fsm_endid_set(fsm, dst_state, env.ids[i])) { env.ok = 0; goto cleanup; } diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 0e40ae884..2801f3703 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -88,10 +88,10 @@ fsm_setend fsm_endid_count fsm_endid_get +fsm_endid_set fsm_setendid fsm_mapendids fsm_increndids -fsm_setendidstate fsm_countedges fsm_countstates diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index fd4d35ee5..69834a5bd 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -163,7 +163,6 @@ struct copy_end_ids_env { static int copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t *ids, size_t num_ids, void *opaque) { - enum fsm_endid_set_res sres; struct copy_end_ids_env *env = opaque; assert(env->tag == 'M'); @@ -172,12 +171,8 @@ copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t *ids, size_t num_ids, void state + env->base_src, id); #endif - sres = fsm_endid_set_bulk(env->dst, state + env->base_src, num_ids, ids, FSM_ENDID_BULK_REPLACE); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - return 0; - } - - return 1; + return fsm_endid_set_bulk(env->dst, state + env->base_src, + num_ids, ids, FSM_ENDID_BULK_REPLACE); } static int diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index 30caa03bd..71ee0a89c 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -340,7 +340,7 @@ @}; : (s :state, id :endid) -> () = @{ - if (!fsm_setendidstate(fsm, @s, @id)) { + if (!fsm_endid_set(fsm, @s, @id)) { @!; } @}; diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 0ae7ee763..442449a44 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -1335,7 +1335,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) { #line 343 "src/libfsm/parser.act" - if (!fsm_setendidstate(fsm, (ZIs), (ZIid))) { + if (!fsm_endid_set(fsm, (ZIs), (ZIid))) { goto ZL1; } diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 4c5b31cc8..6b33d045f 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -165,7 +165,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\tif (fsm_isend(fsm, s[%u])) {\n", end); for (size_t id = 0; id < count; id++) { - fprintf(f, "\t\tif (!fsm_setendidstate(fsm, s[%u], %zu)) { return 0; }\n", end, id); + fprintf(f, "\t\tif (!fsm_endid_set(fsm, s[%u], %zu)) { return 0; }\n", end, id); } fprintf(f, "\t}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index d57407d23..9320dbafd 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -298,24 +298,20 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, assert(nwritten == num_b_endids); } - { - enum fsm_endid_set_res ret; - - ret = fsm_endid_set_bulk( - data->new, - p->comb, - total_num_endids, - &endids[0], - FSM_ENDID_BULK_REPLACE); - if (ret == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - int errsv = errno; - free(endids); - errno = errsv; - return NULL; - } - } + ret = fsm_endid_set_bulk( + data->new, + p->comb, + total_num_endids, + &endids[0], + FSM_ENDID_BULK_REPLACE); free(endids); + + if (!ret) { + int errsv = errno; + errno = errsv; + return NULL; + } } } diff --git a/src/libre/ac.c b/src/libre/ac.c index efe13df36..e9412602e 100644 --- a/src/libre/ac.c +++ b/src/libre/ac.c @@ -332,8 +332,7 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, fsm_state_t state; state_set_reset(ts->endids, &si); while (state_set_next(&si, &state)) { - fsm_end_id_t endid = (fsm_end_id_t)state; - if (!fsm_setendidstate(fsm, st, endid)) { + if (!fsm_endid_set(fsm, st, (fsm_end_id_t) state)) { return 0; } } From 05b246718c2ee56b874a135559b281b2f944e28c Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 12 Jun 2024 07:48:09 -0700 Subject: [PATCH 143/373] No need for the static buffer here. --- src/lx/ast.c | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/src/lx/ast.c b/src/lx/ast.c index 50d780855..7dd1ad72c 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -9,8 +9,10 @@ #include #include #include +#include #include +#include #include @@ -233,42 +235,42 @@ ast_getendmappingbyendid(fsm_end_id_t id) struct ast_mapping * ast_getendmapping(const struct fsm *fsm, fsm_state_t s) { - #define ID_STACK_BUF_COUNT 4 - size_t id_count; - fsm_end_id_t *id_buf_dynamic = NULL; - fsm_end_id_t id_buf[ID_STACK_BUF_COUNT]; - size_t written; + fsm_end_id_t *ids; + size_t count, written; struct ast_mapping *m; int res; - id_count = fsm_endid_count(fsm, s); - if (id_count > ID_STACK_BUF_COUNT) { - id_buf_dynamic = malloc(id_count * sizeof(id_buf_dynamic[0])); - if (id_buf_dynamic == NULL) { - return NULL; - } + if (!fsm_isend(fsm, s)) { + errno = EINVAL; + return NULL; + } + + count = fsm_endid_count(fsm, s); + if (count == 0) { + errno = EINVAL; + return NULL; } - res = fsm_endid_get(fsm, - s, id_count, - id_buf_dynamic == NULL ? id_buf : id_buf_dynamic, - &written); + ids = malloc(count * sizeof *ids); + if (ids == NULL) { + return NULL; + } + + res = fsm_endid_get(fsm, s, count, ids, &written); - /* Should always have an appropriately sized buffer, - * or fail to allocate above */ assert(res == 1); - assert(written == id_count); + assert(written == count); assert(written > 0); - m = ast_getendmappingbyendid(id_buf[0]); + m = ast_getendmappingbyendid(ids[0]); if (LOG()) { fprintf(stderr, "ast_getendmapping: got mapping %p mappings[%d]\n", - (void *)m, id_buf[0]); + (void *) m, ids[0]); } - if (id_buf_dynamic != NULL) { - free(id_buf_dynamic); - } + free(ids); + return m; } + From 1c88c759249471a712eabcd9cf7d8e780605f3ba Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 14 Jun 2024 13:03:23 +0100 Subject: [PATCH 144/373] Statements after declarations for generated code. This keeps the generated code C89-compliant. --- src/libfsm/print/c.c | 9 ++++++--- src/libfsm/print/vmc.c | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index dbca1d3fa..fcc036633 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -427,31 +427,34 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, case FSM_IO_GETC: fprintf(f, "(int (*fsm_getc)(void *opaque), void *opaque)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void)opaque;\n"); if (ir->n > 0) { fprintf(f, "\tint c;\n"); fprintf(f, "\n"); } + fprintf(f, "\t(void) opaque;\n"); + fprintf(f, "\n"); break; case FSM_IO_STR: fprintf(f, "(const char *s, void *opaque)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void)opaque;\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); } + fprintf(f, "\t(void) opaque;\n"); + fprintf(f, "\n"); break; case FSM_IO_PAIR: fprintf(f, "(const char *b, const char *e, void *opaque)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void)opaque;\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); } + fprintf(f, "\t(void) opaque;\n"); + fprintf(f, "\n"); break; } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 911619a7f..1e60a9b50 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -474,13 +474,15 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt case FSM_IO_STR: fprintf(f, "(const char *s, void *opaque)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void)opaque;\n"); + fprintf(f, "\t(void) opaque;\n"); + fprintf(f, "\n"); break; case FSM_IO_PAIR: fprintf(f, "(const char *b, const char *e, void *opaque)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void)opaque;\n"); + fprintf(f, "\t(void) opaque;\n"); + fprintf(f, "\n"); break; } From 0620f5528f67b889be1d091b60fa56d36b2e08ce Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 14 Jun 2024 15:42:38 +0100 Subject: [PATCH 145/373] No need to mark unused here, this opaque pointer is passed to the getc callback. --- src/libfsm/print/c.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index fcc036633..5de4ce4ee 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -431,8 +431,6 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, fprintf(f, "\tint c;\n"); fprintf(f, "\n"); } - fprintf(f, "\t(void) opaque;\n"); - fprintf(f, "\n"); break; case FSM_IO_STR: From e577435cddabf684c2b3e1809d179b1124c6b218 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 14 Jun 2024 18:25:27 +0100 Subject: [PATCH 146/373] No need for the `void *opaque` arguments here. If these were neccessary for caller-supplied generated code from the endleaf callbacks, the actual way I'm intending a caller to pass information there, is to output with `opt.fragment` set. Then there's no need to gi via `void *` for any caller state, that can all be exposed as appropriately typed variables in scope for the generated code (or wrapped in a similar function as the non-fragment code, and exposed as appropriately-typed arguments to that function). The existing `void *opaque` here was nothing to do with this, it was actually for the `.getc()` callback. I've renamed it to keep it from getting mixed up. --- src/libfsm/print/c.c | 12 ++++-------- src/libfsm/print/vmc.c | 12 ++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 5de4ce4ee..6ae17486c 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -377,7 +377,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "\twhile (c = fsm_getc(opaque), c != EOF) {\n"); + fprintf(f, "\twhile (c = fsm_getc(getc_opaque), c != EOF) {\n"); break; case FSM_IO_STR: @@ -425,7 +425,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "(int (*fsm_getc)(void *opaque), void *opaque)\n"); + fprintf(f, "(int (*fsm_getc)(void *getc_opaque), void *getc_opaque)\n"); fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tint c;\n"); @@ -434,25 +434,21 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, break; case FSM_IO_STR: - fprintf(f, "(const char *s, void *opaque)\n"); + fprintf(f, "(const char *s)\n"); fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); } - fprintf(f, "\t(void) opaque;\n"); - fprintf(f, "\n"); break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e, void *opaque)\n"); + fprintf(f, "(const char *b, const char *e)\n"); fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); } - fprintf(f, "\t(void) opaque;\n"); - fprintf(f, "\n"); break; } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 1e60a9b50..da6927acd 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -121,7 +121,7 @@ print_fetch(FILE *f, const struct fsm_options *opt) * Per its API, fsm_getc() is expected to return a positive character * value (as if cast via unsigned char), or EOF. Just like fgetc() does. */ - fprintf(f, "if (c = fsm_getc(opaque), c == EOF) "); + fprintf(f, "if (c = fsm_getc(getc_opaque), c == EOF) "); break; case FSM_IO_STR: @@ -467,22 +467,18 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "(int (*fsm_getc)(void *opaque), void *opaque)\n"); + fprintf(f, "(int (*fsm_getc)(void *opaque), void *getc_opaque)\n"); fprintf(f, "{\n"); break; case FSM_IO_STR: - fprintf(f, "(const char *s, void *opaque)\n"); + fprintf(f, "(const char *s)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void) opaque;\n"); - fprintf(f, "\n"); break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e, void *opaque)\n"); + fprintf(f, "(const char *b, const char *e)\n"); fprintf(f, "{\n"); - fprintf(f, "\t(void) opaque;\n"); - fprintf(f, "\n"); break; } From 6ec0aef060a27fcfafb7b9b50043c6affee4cd0b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 14 Jun 2024 18:56:24 +0100 Subject: [PATCH 147/373] No need for the end_ids struct here. I'm not removing this for any profound reason, just that fewer kinds of structs help me better see what's happening. --- include/fsm/fsm.h | 6 --- include/fsm/options.h | 4 +- src/libfsm/print/awk.c | 22 ++++++--- src/libfsm/print/c.c | 18 +++++--- src/libfsm/print/dot.c | 29 ++++++------ src/libfsm/print/go.c | 16 +++++-- src/libfsm/print/ir.c | 40 ++++++++++------- src/libfsm/print/ir.h | 5 ++- src/libfsm/print/irdot.c | 8 ++-- src/libfsm/print/irjson.c | 8 ++-- src/libfsm/print/json.c | 25 +++++------ src/libfsm/print/rust.c | 17 +++++-- src/libfsm/print/sh.c | 11 ++++- src/libfsm/print/vmasm.c | 6 ++- src/libfsm/print/vmc.c | 16 +++++-- src/libfsm/print/vmdot.c | 19 +++++--- src/libfsm/print/vmops.c | 11 +++-- src/lx/print/c.c | 8 ++-- src/lx/print/dot.c | 8 ++-- src/re/main.c | 94 +++++++++++++++++++-------------------- 20 files changed, 214 insertions(+), 157 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 02f04c951..ecc7c5937 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -37,12 +37,6 @@ typedef unsigned int fsm_end_id_t; #define FSM_END_ID_MAX UINT_MAX -/* struct used to return a collection of end IDs. */ -struct fsm_end_ids { - unsigned count; - fsm_end_id_t ids[1]; -}; - /* * Create a new FSM. This is to be freed with fsm_free(). A structure allocated * from fsm_new() is expected to be passed as the "fsm" argument to the diff --git a/include/fsm/options.h b/include/fsm/options.h index b4193cc78..759b3b8eb 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -65,12 +65,12 @@ struct fsm_options { const char *cp; /* TODO: explain. for C code fragment output */ - int (*leaf)(FILE *, const struct fsm_end_ids *ids, + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); void *leaf_opaque; /* TODO: explain. for C code fragment output */ - int (*endleaf)(FILE *, const struct fsm_end_ids *ids, + int (*endleaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque); void *endleaf_opaque; diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index c7083e6e2..462a3ab7f 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -32,13 +32,14 @@ #define START UINT32_MAX static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); (void) leaf_opaque; (void) ids; + (void) count; /* XXX: this should be FSM_UNKNOWN or something non-EOF, * maybe user defined */ @@ -99,7 +100,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { return -1; } } else { @@ -110,10 +114,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, /* awk can't return an array */ fprintf(f, "return \""); - for (i = 0; i < op->ir_state->end_ids->count; i++) { - fprintf(f, "%u", op->ir_state->end_ids->ids[i]); + for (i = 0; i < op->ir_state->endids.count; i++) { + fprintf(f, "%u", op->ir_state->endids.ids[i]); - if (i < op->ir_state->end_ids->count - 1) { + if (i < op->ir_state->endids.count - 1) { fprintf(f, ","); } } @@ -142,14 +146,18 @@ print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) static int fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, const char *prefix, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(ir != NULL); diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 6ae17486c..21f15f837 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -53,12 +53,13 @@ ir_hasend(const struct ir *ir) } static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); (void) ids; + (void) count; (void) leaf_opaque; /* XXX: this should be FSM_UNKNOWN or something non-EOF, @@ -136,7 +137,7 @@ static int print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, struct ir_state *cs, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { assert(ir != NULL); @@ -149,7 +150,7 @@ print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); - if (-1 == leaf(f, cs->end_ids, leaf_opaque)) { + if (-1 == leaf(f, cs->endids.ids, cs->endids.count, leaf_opaque)) { return -1; } fprintf(f, "\n"); @@ -178,7 +179,7 @@ print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_groups(f, opt, ir_indexof(ir, cs), cs->u.partial.groups, cs->u.partial.n); fprintf(f, "\t\t\tdefault: "); - if (-1 == leaf(f, cs->end_ids, leaf_opaque)) { + if (-1 == leaf(f, cs->endids.ids, cs->endids.count, leaf_opaque)) { return -1; } fprintf(f, "\n"); @@ -209,7 +210,7 @@ print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_ranges(f, opt, cs->u.error.error.ranges, cs->u.error.error.n); fprintf(f, " "); - if (-1 == leaf(f, cs->end_ids, leaf_opaque)) { + if (-1 == leaf(f, cs->endids.ids, cs->endids.count, leaf_opaque)) { return -1; } fprintf(f, "\n"); @@ -289,7 +290,10 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) fprintf(f, "\tcase S%u: ", i); if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, ir->states[i].end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + ir->states[i].endids.ids, ir->states[i].endids.count, + opt->endleaf_opaque)) + { return -1; } } else { @@ -306,7 +310,7 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) int fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { unsigned i; diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 1f070b382..459584e63 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -167,23 +167,22 @@ print_dotfrag(FILE *f, const struct fsm *fsm) s, s); } } else { - size_t written; - struct fsm_end_ids *ids = NULL; - const size_t count = fsm_endid_count(fsm, s); + fsm_end_id_t *ids; + size_t count, written; + + ids = NULL; + count = fsm_endid_count(fsm, s); if (count > 0) { int res; - ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); - assert(ids != NULL); + ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } - res = fsm_endid_get(fsm, s, count, ids->ids, &written); - if (res == 1) { - ids->count = (unsigned)written; - } + res = fsm_endid_get(fsm, s, count, ids, &written); + assert(res == 1); + assert(count == written); } fprintf(f, "\t%sS%-2u [ shape = doublecircle", @@ -191,7 +190,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) if (fsm->opt->endleaf != NULL) { fprintf(f, ", "); - if (-1 == fsm->opt->endleaf(f, ids, fsm->opt->endleaf_opaque)) { + if (-1 == fsm->opt->endleaf(f, ids, count, fsm->opt->endleaf_opaque)) { return -1; } } else { @@ -204,15 +203,15 @@ print_dotfrag(FILE *f, const struct fsm *fsm) if (!fsm->opt->anonymous_states) { fprintf(f, "%u", s); - if (ids->count > 0) { + if (count > 0) { fprintf(f, "
"); } } - for (size_t i = 0; i < ids->count; i++) { - fprintf(f, "#%u", ids->ids[i]); + for (size_t i = 0; i < count; i++) { + fprintf(f, "#%u", ids[i]); - if (i < ids->count - 1) { + if (i < count - 1) { fprintf(f, ","); } } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index d0a4edc47..22ed45ef5 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -29,12 +29,13 @@ #include "ir.h" static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); (void) ids; + (void) count; (void) leaf_opaque; /* XXX: this should be FSM_UNKNOWN or something non-EOF, @@ -98,7 +99,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { return -1; } } else { @@ -132,14 +136,18 @@ print_fetch(FILE *f, const struct fsm_options *opt) static int fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(ir != NULL); diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 96e2f304e..4af6a8c11 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -467,25 +467,32 @@ make_ir(const struct fsm *fsm) assert(i < ir->n); ir->states[i].isend = fsm_isend(fsm, i); - ir->states[i].end_ids = NULL; + ir->states[i].endids.ids = NULL; + ir->states[i].endids.count = 0; + if (fsm_isend(fsm, i)) { - int res; - size_t written; - const size_t count = fsm_endid_count(fsm, i); - struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + (count == 0 ? 0 : (count - 1) * sizeof(ids->ids[0]))); - if (ids == NULL) { - goto error; - } + fsm_end_id_t *ids; + size_t count; - res = fsm_endid_get(fsm, i, count, - ids->ids, &written); - if (res == 1) { - ids->count = (unsigned)written; - ir->states[i].end_ids = ids; + count = fsm_endid_count(fsm, i); + if (count == 0) { + ids = NULL; } else { - f_free(fsm->opt->alloc, ids); + int res; + size_t written; + + ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + if (ids == NULL) { + goto error; + } + + res = fsm_endid_get(fsm, i, count, ids, &written); + assert(res == 1); + assert(count = written); } + + ir->states[i].endids.ids = ids; + ir->states[i].endids.count = count; } if (make_state(fsm, i, &ir->states[i]) == -1) { @@ -592,8 +599,7 @@ free_ir(const struct fsm *fsm, struct ir *ir) for (i = 0; i < ir->n; i++) { f_free(fsm->opt->alloc, (void *) ir->states[i].example); - - f_free(fsm->opt->alloc, (void *) ir->states[i].end_ids); + f_free(fsm->opt->alloc, (void *) ir->states[i].endids.ids); switch (ir->states[i].strategy) { case IR_TABLE: diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index f9f5c40c3..05ad758e7 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -53,7 +53,10 @@ struct ir_state { const char *example; unsigned int isend:1; - struct fsm_end_ids *end_ids; /* NULL -> 0 */ + struct { + fsm_end_id_t *ids; /* NULL -> 0 */ + size_t count; + } endids; enum ir_strategy strategy; union { diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index d388aa88e..039b9d568 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -191,13 +191,13 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\t S%u%s\n", ir_indexof(ir, cs), strategy_name(cs->strategy)); - if (cs->isend && cs->end_ids != NULL) { + if (cs->isend && cs->endids.count > 0) { fprintf(f, "\t\t end id"); - for (size_t i = 0; i < cs->end_ids->count; i++) { - fprintf(f, "#%u", cs->end_ids->ids[i]); + for (size_t i = 0; i < cs->endids.count; i++) { + fprintf(f, "#%u", cs->endids.ids[i]); - if (i < cs->end_ids->count - 1) { + if (i < cs->endids.count - 1) { fprintf(f, " "); } } diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index a1e154942..0de7459ae 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -123,12 +123,12 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\t{\n"); fprintf(f, "\t\t\t\"end\": %s,\n", cs->isend ? "true" : "false"); - if (cs->isend && cs->end_ids != NULL) { + if (cs->isend && cs->endids.count > 0) { fprintf(f, "\t\t\t\"end_id\": ["); - for (size_t i = 0; i < cs->end_ids->count; i++) { - fprintf(f, "%u", cs->end_ids->ids[i]); + for (size_t i = 0; i < cs->endids.count; i++) { + fprintf(f, "%u", cs->endids.ids[i]); - if (i < cs->end_ids->count - 1) { + if (i < cs->endids.count - 1) { fprintf(f, ", "); } } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index e76da56e3..8eb1a32e8 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -247,28 +247,27 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " \"endleaf\": [ "); for (i = 0; i < fsm->statecount; i++) { if (fsm_isend(fsm, i)) { + fsm_end_id_t *ids; + size_t count, written; int res; - size_t written; - const size_t count = fsm_endid_count(fsm, i); - struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + ((count - 1) * sizeof(ids->ids[0]))); - assert(ids != NULL); - - res = fsm_endid_get(fsm, i, count, - ids->ids, &written); - if (res == 1) { - ids->count = (unsigned)written; - } else { - ids->count = 0; + + count = fsm_endid_count(fsm, i); + + ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; } + res = fsm_endid_get(fsm, i, count, ids, &written); + assert(res == 1); + if (notfirst) { fprintf(f, ", "); } fprintf(f, "{ %u, ", i); - fsm->opt->endleaf(f, ids, fsm->opt->endleaf_opaque); + fsm->opt->endleaf(f, ids, count, fsm->opt->endleaf_opaque); fprintf(f, " }"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index e249c6aad..af2b0f302 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -32,12 +32,13 @@ #define START UINT32_MAX static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); (void) ids; + (void) count; (void) leaf_opaque; /* XXX: this should be FSM_UNKNOWN or something non-EOF, @@ -109,7 +110,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { return -1; } } else { @@ -144,7 +148,7 @@ static int fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, const struct ir *ir, const struct fsm_options *opt, const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { struct dfavm_op_ir *op; @@ -362,9 +366,14 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix, const char *cp) { static const struct dfavm_assembler_ir zero; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; struct dfavm_assembler_ir a; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; + assert(f != NULL); assert(ir != NULL); assert(opt != NULL); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 7b6bee8d2..04e84663e 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -127,7 +127,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { return -1; } } else { @@ -156,7 +159,11 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(ir != NULL); diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 8e3085b86..0c81bbf09 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -372,7 +372,11 @@ print_vmasm_encoding(FILE *f, const struct fsm *fsm, enum asm_dialect dialect) static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(fsm != NULL); diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index da6927acd..f94cee60d 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -29,12 +29,13 @@ #include "ir.h" static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); (void) ids; + (void) count; (void) leaf_opaque; /* XXX: this should be FSM_UNKNOWN or something non-EOF, @@ -96,7 +97,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { return -1; } } else { @@ -279,14 +283,18 @@ walk_sequence(struct dfavm_op_ir *op, static int fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(ir != NULL); diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 775714798..bdcb2e3de 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -92,19 +92,22 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { return -1; } } else { fprintf(f, "ret %td", op->ir_state - ir->states); - if (op->ir_state->end_ids != NULL) { + if (op->ir_state->endids.count > 0) { fprintf(f, " / "); - for (size_t i = 0; i < op->ir_state->end_ids->count; i++) { - fprintf(f, "#%u", op->ir_state->end_ids->ids[i]); + for (size_t i = 0; i < op->ir_state->endids.count; i++) { + fprintf(f, "#%u", op->ir_state->endids.ids[i]); - if (i < op->ir_state->end_ids->count - 1) { + if (i < op->ir_state->endids.count - 1) { fprintf(f, " "); } } @@ -300,7 +303,11 @@ fsm_print_vmdotfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(ir != NULL); diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 7804d3cd9..976b47b00 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -35,12 +35,13 @@ enum vmops_dialect { }; static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); (void) ids; + (void) count; (void) leaf_opaque; /* XXX: this should be FSM_UNKNOWN or something non-EOF, @@ -137,14 +138,18 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) /* TODO: eventually to be non-static */ static int fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; assert(f != NULL); assert(ir != NULL); diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 6308ba0c1..a8ae67eec 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -29,7 +29,7 @@ int fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *opaque); static int @@ -138,7 +138,7 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, } static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { const struct ast *ast; const struct ast_mapping *m; @@ -150,8 +150,8 @@ leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) if (ids == NULL) { m = NULL; } else { - assert(ids->count > 0); - m = ast_getendmappingbyendid(ids->ids[0]); + assert(count > 0); + m = ast_getendmappingbyendid(ids[0]); } if (m == NULL) { diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index 4904c246c..5ad9d6212 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -65,7 +65,7 @@ mapping(FILE *f, const struct ast_mapping *m, const struct ast *ast) } static int -endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, +endleaf_dot(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque) { const struct ast_mapping *m; @@ -74,9 +74,9 @@ endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, assert(f != NULL); assert(endleaf_opaque != NULL); - (void)end_ids; /* TODO */ - assert(end_ids->count > 0); - m = ast_getendmappingbyendid(end_ids->ids[0]); + assert(ids != NULL); + assert(count > 0); + m = ast_getendmappingbyendid(ids[0]); ast = endleaf_opaque; diff --git a/src/re/main.c b/src/re/main.c index 895b832ea..9c54faa99 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -51,7 +51,7 @@ */ struct match { - fsm_end_id_t i; + fsm_end_id_t id; const char *s; struct match *next; }; @@ -276,7 +276,7 @@ find_match_with_id(fsm_end_id_t id) struct matches_list *res = all_matches; while (res != NULL) { struct match *m = res->head; - if (m->i == id) { + if (m->id == id) { return m; } res = res->next; @@ -315,14 +315,14 @@ find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) } static struct match * -addmatch(struct match **head, int i, const char *s) +addmatch(struct match **head, fsm_end_id_t id, const char *s) { struct match *new; assert(head != NULL); assert(s != NULL); - if ((1U << i) > INT_MAX) { + if ((1U << id) > INT_MAX) { fprintf(stderr, "Too many patterns for int bitmap\n"); exit(EXIT_FAILURE); } @@ -344,7 +344,7 @@ addmatch(struct match **head, int i, const char *s) new = xmalloc(sizeof *new); - new->i = i; + new->id = id; new->s = s; new->next = *head; @@ -414,12 +414,13 @@ printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) } static int -endleaf_c(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +endleaf_c(FILE *f, + const fsm_end_id_t *ids, size_t count, + const void *endleaf_opaque) { const struct match *m; - int n; - size_t i, end_ids_count; + unsigned n; + size_t i; assert(endleaf_opaque == NULL); @@ -428,11 +429,9 @@ endleaf_c(FILE *f, const struct fsm_end_ids *end_ids, n = 0; - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); - - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - n |= 1 << m->i; + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + n |= 1U << m->id; } } @@ -440,11 +439,11 @@ endleaf_c(FILE *f, const struct fsm_end_ids *end_ids, fprintf(f, " /* "); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ - if (m->next != NULL || i < end_ids_count - 1) { + if (m->next != NULL || i < count - 1) { fprintf(f, ", "); } } @@ -456,12 +455,13 @@ endleaf_c(FILE *f, const struct fsm_end_ids *end_ids, } static int -endleaf_rust(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +endleaf_rust(FILE *f, + const fsm_end_id_t *ids, size_t count, + const void *endleaf_opaque) { const struct match *m; - int n; - size_t i, end_ids_count; + unsigned n; + size_t i; assert(endleaf_opaque == NULL); @@ -470,11 +470,9 @@ endleaf_rust(FILE *f, const struct fsm_end_ids *end_ids, n = 0; - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); - - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - n |= 1 << m->i; + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + n |= 1U << m->id; } } @@ -482,11 +480,11 @@ endleaf_rust(FILE *f, const struct fsm_end_ids *end_ids, fprintf(f, " /* "); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ - if (m->next != NULL || i < end_ids_count - 1) { + if (m->next != NULL || i < count - 1) { fprintf(f, ", "); } } @@ -498,11 +496,12 @@ endleaf_rust(FILE *f, const struct fsm_end_ids *end_ids, } static int -endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +endleaf_dot(FILE *f, + const fsm_end_id_t *ids, size_t count, + const void *endleaf_opaque) { const struct match *m; - size_t i, end_ids_count; + size_t i; assert(f != NULL); assert(endleaf_opaque == NULL); @@ -511,13 +510,11 @@ endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, fprintf(f, "label = <"); - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); - - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "#%u", m->i); + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + fprintf(f, "#%u", m->id); - if (m->next != NULL || i < end_ids_count - 1) { + if (m->next != NULL || i < count - 1) { fprintf(f, ","); } } @@ -531,11 +528,11 @@ endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, #if 0 fprintf(f, " # "); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ - if (m->next != NULL || i < end_ids_count - 1) { + if (m->next != NULL || i < count - 1) { fprintf(f, ", "); } } @@ -548,11 +545,12 @@ endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, } static int -endleaf_json(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +endleaf_json(FILE *f, + const fsm_end_id_t *ids, size_t count, + const void *endleaf_opaque) { const struct match *m; - size_t i, end_ids_count; + size_t i; assert(f != NULL); assert(endleaf_opaque == NULL); @@ -561,11 +559,9 @@ endleaf_json(FILE *f, const struct fsm_end_ids *end_ids, fprintf(f, "[ "); - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); - - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "%u", m->i); + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + fprintf(f, "%u", m->id); if (m->next != NULL) { fprintf(f, ", "); From 1e5cf21b148db89816dad710b1c2d53c6688c40b Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 15 Jun 2024 00:19:27 +0100 Subject: [PATCH 148/373] Normalization; attempting to get consistency around endleaf handling. --- include/fsm/options.h | 3 +++ src/libfsm/print/api.c | 2 ++ src/libfsm/print/c.c | 2 ++ src/libfsm/print/dot.c | 44 ++++++++++++++++++----------------- src/libfsm/print/fsm.c | 17 +++++++++++--- src/libfsm/print/go.c | 7 +++++- src/libfsm/print/irdot.c | 21 +++++++++++++++-- src/libfsm/print/irjson.c | 20 ++++++++++++++-- src/libfsm/print/json.c | 48 +++++++++++++++++++++++---------------- src/libfsm/print/vmasm.c | 11 +++++++++ src/libfsm/print/vmops.c | 19 ++++++++++++---- 11 files changed, 141 insertions(+), 53 deletions(-) diff --git a/include/fsm/options.h b/include/fsm/options.h index 759b3b8eb..e66c16b71 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -70,6 +70,9 @@ struct fsm_options { void *leaf_opaque; /* TODO: explain. for C code fragment output */ + /* Placement in the output stream depends on the format. + * This replaces an entire "return xyz;" statement for C-like formats, + * but appends extra information for others. */ int (*endleaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque); void *endleaf_opaque; diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 6b33d045f..54f39fc09 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -142,6 +142,8 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\t}\n"); fprintf(f, "\n"); + /* endleaf() doesn't make sense for this format */ + { size_t count; diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 21f15f837..00e71adb1 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -289,6 +289,7 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) } fprintf(f, "\tcase S%u: ", i); + if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, ir->states[i].endids.ids, ir->states[i].endids.count, @@ -299,6 +300,7 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) } else { fprintf(f, "return %u;", i); } + fprintf(f, "\n"); } fprintf(f, "\tdefault: return -1; /* unexpected EOT */\n"); diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 459584e63..e64edbb49 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -188,37 +188,39 @@ print_dotfrag(FILE *f, const struct fsm *fsm) fprintf(f, "\t%sS%-2u [ shape = doublecircle", fsm->opt->prefix != NULL ? fsm->opt->prefix : "", s); - if (fsm->opt->endleaf != NULL) { - fprintf(f, ", "); - if (-1 == fsm->opt->endleaf(f, ids, count, fsm->opt->endleaf_opaque)) { - return -1; - } - } else { - assert(f != NULL); + assert(f != NULL); - fprintf(f, ", "); + fprintf(f, ", "); - fprintf(f, "label = <"); + fprintf(f, "label = <"); - if (!fsm->opt->anonymous_states) { - fprintf(f, "%u", s); + if (fsm->opt->endleaf != NULL) { + if (-1 == fsm->opt->endleaf(f, + ids, count, + fsm->opt->endleaf_opaque)) + { + return -1; + } + } else if (!fsm->opt->anonymous_states) { + fprintf(f, "%u", s); + } - if (count > 0) { - fprintf(f, "
"); - } + if (fsm->opt->endleaf != NULL || !fsm->opt->anonymous_states) { + if (count > 0) { + fprintf(f, "
"); } + } - for (size_t i = 0; i < count; i++) { - fprintf(f, "#%u", ids[i]); + for (size_t i = 0; i < count; i++) { + fprintf(f, "#%u", ids[i]); - if (i < count - 1) { - fprintf(f, ","); - } + if (i < count - 1) { + fprintf(f, ","); } - - fprintf(f, ">"); } + fprintf(f, ">"); + fprintf(f, " ];\n"); f_free(fsm->opt->alloc, ids); diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 762bb1c30..2de984445 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -291,6 +291,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) fprintf(f, "end: "); for (s = 0; s < fsm->statecount; s++) { + fsm_end_id_t *ids; size_t count; if (!fsm_isend(fsm, s)) { @@ -299,11 +300,8 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) end--; - fprintf(f, "%u", s); - count = fsm_endid_count(fsm, s); if (count > 0) { - fsm_end_id_t *ids; size_t written; int res; @@ -315,7 +313,20 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) res = fsm_endid_get(fsm, s, count, ids, &written); assert(res == 1); assert(written == count); + } + if (fsm->opt->endleaf != NULL) { + if (-1 == fsm->opt->endleaf(f, + ids, count, + fsm->opt->endleaf_opaque)) + { + return -1; + } + } else { + fprintf(f, "%u", s); + } + + if (count > 0) { qsort(ids, count, sizeof *ids, comp_end_id); fprintf(f, " = ["); diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 22ed45ef5..7b3616f9d 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -98,6 +98,9 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return 0; } + fprintf(f, "{\n"); + fprintf(f, "\t\t"); + if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, op->ir_state->endids.ids, op->ir_state->endids.count, @@ -106,9 +109,11 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "{\n\t\treturn %td\n\t}\n", op->ir_state - ir->states); + fprintf(f, "return %td", op->ir_state - ir->states); } + fprintf(f, "\n\t}\n"); + return 0; } diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 039b9d568..8e91029ca 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -173,7 +173,7 @@ print_grouplinks(FILE *f, unsigned self, } } -static void +static int print_cs(FILE *f, const struct fsm_options *opt, const struct ir *ir, const struct ir_state *cs) { @@ -188,6 +188,7 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\tcs%u [ label =\n", ir_indexof(ir, cs)); fprintf(f, "\t\t<\n"); + fprintf(f, "\t\t \n", ir_indexof(ir, cs), strategy_name(cs->strategy)); @@ -213,6 +214,18 @@ print_cs(FILE *f, const struct fsm_options *opt, /* TODO: leaf callback for dot output */ + /* showing endleaf in addition to existing content */ + if (cs->isend && opt->endleaf != NULL) { + fprintf(f, "\t\t \n"); + } + switch (cs->strategy) { case IR_NONE: break; @@ -304,6 +317,8 @@ print_cs(FILE *f, const struct fsm_options *opt, default: ; } + + return 0; } int @@ -334,7 +349,9 @@ fsm_print_ir(FILE *f, const struct fsm *fsm) } for (i = 0; i < ir->n; i++) { - print_cs(f, fsm->opt, ir, &ir->states[i]); + if (-1 == print_cs(f, fsm->opt, ir, &ir->states[i])) { + return -1; + } } fprintf(f, "}\n"); diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 0de7459ae..189d4b571 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -112,7 +112,7 @@ print_groups(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\t\t]\n"); } -static void +static int print_cs(FILE *f, const struct fsm_options *opt, const struct ir_state *cs) { @@ -135,6 +135,18 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "],\n"); } + /* showing endleaf in addition to existing content */ + if (cs->isend && opt->endleaf != NULL) { + fprintf(f, "\t\t\t\"endleaf\": "); + if (-1 == opt->endleaf(f, + cs->endids.ids, cs->endids.count, + opt->endleaf_opaque)) + { + return -1; + } + fprintf(f, ",\n"); + } + fprintf(f, "\t\t\t\"strategy\": \"%s\"", strategy_name(cs->strategy)); if (cs->example != NULL || cs->strategy != IR_NONE) { fprintf(f, ","); @@ -195,6 +207,8 @@ print_cs(FILE *f, const struct fsm_options *opt, } fprintf(f, "\t\t}"); + + return 0; } int @@ -217,7 +231,9 @@ fsm_print_irjson(FILE *f, const struct fsm *fsm) fprintf(f, "\t\"states\": [\n"); for (i = 0; i < ir->n; i++) { - print_cs(f, fsm->opt, &ir->states[i]); + if (-1 == print_cs(f, fsm->opt, &ir->states[i])) { + return -1; + } if (i + 1 < ir->n) { fprintf(f, ","); diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 8eb1a32e8..49f751778 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -239,6 +239,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " ],\n"); } + /* showing endleaf in addition to existing content */ if (fsm->opt->endleaf != NULL) { int notfirst; @@ -246,35 +247,42 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " \"endleaf\": [ "); for (i = 0; i < fsm->statecount; i++) { - if (fsm_isend(fsm, i)) { - fsm_end_id_t *ids; - size_t count, written; - int res; + fsm_end_id_t *ids; + size_t count, written; + int res; - count = fsm_endid_count(fsm, i); + if (!fsm_isend(fsm, i)) { + continue; + } - ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); - if (ids == NULL) { - return -1; - } + count = fsm_endid_count(fsm, i); - res = fsm_endid_get(fsm, i, count, ids, &written); - assert(res == 1); + ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; + } - if (notfirst) { - fprintf(f, ", "); - } + res = fsm_endid_get(fsm, i, count, ids, &written); + assert(res == 1); - fprintf(f, "{ %u, ", i); + if (notfirst) { + fprintf(f, ", "); + } - fsm->opt->endleaf(f, ids, count, fsm->opt->endleaf_opaque); + fprintf(f, "{ %u, ", i); - fprintf(f, " }"); + if (-1 == fsm->opt->endleaf(f, + ids, count, + fsm->opt->endleaf_opaque)) + { + return -1; + } - f_free(fsm->opt->alloc, ids); + fprintf(f, " }"); - notfirst = 1; - } + f_free(fsm->opt->alloc, ids); + + notfirst = 1; } fprintf(f, " ],\n"); } diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 0c81bbf09..046610c44 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -171,7 +171,18 @@ print_asm_amd64(FILE *f, const char *prefix, break; } + + /* endleaf in addition to existing instructions */ + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { + return -1; + } + } } else { + /* TODO: we don't have a way to override the -1 (or its API) */ switch (dialect) { case AMD64_ATT: fprintf(f, "\tmovl $-1, %%%s\n", ret_reg); diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 976b47b00..11972ffa5 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -98,7 +98,7 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const char *prefix, +print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const char *prefix, enum dfavm_op_end end_bits, const struct ir *ir) { if (end_bits == VM_END_FAIL) { @@ -106,7 +106,18 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const char *prefix, return 0; } - fprintf(f, "%sactionRET, %td},\n", prefix, op->ir_state - ir->states); + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { + return -1; + } + } else { + fprintf(f, "%sactionRET, %td", prefix, op->ir_state - ir->states); + } + + fprintf(f, "},\n"); return 0; } @@ -177,7 +188,7 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, prefix, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, prefix, op->u.stop.end_bits, ir)) { return -1; } break; @@ -186,7 +197,7 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, if (-1 == print_fetch(f, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, prefix, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, prefix, op->u.fetch.end_bits, ir)) { return -1; } break; From f7bcbdda56d25cda5605e7559353538ea86e8c5b Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 15 Jun 2024 14:04:34 +0100 Subject: [PATCH 149/373] Remove dependencies on *nwritten, fsm_endid_count() beforehand instead. --- src/libfsm/minimise_test_oracle.c | 73 ++++++++++++++++++------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 70209f865..99b22025c 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -175,39 +175,50 @@ fsm_minimise_test_oracle(const struct fsm *fsm) * any of the other end states. If so, assign it to the same endid * group, otherwise assign a new one and mark it as the leader. */ for (size_t i = 0; i < state_count; i++) { - if (fsm_isend(fsm, i)) { - size_t written_a; - int eres = fsm_endid_get(fsm, i, - max_endid_count, endid_buf_a, &written_a); - if (eres == 0) { - assert(written_a == 0); - } else { - assert(written_a > 0); - bool found = false; - /* note: skipping eg 0 here since that's the empty set */ - for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { - size_t written_b; - eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], - max_endid_count, endid_buf_b, &written_b); - assert(eres != 0); - if (written_b != written_a) { continue; } - if (0 == memcmp(endid_buf_a, endid_buf_b, written_a * sizeof(endid_buf_a[0]))) { - found = true; - endid_group_assignments[i] = eg_i; - break; - } else { - continue; - } - } + if (!fsm_isend(fsm, i)) { + endid_group_assignments[i] = 0; /* none */ + continue; + } - if (!found) { - endid_group_assignments[i] = endid_group_count; - endid_group_leaders[endid_group_count] = i; - endid_group_count++; - } + size_t count_a = fsm_endid_count(fsm, i); + assert(count_a <= max_endid_count); + if (count_a == 0) { + continue; + } + + size_t written_a; + int eres = fsm_endid_get(fsm, i, + count_a, endid_buf_a, &written_a); + assert(eres == 1); + assert(written_a > 0); + + bool found = false; + /* note: skipping eg 0 here since that's the empty set */ + for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { + size_t written_b; + size_t count_b = fsm_endid_count(fsm, endid_group_leaders[eg_i]); + if (count_b != count_a) { + continue; } - } else { - endid_group_assignments[i] = 0; /* none */ + + assert(count_b > 0); + assert(count_b <= max_endid_count); + eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], + count_b, endid_buf_b, &written_b); + assert(eres == 1); + assert(written_b == count_b); + + if (0 == memcmp(endid_buf_a, endid_buf_b, count_a * sizeof(endid_buf_a[0]))) { + found = true; + endid_group_assignments[i] = eg_i; + break; + } + } + + if (!found) { + endid_group_assignments[i] = endid_group_count; + endid_group_leaders[endid_group_count] = i; + endid_group_count++; } } From 6b2426db38799e999e646031be125db3727388a0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 15 Jun 2024 18:13:19 +0100 Subject: [PATCH 150/373] No need for the *nwritten parameter. Now this is always equivalent to the count passed in, when the return status is 1. And the return status is always 1 when the count is enough. In all situations we know the count is enough. --- include/fsm/fsm.h | 9 ++++---- src/libfsm/endids.c | 11 ++------- src/libfsm/minimise.c | 4 +--- src/libfsm/minimise_test_oracle.c | 8 ++----- src/libfsm/print/api.c | 4 +--- src/libfsm/print/dot.c | 5 ++-- src/libfsm/print/fsm.c | 4 +--- src/libfsm/print/ir.c | 4 +--- src/libfsm/print/json.c | 4 ++-- src/libfsm/walk2.c | 8 ++----- src/lx/ast.c | 7 ++---- src/re/main.c | 4 +--- tests/capture/captest.c | 6 ++--- tests/capture/capture4.c | 7 +++--- tests/endids/endids0.c | 6 +---- tests/endids/endids0_many_endids.c | 15 ++++++------ tests/endids/endids1_determinise.c | 6 +---- .../endids/endids1_determinise_and_minimise.c | 6 +---- tests/endids/endids2_union.c | 10 +++----- tests/endids/endids2_union_many_endids.c | 14 ++++------- tests/endids/endids4.c | 6 +---- tests/endids/endids5.c | 6 +---- tests/endids/endids6.c | 23 +++++++------------ tests/endids/endids7.c | 21 ++++++----------- tests/endids/endids7_with_duplicates.c | 21 ++++++----------- tests/endids/endids8.c | 6 +---- tests/endids/endids9.c | 14 ++++------- tests/endids/utils.c | 3 +-- tests/gen/gtest.c | 5 ++-- tests/re_strings/testutil.c | 5 ++-- 30 files changed, 78 insertions(+), 174 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index ecc7c5937..00e9cf095 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -220,16 +220,15 @@ fsm_endid_set(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); /* Get the end IDs associated with an end state, if any. * If id_buf has enough cells to store all the end IDs (according - * to id_buf_count) then they are written into id_buf[] and - * *ids_written is set to the number of IDs. The end IDs in the - * buffer may appear in any order, but will not have duplicates. + * to id_buf_count) then they are written into id_buf[]. + * The end IDs in the buffer may appear in any order, + * but will not have duplicates. * * Returns 0 if there is not enough space in id_buf for the * end IDs, or 1 if zero or more end IDs were returned. */ int fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written); + size_t id_buf_count, fsm_end_id_t *id_buf); /* Get the number of end IDs associated with an end state. */ size_t diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index fb63baf66..df87da617 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -683,24 +683,19 @@ fsm_endid_count(const struct fsm *fsm, int fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written) + size_t id_buf_count, fsm_end_id_t *id_buf) { size_t i; - size_t written = 0; const struct endid_info *ei = NULL; uint64_t hash = hash_id(end_state); uint64_t mask; - (void)written; - assert(fsm != NULL); ei = fsm->endid_info; assert(ei != NULL); assert(id_buf != NULL); - assert(ids_written != NULL); mask = ei->bucket_count - 1; /* bucket count is a power of 2 */ @@ -721,8 +716,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, #if LOG_ENDIDS > 2 fprintf(stderr, "fsm_endid_get: not found\n"); #endif - *ids_written = 0; /* not found */ - return 0; + return 0; /* not found */ } else if (b->state == end_state) { size_t id_i; if (b->ids->count > id_buf_count) { @@ -739,7 +733,6 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, } /* todo: could sort them here, if it matters. */ - *ids_written = b->ids->count; return 1; } else { /* collision */ #if LOG_ENDIDS > 4 diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index a55de5d61..8100e22da 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -962,10 +962,8 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, return 0; } - size_t written; - int res = fsm_endid_get(fsm, s, e->count, e->ids, &written); + int res = fsm_endid_get(fsm, s, e->count, e->ids); assert(res == 1); - assert(written == e->count); /* sort, to make comparison easier later */ qsort(e->ids, e->count, diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 99b22025c..cc4d9efac 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -186,16 +186,13 @@ fsm_minimise_test_oracle(const struct fsm *fsm) continue; } - size_t written_a; int eres = fsm_endid_get(fsm, i, - count_a, endid_buf_a, &written_a); + count_a, endid_buf_a); assert(eres == 1); - assert(written_a > 0); bool found = false; /* note: skipping eg 0 here since that's the empty set */ for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { - size_t written_b; size_t count_b = fsm_endid_count(fsm, endid_group_leaders[eg_i]); if (count_b != count_a) { continue; @@ -204,9 +201,8 @@ fsm_minimise_test_oracle(const struct fsm *fsm) assert(count_b > 0); assert(count_b <= max_endid_count); eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], - count_b, endid_buf_b, &written_b); + count_b, endid_buf_b); assert(eres == 1); - assert(written_b == count_b); if (0 == memcmp(endid_buf_a, endid_buf_b, count_a * sizeof(endid_buf_a[0]))) { found = true; diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 54f39fc09..7a0a9ffa5 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -150,7 +150,6 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) count = fsm_endid_count(fsm, end); if (count > 0) { fsm_end_id_t *ids; - size_t written; int res; ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); @@ -159,9 +158,8 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) goto error; } - res = fsm_endid_get(fsm, end, count, ids, &written); + res = fsm_endid_get(fsm, end, count, ids); assert(res == 1); - assert(written == count); qsort(ids, count, sizeof *ids, comp_end_id); diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index e64edbb49..54d151a96 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -168,7 +168,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) } } else { fsm_end_id_t *ids; - size_t count, written; + size_t count; ids = NULL; count = fsm_endid_count(fsm, s); @@ -180,9 +180,8 @@ print_dotfrag(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_endid_get(fsm, s, count, ids, &written); + res = fsm_endid_get(fsm, s, count, ids); assert(res == 1); - assert(count == written); } fprintf(f, "\t%sS%-2u [ shape = doublecircle", diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 2de984445..46bc428ff 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -302,7 +302,6 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) count = fsm_endid_count(fsm, s); if (count > 0) { - size_t written; int res; ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); @@ -310,9 +309,8 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_endid_get(fsm, s, count, ids, &written); + res = fsm_endid_get(fsm, s, count, ids); assert(res == 1); - assert(written == count); } if (fsm->opt->endleaf != NULL) { diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 4af6a8c11..f23ce238a 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -479,16 +479,14 @@ make_ir(const struct fsm *fsm) ids = NULL; } else { int res; - size_t written; ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); if (ids == NULL) { goto error; } - res = fsm_endid_get(fsm, i, count, ids, &written); + res = fsm_endid_get(fsm, i, count, ids); assert(res == 1); - assert(count = written); } ir->states[i].endids.ids = ids; diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 49f751778..11c7a5f24 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -248,7 +248,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " \"endleaf\": [ "); for (i = 0; i < fsm->statecount; i++) { fsm_end_id_t *ids; - size_t count, written; + size_t count; int res; if (!fsm_isend(fsm, i)) { @@ -262,7 +262,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) return -1; } - res = fsm_endid_get(fsm, i, count, ids, &written); + res = fsm_endid_get(fsm, i, count, ids); assert(res == 1); if (notfirst) { diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index 9320dbafd..20580dd60 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -285,17 +285,13 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, } if (num_a_endids > 0) { - size_t nwritten = 0; - ret = fsm_endid_get(fsm_a, a, num_a_endids, endids, &nwritten); + ret = fsm_endid_get(fsm_a, a, num_a_endids, endids); assert(ret == 1); - assert(nwritten == num_a_endids); } if (num_b_endids > 0) { - size_t nwritten = 0; - ret = fsm_endid_get(fsm_b, b, num_b_endids, endids + num_a_endids, &nwritten); + ret = fsm_endid_get(fsm_b, b, num_b_endids, endids + num_a_endids); assert(ret == 1); - assert(nwritten == num_b_endids); } ret = fsm_endid_set_bulk( diff --git a/src/lx/ast.c b/src/lx/ast.c index 7dd1ad72c..5852c7055 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -236,7 +236,7 @@ struct ast_mapping * ast_getendmapping(const struct fsm *fsm, fsm_state_t s) { fsm_end_id_t *ids; - size_t count, written; + size_t count; struct ast_mapping *m; int res; @@ -256,11 +256,8 @@ ast_getendmapping(const struct fsm *fsm, fsm_state_t s) return NULL; } - res = fsm_endid_get(fsm, s, count, ids, &written); - + res = fsm_endid_get(fsm, s, count, ids); assert(res == 1); - assert(written == count); - assert(written > 0); m = ast_getendmappingbyendid(ids[0]); diff --git a/src/re/main.c b/src/re/main.c index 9c54faa99..ea2c68ae8 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -289,7 +289,6 @@ find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) { fsm_end_id_t *ids, id; size_t count; - size_t nwritten; int res; if (!fsm_isend(dfa, s)) { @@ -303,9 +302,8 @@ find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) ids = xmalloc(count * sizeof *ids); - res = fsm_endid_get(dfa, s, count, ids, &nwritten); + res = fsm_endid_get(dfa, s, count, ids); assert(res == 1); - assert(nwritten == count); id = ids[0]; diff --git a/tests/capture/captest.c b/tests/capture/captest.c index 5a6384686..b882043a8 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -91,12 +91,11 @@ captest_run_single(const struct captest_single_fsm_test_info *info) { fsm_end_id_t id_buf[1] = { ~0 }; int gres; - size_t written; if (1 != fsm_endid_count(fsm, end)) { FAIL("did not have exactly one end ID"); } - gres = fsm_endid_get(fsm, end, 1, id_buf, &written); + gres = fsm_endid_get(fsm, end, 1, id_buf); if (gres != 1) { FAIL("failed to get end IDs"); } @@ -183,7 +182,6 @@ captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, { fsm_end_id_t id_buf[1] = { ~0 }; int gres; - size_t written; const char *unused; if (msg == NULL) { @@ -195,7 +193,7 @@ captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, return 0; } - gres = fsm_endid_get(fsm, end_state, 1, id_buf, &written); + gres = fsm_endid_get(fsm, end_state, 1, id_buf); if (gres != 1) { *msg = "failed to get end IDs"; return 0; diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 4394b637f..0d9685358 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -222,18 +222,17 @@ check(const struct fsm *fsm, const char *string, { int gres; fsm_end_id_t id_buf[2]; - size_t written; - gres = fsm_endid_get(fsm, end, 2, id_buf, &written); + gres = fsm_endid_get(fsm, end, 2, id_buf); if (gres != 1) { assert(!"fsm_getendids failed"); } if (expected_ends == 0x2) { - assert(written == 1); + assert(fsm_endid_count(fsm, end) == 1); assert(id_buf[0] == 1); } else if (expected_ends == 0x3) { - assert(written == 2); + assert(fsm_endid_count(fsm, end) == 2); assert(id_buf[0] == 0); assert(id_buf[1] == 1); } else { diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index 69df62775..2bbe6a237 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -44,21 +44,17 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; - size_t nwritten; int ret; assert( fsm_endid_count(fsm, state_ind) == 1); - nwritten = 0; ret = fsm_endid_get( fsm, state_ind, 1, - &endid, - &nwritten); + &endid); assert(ret == 1); - assert(nwritten == 1); assert(endid == 1); nend++; diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index f451b9acc..62f52e1a2 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -85,27 +85,26 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[sizeof all_end_ids / sizeof all_end_ids[0]]; - size_t i, nwritten; + size_t i; int ret; memset(&endids[0], 0, sizeof endids); assert(fsm_endid_count(fsm, state_ind) == sizeof all_end_ids / sizeof all_end_ids[0]); - nwritten = 0; ret = fsm_endid_get( fsm, state_ind, - sizeof endids/sizeof endids[0], - &endids[0], - &nwritten); + sizeof endids / sizeof endids[0], + &endids[0]); assert(ret == 1); - assert(nwritten == sizeof all_end_ids / sizeof all_end_ids[0]); /* sort endids and compare */ - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); - for (i=0; i < nwritten; i++) { + qsort(&endids[0], + sizeof endids / sizeof endids[0], sizeof endids[0], + cmp_endids); + for (i=0; i < fsm_endid_count(fsm, state_ind); i++) { assert(endids[i] == sorted_all_end_ids[i]); } diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 5d61e1db4..3ce6f6b0f 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -40,21 +40,17 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; - size_t nwritten; int ret; assert( fsm_endid_count(fsm, state_ind) == 1); - nwritten = 0; ret = fsm_endid_get( fsm, state_ind, 1, - &endid, - &nwritten); + &endid); assert(ret == 1); - assert(nwritten == 1); assert( endid == 1 ); } } diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index 91adf3f44..a62d7979b 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -45,21 +45,17 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; - size_t nwritten; int ret; assert(fsm_endid_count(fsm, state_ind) == 1); - nwritten = 0; ret = fsm_endid_get( fsm, state_ind, 1, - &endid, - &nwritten); + &endid); assert(ret == 1); - assert(nwritten == 1); assert( endid == 1 ); } } diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index 905c4b324..e939c5474 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -55,11 +55,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(comb, state_ind); // fprintf(stderr, "state %u, count = %zu\n", state_ind, count); @@ -69,18 +67,16 @@ int main(void) comb, state_ind, 2, - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); - if (nwritten == 1) { + if (count == 1) { assert(endids[0] == 1 || endids[0] == 2); has_endid_1 = has_endid_1 || (endids[0] == 1); has_endid_2 = has_endid_2 || (endids[0] == 2); - } else if (nwritten == 2) { + } else if (count == 2) { assert((endids[0] == 1 && endids[1] == 2) || (endids[0] == 2 && endids[1] == 1)); diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index e292be47b..830421e66 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -183,12 +183,11 @@ int main(void) if (fsm_isend(fsm, state_ind)) { int tested_pattern[NUM_PATTERNS]; fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, count, j; + size_t count, j; int ret; memset(&endids[0], 0, sizeof endids); - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert(count > 0 && count <= sizeof endids/sizeof endids[0]); @@ -197,11 +196,9 @@ int main(void) fsm, state_ind, sizeof endids/sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); memset(&tested_pattern[0], 0, sizeof tested_pattern); @@ -250,12 +247,11 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, count; + size_t count; int ret; memset(&endids[0], 0, sizeof endids); - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert(count <= NUM_ENDIDS_TOTAL); @@ -264,11 +260,9 @@ int main(void) fsm, state_ind, sizeof endids/sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); } } diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index 1b022aad4..64838cc83 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -67,11 +67,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(comb, state_ind); printf("state %u count = %zu\n", state_ind, count); @@ -82,11 +80,9 @@ int main(void) comb, state_ind, 2, - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2); diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index 3639ab0ac..2fce9ea92 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -68,11 +68,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(comb, state_ind); printf("state %u count = %zu\n", state_ind, count); @@ -83,11 +81,9 @@ int main(void) comb, state_ind, 2, - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); assert(endids[0] == 1); } diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index 657a314dc..77976991c 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -92,11 +92,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert( count > 0 && count <= 2); @@ -105,20 +103,18 @@ int main(void) fsm, state_ind, sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].count = nwritten; + info[ninfo].count = count; - if (nwritten == 1) { + if (count == 1) { assert(endids[0] == 1 || endids[0] == 2); info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); + } else if (count == 2) { + qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2); info[ninfo].endids[0] = endids[0]; info[ninfo].endids[1] = endids[1]; @@ -137,12 +133,11 @@ int main(void) for (state_ind = 0, info_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten, count, ind; + size_t count, ind; int ret; assert( state_ind == info[info_ind].state ); - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert(count > 0 && count <= 2); @@ -152,13 +147,11 @@ int main(void) fsm, state_ind, sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); - if (nwritten > 1) { + if (count > 1) { qsort(&endids[0], count, sizeof endids[0], cmp_endids); } diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index 488e1e3e0..451cf342a 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -104,11 +104,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert( count > 0 && count <= 2); @@ -117,20 +115,18 @@ int main(void) fsm, state_ind, sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].count = nwritten; + info[ninfo].count = count; - if (nwritten == 1) { + if (count == 1) { assert(endids[0] == 1 || endids[0] == 2); info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); + } else if (count == 2) { + qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2); info[ninfo].endids[0] = endids[0]; info[ninfo].endids[1] = endids[1]; @@ -150,12 +146,11 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; fsm_end_id_t expected; - size_t nwritten, count; + size_t count; int ret; assert( state_ind == info[info_ind].state ); - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert(count == 1); @@ -163,11 +158,9 @@ int main(void) fsm, state_ind, 1, - &endid, - &nwritten); + &endid); assert(ret == 1); - assert(nwritten == count); if (info[info_ind].count == 2) { expected = 3; diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index b4d0701d3..40acaac8b 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -108,11 +108,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert( count > 0 && count <= 2); @@ -121,20 +119,18 @@ int main(void) fsm, state_ind, sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].count = nwritten; + info[ninfo].count = count; - if (nwritten == 1) { + if (count == 1) { assert(endids[0] == 1 || endids[0] == 2); info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); + } else if (count == 2) { + qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2); info[ninfo].endids[0] = endids[0]; info[ninfo].endids[1] = endids[1]; @@ -154,12 +150,11 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; fsm_end_id_t expected; - size_t nwritten, count; + size_t count; int ret; assert( state_ind == info[info_ind].state ); - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert(count == 1); @@ -167,11 +162,9 @@ int main(void) fsm, state_ind, 1, - &endid, - &nwritten); + &endid); assert(ret == 1); - assert(nwritten == count); if (info[info_ind].count == 2) { expected = 3; diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index 0ef6e024b..dedd750b0 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -73,11 +73,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[3] = {0,0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(comb, state_ind); printf("state %u count = %zu\n", state_ind, count); @@ -88,11 +86,9 @@ int main(void) comb, state_ind, sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2 && endids[2] == 4); diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index ef15efe13..38328f5a4 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -72,11 +72,9 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; size_t count; int ret; - nwritten = 0; count = fsm_endid_count(fsm, state_ind); assert( count > 0 && count <= 2); @@ -85,20 +83,18 @@ int main(void) fsm, state_ind, sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); + &endids[0]); assert(ret == 1); - assert(nwritten == count); info[ninfo].state = state_ind; - info[ninfo].count = nwritten; + info[ninfo].count = count; - if (nwritten == 1) { + if (count == 1) { assert(endids[0] == 11 || endids[0] == 12); info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); + } else if (count == 2) { + qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 11 && endids[1] == 12); info[ninfo].endids[0] = endids[0]; info[ninfo].endids[1] = endids[1]; diff --git a/tests/endids/utils.c b/tests/endids/utils.c index 7457fc70e..a5d96f020 100644 --- a/tests/endids/utils.c +++ b/tests/endids/utils.c @@ -20,14 +20,13 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end count = fsm_endid_count(fsm, end); if (count > 0) { int ret; - size_t nwritten = 0; fsm_end_id_t *endids = calloc(count, sizeof *endids); if (endids == NULL) { return -1; } - ret = fsm_endid_get(fsm, end, count, endids, &nwritten); + ret = fsm_endid_get(fsm, end, count, endids); if (ret == 0) { free(endids); errno = EINVAL; diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index aff46ba3a..53fc89602 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -59,9 +59,8 @@ gtest_matches_cb(const struct fsm *fsm, #define ID_BUF_COUNT 1 fsm_end_id_t id_buf[ID_BUF_COUNT]; - size_t written; int gres = fsm_endid_get(fsm, - end_state, ID_BUF_COUNT, id_buf, &written); + end_state, ID_BUF_COUNT, id_buf); if (gres != 1) { fprintf(stderr, @@ -70,7 +69,7 @@ gtest_matches_cb(const struct fsm *fsm, return FSM_GENERATE_MATCHES_CB_RES_HALT; } - if (written != 1 || id_buf[0] != m_i) { + if (fsm_endid_count(fsm, end_state) != 1 || id_buf[0] != m_i) { fprintf(stderr, "ERROR: endid mismatch, expected %zu, got %u\n", m_i, id_buf[0]); return FSM_GENERATE_MATCHES_CB_RES_HALT; diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 197415492..59d972512 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -47,12 +47,11 @@ run_test(const char **strings) const int res = fsm_exec(fsm, fsm_sgetc, string, &end, NULL); assert(res > 0); /* match */ - size_t written; int eres = fsm_endid_get(fsm, end, - MAX_INPUTS, id_buf, &written); + MAX_INPUTS, id_buf); assert(eres == 1); bool found = false; - for (size_t i = 0; i < written; i++) { + for (size_t i = 0; i < fsm_endid_count(fsm, end); i++) { if (id_buf[i] == id) { found = true; break; From 99fa19cc6b047fe13d5600d2377169e9c18d6d6c Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 15 Jun 2024 21:10:25 +0100 Subject: [PATCH 151/373] Getting 0 end IDs is not an error. --- include/fsm/fsm.h | 12 ++++++++++-- src/libfsm/endids.c | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 00e9cf095..269ee5c87 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -219,11 +219,19 @@ int fsm_endid_set(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); /* Get the end IDs associated with an end state, if any. - * If id_buf has enough cells to store all the end IDs (according - * to id_buf_count) then they are written into id_buf[]. + * id_buf is expected to have enough cells (according to id_buf_count) + * to store all the end IDs. You can find this with fsm_endid_count(). + * * The end IDs in the buffer may appear in any order, * but will not have duplicates. * + * A state with no end IDs set is considered equivalent to a state + * that has the empty set, this API does not distinguish these cases. + * This is not an error. + * + * It is an error to attempt to get end IDs associated with a state + * that is not marked as an end state. + * * Returns 0 if there is not enough space in id_buf for the * end IDs, or 1 if zero or more end IDs were returned. */ int diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index df87da617..135392a74 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -716,8 +716,10 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, #if LOG_ENDIDS > 2 fprintf(stderr, "fsm_endid_get: not found\n"); #endif - return 0; /* not found */ - } else if (b->state == end_state) { + return 1; /* not an error */ + } + + if (b->state == end_state) { size_t id_i; if (b->ids->count > id_buf_count) { #if LOG_ENDIDS > 2 From f663cd4274d0ebb7235d358aca483696599551c2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 16 Jun 2024 22:38:44 +0100 Subject: [PATCH 152/373] Naming; ids[] for endids. --- src/libfsm/endids.c | 8 +-- src/libfsm/minimise_test_oracle.c | 26 ++++----- src/libfsm/walk2.c | 39 ++++++++------ tests/capture/captest.c | 12 ++--- tests/capture/capture4.c | 10 ++-- tests/endids/endids0.c | 30 +++++------ tests/endids/endids0_many_endids.c | 54 +++++++++---------- tests/endids/endids1_determinise.c | 24 ++++----- .../endids/endids1_determinise_and_minimise.c | 24 ++++----- tests/endids/endids2_union.c | 26 ++++----- tests/endids/endids2_union_many_endids.c | 34 ++++++------ tests/endids/endids4.c | 26 ++++----- tests/endids/endids5.c | 26 ++++----- tests/endids/endids6.c | 54 +++++++++---------- tests/endids/endids7.c | 50 ++++++++--------- tests/endids/endids7_with_duplicates.c | 42 +++++++-------- tests/endids/endids8.c | 14 ++--- tests/endids/endids9.c | 20 +++---- tests/endids/utils.c | 16 +++--- tests/gen/gtest.c | 8 +-- tests/re_strings/testutil.c | 6 +-- 21 files changed, 279 insertions(+), 270 deletions(-) diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 135392a74..66860e88a 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -683,7 +683,7 @@ fsm_endid_count(const struct fsm *fsm, int fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf) + size_t count, fsm_end_id_t *ids) { size_t i; const struct endid_info *ei = NULL; @@ -695,7 +695,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, ei = fsm->endid_info; assert(ei != NULL); - assert(id_buf != NULL); + assert(ids != NULL); mask = ei->bucket_count - 1; /* bucket count is a power of 2 */ @@ -721,7 +721,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, if (b->state == end_state) { size_t id_i; - if (b->ids->count > id_buf_count) { + if (b->ids->count > count) { #if LOG_ENDIDS > 2 fprintf(stderr, "fsm_endid_get: insufficient space\n"); #endif @@ -731,7 +731,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, #if LOG_ENDIDS > 2 fprintf(stderr, "fsm_endid_get: writing id[%zu]: %d\n", id_i, b->ids->ids[id_i]); #endif - id_buf[id_i] = b->ids->ids[id_i]; + ids[id_i] = b->ids->ids[id_i]; } /* todo: could sort them here, if it matters. */ diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index cc4d9efac..6cd1848c4 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -113,8 +113,8 @@ fsm_minimise_test_oracle(const struct fsm *fsm) unsigned *endid_group_assignments = NULL; size_t endid_group_count = 1; /* group 0 is the empty set */ unsigned *endid_group_leaders = NULL; - fsm_end_id_t *endid_buf_a = NULL; - fsm_end_id_t *endid_buf_b = NULL; + fsm_end_id_t *ids_a = NULL; + fsm_end_id_t *ids_b = NULL; table = calloc(row_words * table_states, sizeof(table[0])); if (table == NULL) { goto cleanup; } @@ -165,10 +165,10 @@ fsm_minimise_test_oracle(const struct fsm *fsm) } } - endid_buf_a = malloc(max_endid_count * sizeof(endid_buf_a[0])); - if (endid_buf_a == NULL) { goto cleanup; } - endid_buf_b = malloc(max_endid_count * sizeof(endid_buf_b[0])); - if (endid_buf_b == NULL) { goto cleanup; } + ids_a = malloc(max_endid_count * sizeof(ids_a[0])); + if (ids_a == NULL) { goto cleanup; } + ids_b = malloc(max_endid_count * sizeof(ids_b[0])); + if (ids_b == NULL) { goto cleanup; } /* For every end state, check if it has endids. If not, assign it * to endid group 0 (none). Otherwise, check if its endids match @@ -187,7 +187,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) } int eres = fsm_endid_get(fsm, i, - count_a, endid_buf_a); + count_a, ids_a); assert(eres == 1); bool found = false; @@ -201,10 +201,10 @@ fsm_minimise_test_oracle(const struct fsm *fsm) assert(count_b > 0); assert(count_b <= max_endid_count); eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], - count_b, endid_buf_b); + count_b, ids_b); assert(eres == 1); - if (0 == memcmp(endid_buf_a, endid_buf_b, count_a * sizeof(endid_buf_a[0]))) { + if (0 == memcmp(ids_a, ids_b, count_a * sizeof(ids_a[0]))) { found = true; endid_group_assignments[i] = eg_i; break; @@ -361,8 +361,8 @@ fsm_minimise_test_oracle(const struct fsm *fsm) free(mapping); free(endid_group_assignments); free(endid_group_leaders); - free(endid_buf_a); - free(endid_buf_b); + free(ids_a); + free(ids_b); return res; @@ -372,8 +372,8 @@ fsm_minimise_test_oracle(const struct fsm *fsm) if (mapping != NULL) { free(mapping); } if (endid_group_assignments != NULL) { free(endid_group_assignments); } if (endid_group_leaders != NULL) { free(endid_group_leaders); } - if (endid_buf_a != NULL) { free(endid_buf_a); } - if (endid_buf_b != NULL) { free(endid_buf_b); } + if (ids_a != NULL) { free(ids_a); } + if (ids_b != NULL) { free(ids_b); } if (res != NULL) { fsm_free(res); } return NULL; } diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index 20580dd60..afd8107ca 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -263,45 +263,54 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, } if (is_end) { - size_t num_a_endids = 0, num_b_endids = 0, total_num_endids; + size_t count_a = 0, count_b = 0, count_total; if (fsm_a != NULL && fsm_isend(fsm_a,a)) { - num_a_endids = fsm_endid_count(fsm_a, a); + count_a = fsm_endid_count(fsm_a, a); } if (fsm_b != NULL && fsm_isend(fsm_b,b)) { - num_b_endids = fsm_endid_count(fsm_b, b); + count_b = fsm_endid_count(fsm_b, b); } - total_num_endids = num_a_endids + num_b_endids; + count_total = count_a + count_b; - if (total_num_endids > 0) { - fsm_end_id_t *endids; + if (count_total > 0) { + const struct fsm_alloc *alloc; + fsm_end_id_t *ids; int ret; - endids = calloc(total_num_endids, sizeof endids[0]); - if (endids == NULL) { + if (fsm_a != NULL) { + alloc = fsm_a->opt->alloc; + } else if (fsm_b != NULL) { + alloc = fsm_b->opt->alloc; + } else { + assert(!"unreached"); + } + + ids = f_malloc(alloc, count_total * sizeof *ids); + if (ids == NULL) { return NULL; } - if (num_a_endids > 0) { - ret = fsm_endid_get(fsm_a, a, num_a_endids, endids); + if (count_a > 0) { + ret = fsm_endid_get(fsm_a, a, count_a, ids); assert(ret == 1); } - if (num_b_endids > 0) { - ret = fsm_endid_get(fsm_b, b, num_b_endids, endids + num_a_endids); + if (count_b > 0) { + ret = fsm_endid_get(fsm_b, b, count_b, ids + count_a); assert(ret == 1); } ret = fsm_endid_set_bulk( data->new, p->comb, - total_num_endids, - &endids[0], + count_total, + &ids[0], FSM_ENDID_BULK_REPLACE); - free(endids); + f_free(alloc, ids); if (!ret) { int errsv = errno; diff --git a/tests/capture/captest.c b/tests/capture/captest.c index b882043a8..d1b5c510a 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -89,18 +89,18 @@ captest_run_single(const struct captest_single_fsm_test_info *info) if (end != strlen(info->string)) { FAIL("exec end pos"); } { - fsm_end_id_t id_buf[1] = { ~0 }; + fsm_end_id_t ids[1] = { ~0 }; int gres; if (1 != fsm_endid_count(fsm, end)) { FAIL("did not have exactly one end ID"); } - gres = fsm_endid_get(fsm, end, 1, id_buf); + gres = fsm_endid_get(fsm, end, 1, ids); if (gres != 1) { FAIL("failed to get end IDs"); } - if (0 != id_buf[0]) { + if (0 != ids[0]) { FAIL("failed to get end ID of 0"); } } @@ -180,7 +180,7 @@ int captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, unsigned expected_end_id, const char **msg) { - fsm_end_id_t id_buf[1] = { ~0 }; + fsm_end_id_t ids[1] = { ~0 }; int gres; const char *unused; @@ -193,13 +193,13 @@ captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, return 0; } - gres = fsm_endid_get(fsm, end_state, 1, id_buf); + gres = fsm_endid_get(fsm, end_state, 1, ids); if (gres != 1) { *msg = "failed to get end IDs"; return 0; } - if (expected_end_id != id_buf[0]) { + if (expected_end_id != ids[0]) { *msg = "failed to get expected end ID"; return 0; } diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 0d9685358..313086c97 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -221,20 +221,20 @@ check(const struct fsm *fsm, const char *string, { int gres; - fsm_end_id_t id_buf[2]; + fsm_end_id_t ids[2]; - gres = fsm_endid_get(fsm, end, 2, id_buf); + gres = fsm_endid_get(fsm, end, 2, ids); if (gres != 1) { assert(!"fsm_getendids failed"); } if (expected_ends == 0x2) { assert(fsm_endid_count(fsm, end) == 1); - assert(id_buf[0] == 1); + assert(ids[0] == 1); } else if (expected_ends == 0x3) { assert(fsm_endid_count(fsm, end) == 2); - assert(id_buf[0] == 0); - assert(id_buf[1] == 1); + assert(ids[0] == 0); + assert(ids[1] == 1); } else { assert(!"test not handled"); } diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index 2bbe6a237..c9117c0b0 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -43,7 +43,7 @@ int main(void) // memset(all_endids, 0, sizeof all_endids); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endid = 0; + fsm_end_id_t id = 0; int ret; assert( fsm_endid_count(fsm, state_ind) == 1); @@ -52,10 +52,10 @@ int main(void) fsm, state_ind, 1, - &endid); + &id); assert(ret == 1); - assert(endid == 1); + assert(id == 1); nend++; } @@ -84,26 +84,26 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { assert( ret == 1 ); - assert( end_ids != NULL ); - assert( end_ids[0] = 1 ); - assert( num_end_ids == 1 ); - assert( end_ids[0] = matches[i].endid ); + assert( ids != NULL ); + assert( ids[0] = 1 ); + assert( count == 1 ); + assert( ids[0] = matches[i].endid ); } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index 62f52e1a2..0af45112b 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -39,18 +39,18 @@ int main(void) * * 17 endids. This should force the array of end ids to resize. */ - static const fsm_end_id_t all_end_ids[17] = { + static const fsm_end_id_t all_ids[17] = { 4, 17, 6, 18, 2, 63, 14, 62, 3, 37, 46, 7, 9, 72, 67, 36, 1, }; - static const fsm_end_id_t sorted_all_end_ids[17] = { + static const fsm_end_id_t sorted_all_ids[17] = { 1, 2, 3, 4, 6, 7, 9, 14, 17, 18, 36, 37, 46, 62, 63, 67, 72, }; - for (size_t i=0; i < sizeof all_end_ids / sizeof all_end_ids[0]; i++) { - ret = fsm_setendid(fsm, all_end_ids[i]); + for (size_t i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { + ret = fsm_setendid(fsm, all_ids[i]); assert(ret == 1); } @@ -69,28 +69,28 @@ int main(void) assert(end_state < nstates); - assert(fsm_endid_count(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); - for (i=0; i < sizeof all_end_ids / sizeof all_end_ids[0]; i++) { + assert(fsm_endid_count(fsm, end_state) == sizeof all_ids / sizeof all_ids[0]); + for (i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { /* add duplicate end ids */ - ret = fsm_setendid(fsm, all_end_ids[i]); + ret = fsm_setendid(fsm, all_ids[i]); /* fsm_setendid should succeed */ assert(ret == 1); /* but it shouldn't add a duplicate id */ - assert(fsm_endid_count(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, end_state) == sizeof all_ids / sizeof all_ids[0]); } nend = 0; for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[sizeof all_end_ids / sizeof all_end_ids[0]]; + fsm_end_id_t endids[sizeof all_ids / sizeof all_ids[0]]; size_t i; int ret; memset(&endids[0], 0, sizeof endids); - assert(fsm_endid_count(fsm, state_ind) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, state_ind) == sizeof all_ids / sizeof all_ids[0]); ret = fsm_endid_get( fsm, @@ -105,13 +105,13 @@ int main(void) sizeof endids / sizeof endids[0], sizeof endids[0], cmp_endids); for (i=0; i < fsm_endid_count(fsm, state_ind); i++) { - assert(endids[i] == sorted_all_end_ids[i]); + assert(endids[i] == sorted_all_ids[i]); } #if 0 /* endids should be sorted */ for (i=0; i < nwritten; i++) { - assert(endids[i] == sorted_all_end_ids[i]); + assert(endids[i] == sorted_all_ids[i]); } #endif /* 0 */ @@ -141,38 +141,38 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); - assert( num_end_ids == sizeof all_end_ids / sizeof all_end_ids[0] ); + assert( ids != NULL ); + assert( count == sizeof all_ids / sizeof all_ids[0] ); #if 0 - assert( end_ids[0] == 1 ); - assert( end_ids[0] == matches[i].endid ); + assert( ids[0] == 1 ); + assert( ids[0] == matches[i].endid ); #endif /* 0 */ /* sort endids and compare */ - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < num_end_ids; j++) { - assert(end_ids[j] == sorted_all_end_ids[j]); + for (j=0; j < count; j++) { + assert(ids[j] == sorted_all_ids[j]); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 3ce6f6b0f..3860aefca 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -75,26 +75,26 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { assert( ret == 1 ); - assert( end_ids != NULL ); - assert( end_ids[0] = 1 ); - assert( num_end_ids == 1 ); - assert( end_ids[0] = matches[i].endid ); + assert( ids != NULL ); + assert( ids[0] = 1 ); + assert( count == 1 ); + assert( ids[0] = matches[i].endid ); } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index a62d7979b..bb0711905 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -80,26 +80,26 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { assert( ret == 1 ); - assert( end_ids != NULL ); - assert( end_ids[0] = 1 ); - assert( num_end_ids == 1 ); - assert( end_ids[0] = matches[i].endid ); + assert( ids != NULL ); + assert( ids[0] = 1 ); + assert( count == 1 ); + assert( ids[0] = matches[i].endid ); } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index e939c5474..c990e7c95 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -143,33 +143,33 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(comb, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); + assert( ids != NULL ); - assert( num_end_ids == matches[i].count ); + assert( count == matches[i].count ); - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < num_end_ids; j++) { - assert( end_ids[j] == matches[i].endid[j] ); + for (j=0; j < count; j++) { + assert( ids[j] == matches[i].endid[j] ); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index 830421e66..deafcff1e 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -121,18 +121,18 @@ size_t generate_examples(struct example_list *l, const struct fsm *fsm, size_t p return (first_index != SIZE_MAX) ? first_index : 0; } -/* test that endids correctly propagate through union, determinise, and minimise */ +/* test that ids correctly propagate through union, determinise, and minimise */ int main(void) { struct fsm *fsm; - fsm_end_id_t all_endids[NUM_PATTERNS]; + fsm_end_id_t all_ids[NUM_PATTERNS]; size_t nstates, state_ind; size_t i; int ret; struct example_list example_list; - memset(all_endids, 0, sizeof all_endids); + memset(all_ids, 0, sizeof all_ids); init_examples(&example_list); @@ -176,27 +176,27 @@ int main(void) ret = fsm_determinise(fsm); assert(ret != 0); - // find end states, make sure we have multiple end states and they each have endids + // find end states, make sure we have multiple end states and they each have ids nstates = fsm_countstates(fsm); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { int tested_pattern[NUM_PATTERNS]; - fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; + fsm_end_id_t ids[NUM_ENDIDS_TOTAL]; size_t count, j; int ret; - memset(&endids[0], 0, sizeof endids); + memset(&ids[0], 0, sizeof ids); count = fsm_endid_count(fsm, state_ind); - assert(count > 0 && count <= sizeof endids/sizeof endids[0]); + assert(count > 0 && count <= sizeof ids/sizeof ids[0]); ret = fsm_endid_get( fsm, state_ind, - sizeof endids/sizeof endids[0], - &endids[0]); + sizeof ids/sizeof ids[0], + &ids[0]); assert(ret == 1); @@ -205,7 +205,7 @@ int main(void) for (j=0; j < count; j++) { size_t k, pattern_index; - pattern_index = (endids[j] - 1)/NUM_ENDIDS_PER_PATTERN; + pattern_index = (ids[j] - 1)/NUM_ENDIDS_PER_PATTERN; assert(pattern_index < NUM_PATTERNS); if (tested_pattern[pattern_index]) { @@ -224,11 +224,11 @@ int main(void) if (ret) { #if 0 printf("end state %zu (end id %u) matches example \"%s\" from pattern /%s/\n", - state_ind, endids[j], ex->example, patterns[pattern_index]); + state_ind, ids[j], ex->example, patterns[pattern_index]); #endif /* 0 */ } else { printf("end state %zu (end id %u) does NOT match example \"%s\" from pattern /%s/\n", - state_ind, endids[j], ex->example, patterns[pattern_index]); + state_ind, ids[j], ex->example, patterns[pattern_index]); abort(); } } @@ -240,17 +240,17 @@ int main(void) assert(ret != 0); /* fsm_minimise should not collapse all the end states to a - * single end state, because they have distinct endids. */ + * single end state, because they have distinct ids. */ assert( fsm_count(fsm, fsm_isend) > 1); nstates = fsm_countstates(fsm); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; + fsm_end_id_t ids[NUM_ENDIDS_TOTAL]; size_t count; int ret; - memset(&endids[0], 0, sizeof endids); + memset(&ids[0], 0, sizeof ids); count = fsm_endid_count(fsm, state_ind); @@ -259,8 +259,8 @@ int main(void) ret = fsm_endid_get( fsm, state_ind, - sizeof endids/sizeof endids[0], - &endids[0]); + sizeof ids/sizeof ids[0], + &ids[0]); assert(ret == 1); } diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index 64838cc83..80afbcc39 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -134,33 +134,33 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(comb, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); + assert( ids != NULL ); - assert( num_end_ids == matches[i].count ); + assert( count == matches[i].count ); - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < num_end_ids; j++) { - assert( end_ids[j] == matches[i].endid[j] ); + for (j=0; j < count; j++) { + assert( ids[j] == matches[i].endid[j] ); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index 2fce9ea92..d6a713843 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -134,12 +134,12 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(comb, matches[i].s, NULL, &ids, &count); printf("match %zu, \"%s\", should_match=%d, count=%zu, endids={ %u, %u }\n", i, matches[i].s, matches[i].should_match, matches[i].count, @@ -149,22 +149,22 @@ int main(void) size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); + assert( ids != NULL ); - assert( num_end_ids == matches[i].count ); + assert( count == matches[i].count ); - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < num_end_ids; j++) { - assert( end_ids[j] == matches[i].endid[j] ); + for (j=0; j < count; j++) { + assert( ids[j] == matches[i].endid[j] ); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index 77976991c..34b261118 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -20,33 +20,33 @@ struct state_info { fsm_state_t state; unsigned count; - fsm_end_id_t endids[2]; + fsm_end_id_t ids[2]; }; /* remap 1 -> 512, 2 -> 1024 */ static int -endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *endids, size_t *num_written, void *opaque) +endid_remap_func(fsm_state_t state, size_t count, fsm_end_id_t *ids, size_t *num_written, void *opaque) { size_t i; (void)state; (void)opaque; - assert(endids != NULL); - for (i=0; i < num_ids; i++) { - fsm_end_id_t orig = endids[i]; + assert(ids != NULL); + for (i=0; i < count; i++) { + fsm_end_id_t orig = ids[i]; - switch (endids[i]) { - case 1: endids[i] = 512; break; - case 2: endids[i] = 1024; break; + switch (ids[i]) { + case 1: ids[i] = 512; break; + case 2: ids[i] = 1024; break; default: break; } printf("remap: state %u id %u -> %u\n", - (unsigned)state, (unsigned)orig, (unsigned)endids[i]); + (unsigned)state, (unsigned)orig, (unsigned)ids[i]); } - *num_written = num_ids; + *num_written = count; return 1; } @@ -91,7 +91,7 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; + fsm_end_id_t ids[2] = {0,0}; size_t count; int ret; @@ -102,8 +102,8 @@ int main(void) ret = fsm_endid_get( fsm, state_ind, - sizeof endids / sizeof endids[0], - &endids[0]); + sizeof ids / sizeof ids[0], + &ids[0]); assert(ret == 1); @@ -111,13 +111,13 @@ int main(void) info[ninfo].count = count; if (count == 1) { - assert(endids[0] == 1 || endids[0] == 2); - info[ninfo].endids[0] = endids[0]; + assert(ids[0] == 1 || ids[0] == 2); + info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + qsort(&ids[0], count, sizeof ids[0], cmp_endids); + assert(ids[0] == 1 && ids[1] == 2); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; @@ -132,7 +132,7 @@ int main(void) for (state_ind = 0, info_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; + fsm_end_id_t ids[2] = {0,0}; size_t count, ind; int ret; @@ -146,18 +146,18 @@ int main(void) ret = fsm_endid_get( fsm, state_ind, - sizeof endids / sizeof endids[0], - &endids[0]); + sizeof ids / sizeof ids[0], + &ids[0]); assert(ret == 1); if (count > 1) { - qsort(&endids[0], count, sizeof endids[0], cmp_endids); + qsort(&ids[0], count, sizeof ids[0], cmp_endids); } for (ind=0; ind < count; ind++) { - fsm_end_id_t expected = info[info_ind].endids[ind]; - switch (info[info_ind].endids[ind]) { + fsm_end_id_t expected = info[info_ind].ids[ind]; + switch (info[info_ind].ids[ind]) { case 1: expected = 512; break; case 2: expected = 1024; break; default: @@ -165,8 +165,8 @@ int main(void) break; } - printf("state %u, id %u, expected %u\n", (unsigned)state_ind, endids[ind], expected); - assert(endids[ind] == expected); + printf("state %u, id %u, expected %u\n", (unsigned)state_ind, ids[ind], expected); + assert(ids[ind] == expected); } info_ind++; diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index 451cf342a..6508308eb 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -20,40 +20,40 @@ struct state_info { fsm_state_t state; unsigned count; - fsm_end_id_t endids[2]; + fsm_end_id_t ids[2]; }; /* remap 1 -> 7, 2 -> 9, both 1&2 -> 3 */ static int -endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *endids, size_t *num_written, void *opaque) +endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *ids, size_t *num_written, void *opaque) { (void)state; (void)opaque; - assert(endids != NULL); + assert(ids != NULL); assert(num_ids > 0 && num_ids <= 2); if (num_ids > 1) { - endids[0] = 3; + ids[0] = 3; *num_written = 1; printf("remap: state %u with %zu endids, set to one endid, value %u\n", - (unsigned)state, num_ids, (unsigned)endids[0]); + (unsigned)state, num_ids, (unsigned)ids[0]); return 1; } if (num_ids == 1) { - fsm_end_id_t orig = endids[0]; + fsm_end_id_t orig = ids[0]; - switch (endids[0]) { - case 1: endids[0] = 7; break; - case 2: endids[0] = 9; break; + switch (ids[0]) { + case 1: ids[0] = 7; break; + case 2: ids[0] = 9; break; default: break; } printf("remap: state %u id %u -> %u\n", - (unsigned)state, (unsigned)orig, (unsigned)endids[0]); + (unsigned)state, (unsigned)orig, (unsigned)ids[0]); } *num_written = num_ids; @@ -103,7 +103,7 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; + fsm_end_id_t ids[2] = {0,0}; size_t count; int ret; @@ -114,8 +114,8 @@ int main(void) ret = fsm_endid_get( fsm, state_ind, - sizeof endids / sizeof endids[0], - &endids[0]); + sizeof ids / sizeof ids[0], + &ids[0]); assert(ret == 1); @@ -123,13 +123,13 @@ int main(void) info[ninfo].count = count; if (count == 1) { - assert(endids[0] == 1 || endids[0] == 2); - info[ninfo].endids[0] = endids[0]; + assert(ids[0] == 1 || ids[0] == 2); + info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + qsort(&ids[0], count, sizeof ids[0], cmp_endids); + assert(ids[0] == 1 && ids[1] == 2); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; @@ -144,7 +144,7 @@ int main(void) for (state_ind = 0, info_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endid = 0; + fsm_end_id_t id = 0; fsm_end_id_t expected; size_t count; int ret; @@ -158,22 +158,22 @@ int main(void) fsm, state_ind, 1, - &endid); + &id); assert(ret == 1); if (info[info_ind].count == 2) { expected = 3; - } else if (info[info_ind].endids[0] == 1) { + } else if (info[info_ind].ids[0] == 1) { expected = 7; - } else if (info[info_ind].endids[0] == 2) { + } else if (info[info_ind].ids[0] == 2) { expected = 9; } else { assert( ! "unexpected value" ); } - printf("state %u, id %u, expected %u\n", (unsigned)state_ind, endid, expected); - assert(endid == expected);; + printf("state %u, id %u, expected %u\n", (unsigned)state_ind, id, expected); + assert(id == expected);; info_ind++; } diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index 40acaac8b..6b01d5f15 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -20,44 +20,44 @@ struct state_info { fsm_state_t state; unsigned count; - fsm_end_id_t endids[2]; + fsm_end_id_t ids[2]; }; /* remap 1 -> 7, 2 -> 9, both 1&2 -> 3,3 */ static int -endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *endids, size_t *num_written, void *opaque) +endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *ids, size_t *num_written, void *opaque) { (void)state; (void)opaque; - assert(endids != NULL); + assert(ids != NULL); assert(num_ids > 0 && num_ids <= 2); if (num_ids > 1) { size_t i; for (i = 0; i < num_ids; i++) { - endids[i] = 3; + ids[i] = 3; } *num_written = num_ids; printf("remap: state %u with %zu endids, set to %zu endid, value %u\n", - (unsigned)state, num_ids, *num_written, (unsigned)endids[0]); + (unsigned)state, num_ids, *num_written, (unsigned) ids[0]); return 1; } if (num_ids == 1) { - fsm_end_id_t orig = endids[0]; + fsm_end_id_t orig = ids[0]; - switch (endids[0]) { - case 1: endids[0] = 7; break; - case 2: endids[0] = 9; break; + switch (ids[0]) { + case 1: ids[0] = 7; break; + case 2: ids[0] = 9; break; default: break; } printf("remap: state %u id %u -> %u\n", - (unsigned)state, (unsigned)orig, (unsigned)endids[0]); + (unsigned)state, (unsigned)orig, (unsigned) ids[0]); } *num_written = num_ids; @@ -107,7 +107,7 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; + fsm_end_id_t ids[2] = {0,0}; size_t count; int ret; @@ -118,8 +118,8 @@ int main(void) ret = fsm_endid_get( fsm, state_ind, - sizeof endids / sizeof endids[0], - &endids[0]); + sizeof ids / sizeof ids[0], + &ids[0]); assert(ret == 1); @@ -127,13 +127,13 @@ int main(void) info[ninfo].count = count; if (count == 1) { - assert(endids[0] == 1 || endids[0] == 2); - info[ninfo].endids[0] = endids[0]; + assert(ids[0] == 1 || ids[0] == 2); + info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + qsort(&ids[0], count, sizeof ids[0], cmp_endids); + assert(ids[0] == 1 && ids[1] == 2); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; @@ -168,9 +168,9 @@ int main(void) if (info[info_ind].count == 2) { expected = 3; - } else if (info[info_ind].endids[0] == 1) { + } else if (info[info_ind].ids[0] == 1) { expected = 7; - } else if (info[info_ind].endids[0] == 2) { + } else if (info[info_ind].ids[0] == 2) { expected = 9; } else { assert( ! "unexpected value" ); diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index dedd750b0..86a2b28c8 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -24,7 +24,7 @@ int main(void) const char *s1, *s2; int ret; - fsm_end_id_t all_endids[2]; + fsm_end_id_t all_ids[2]; unsigned nstates, state_ind; s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); @@ -69,10 +69,10 @@ int main(void) nstates = fsm_countstates(comb); // all end states should have BOTH endids - memset(all_endids, 0, sizeof all_endids); + memset(all_ids, 0, sizeof all_ids); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { - fsm_end_id_t endids[3] = {0,0,0}; + fsm_end_id_t ids[3] = {0,0,0}; size_t count; int ret; @@ -85,13 +85,13 @@ int main(void) ret = fsm_endid_get( comb, state_ind, - sizeof endids / sizeof endids[0], - &endids[0]); + sizeof ids / sizeof ids[0], + &ids[0]); assert(ret == 1); - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2 && endids[2] == 4); + qsort(&ids[0], count, sizeof ids[0], cmp_endids); + assert(ids[0] == 1 && ids[1] == 2 && ids[2] == 4); } } diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index 38328f5a4..33a872155 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -20,7 +20,7 @@ struct state_info { fsm_state_t state; unsigned count; - fsm_end_id_t endids[2]; + fsm_end_id_t ids[2]; }; /* use fsm_increndids to change the endids before a union/intersection */ @@ -71,7 +71,7 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; + fsm_end_id_t ids[2] = {0,0}; size_t count; int ret; @@ -82,8 +82,8 @@ int main(void) ret = fsm_endid_get( fsm, state_ind, - sizeof endids / sizeof endids[0], - &endids[0]); + sizeof ids / sizeof ids[0], + &ids[0]); assert(ret == 1); @@ -91,13 +91,13 @@ int main(void) info[ninfo].count = count; if (count == 1) { - assert(endids[0] == 11 || endids[0] == 12); - info[ninfo].endids[0] = endids[0]; + assert(ids[0] == 11 || ids[0] == 12); + info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&endids[0], count, sizeof endids[0], cmp_endids); - assert(endids[0] == 11 && endids[1] == 12); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + qsort(&ids[0], count, sizeof ids[0], cmp_endids); + assert(ids[0] == 11 && ids[1] == 12); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; diff --git a/tests/endids/utils.c b/tests/endids/utils.c index a5d96f020..882df92c4 100644 --- a/tests/endids/utils.c +++ b/tests/endids/utils.c @@ -4,7 +4,7 @@ #include int -match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **endids_ptr, size_t *count_ptr) +match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **ids_ptr, size_t *count_ptr) { fsm_state_t end = 0; int ret; @@ -21,25 +21,25 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end if (count > 0) { int ret; - fsm_end_id_t *endids = calloc(count, sizeof *endids); - if (endids == NULL) { + fsm_end_id_t *ids = calloc(count, sizeof *ids); + if (ids == NULL) { return -1; } - ret = fsm_endid_get(fsm, end, count, endids); + ret = fsm_endid_get(fsm, end, count, ids); if (ret == 0) { - free(endids); + free(ids); errno = EINVAL; return -1; } - if (endids_ptr != NULL) { - *endids_ptr = endids; + if (ids_ptr != NULL) { + *ids_ptr = ids; if (count_ptr != NULL) { *count_ptr = count; } } else { - free(endids); + free(ids); } } } diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index 53fc89602..124c8be1b 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -58,9 +58,9 @@ gtest_matches_cb(const struct fsm *fsm, m->found = true; #define ID_BUF_COUNT 1 - fsm_end_id_t id_buf[ID_BUF_COUNT]; + fsm_end_id_t ids[ID_BUF_COUNT]; int gres = fsm_endid_get(fsm, - end_state, ID_BUF_COUNT, id_buf); + end_state, ID_BUF_COUNT, ids); if (gres != 1) { fprintf(stderr, @@ -69,9 +69,9 @@ gtest_matches_cb(const struct fsm *fsm, return FSM_GENERATE_MATCHES_CB_RES_HALT; } - if (fsm_endid_count(fsm, end_state) != 1 || id_buf[0] != m_i) { + if (fsm_endid_count(fsm, end_state) != 1 || ids[0] != m_i) { fprintf(stderr, "ERROR: endid mismatch, expected %zu, got %u\n", - m_i, id_buf[0]); + m_i, ids[0]); return FSM_GENERATE_MATCHES_CB_RES_HALT; } diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 59d972512..50cc21be0 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -12,7 +12,7 @@ static struct fsm_options opt; #define MAX_INPUTS 100 -static fsm_end_id_t id_buf[MAX_INPUTS]; +static fsm_end_id_t ids[MAX_INPUTS]; int run_test(const char **strings) @@ -48,11 +48,11 @@ run_test(const char **strings) assert(res > 0); /* match */ int eres = fsm_endid_get(fsm, end, - MAX_INPUTS, id_buf); + MAX_INPUTS, ids); assert(eres == 1); bool found = false; for (size_t i = 0; i < fsm_endid_count(fsm, end); i++) { - if (id_buf[i] == id) { + if (ids[i] == id) { found = true; break; } From a39f723d16509d5fe17486463118153acf93dffe Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 8 May 2024 14:38:44 +0100 Subject: [PATCH 153/373] First cut at introducing rx, a tool to compile a set of regular expressions. --- Makefile | 1 + src/rx/Makefile | 14 + src/rx/main.c | 1184 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1199 insertions(+) create mode 100644 src/rx/Makefile create mode 100644 src/rx/main.c diff --git a/Makefile b/Makefile index 3613bc941..997499f98 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ SUBDIR += src/libre/print SUBDIR += src/libre SUBDIR += src/fsm SUBDIR += src/re +SUBDIR += src/rx SUBDIR += src/retest SUBDIR += src/lx/print SUBDIR += src/lx diff --git a/src/rx/Makefile b/src/rx/Makefile new file mode 100644 index 000000000..2facbfd6c --- /dev/null +++ b/src/rx/Makefile @@ -0,0 +1,14 @@ +.include "../../share/mk/top.mk" + +SRC += src/rx/main.c + +PROG += rx + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${BUILD}/bin/rx: ${BUILD}/lib/${lib:R}.a +.endfor + +.for src in ${SRC:Msrc/rx/*.c} +${BUILD}/bin/rx: ${BUILD}/${src:R}.o +.endfor + diff --git a/src/rx/main.c b/src/rx/main.c new file mode 100644 index 000000000..3bcd84ddf --- /dev/null +++ b/src/rx/main.c @@ -0,0 +1,1184 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#define _POSIX_C_SOURCE 200809L + +#ifdef __linux__ +#include +#endif + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct literal { + const void *p; + size_t n; + fsm_end_id_t id; +}; + +// TODO: no need to be global now +static size_t declined_count, general_count; +static size_t literals_none_count, literals_left_count, literals_right_count, literals_both_count; +static fsm_end_id_t *declined; +static fsm_end_id_t *general; + +// XXX: xalloc()ing these conservatively is large, i don't like it +static struct literal *literals_none; +static struct literal *literals_left; +static struct literal *literals_right; +static struct literal *literals_both; + +static void * +xalloc(size_t size) +{ + void *p; + + p = malloc(size); + if (p == NULL) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + return p; +} +static char * +xstrndup(const char *s, size_t n) +{ + char *new = strndup(s, n); + if (new == NULL) { + perror("strndup"); + exit(EXIT_FAILURE); + } + + return new; +} + +static bool +permitted_chars(const char *p, size_t n, const char *accept, const char *reject) +{ + char *s; + bool r = true; + + assert(p != NULL); + assert(accept != NULL || reject != NULL); + + /* + * p,n is a slice into a \n-terminated buffer, so we can't strcspn() here. + * We could provide our own implementation (i.e. memcspn), but until it's + * actually a problem I'd prefer to allocate and \0-terminate here just + * for the sake of using strcspn(). + */ + s = xstrndup(p, n); + + if (accept != NULL) { + r &= s[strspn(s, accept)] == '\0'; + } + + if (reject != NULL) { + r &= s[strcspn(s, reject)] == '\0'; + } + + free(s); + + return r; +} + +int +nlgetc(void *opaque) +{ + const char **s = opaque; + char c; + + assert(s != NULL); + assert(*s != NULL); + + c = **s; + + /* skip the newline too */ + (*s)++; + + if (c == '\n') { + return EOF; + } + + return (unsigned char) c; +} + +static void +append_id(fsm_end_id_t *a, size_t *count, fsm_end_id_t id) +{ + assert(a != NULL); + + a[*count] = id; + (*count)++; +} + +static void +append_literal(struct literal *a, size_t *count, fsm_end_id_t id, const char *p, size_t n) +{ + assert(a != NULL); + assert(p != NULL); + + a[*count].id = id; + a[*count].p = p; + a[*count].n = n; + (*count)++; +} + +/* ^[abc..]*$ */ +// XXX: this interface doesn't allow us to have \0 in the character set +static struct fsm * +intersect_charset(const struct fsm_options *opt, bool show_stats, + const char *charset, struct fsm *b) +{ + struct fsm *a, *q; + + if (charset == NULL) { + return b; + } + + /* + * Since intersection is destructive, there's no point in making + * the charset FSM in advance and then fsm_clone()ing it. + * We may as well just make a new one each time. + */ + { + fsm_state_t state; + + // TODO: pass .statealloc explicitly, we know it's 1. the default is overkill + a = fsm_new(opt); + if (a == NULL) { + perror("fsm_new"); + exit(EXIT_FAILURE); + } + + if (!fsm_addstate(a, &state)) { + perror("fsm_addstate"); + exit(EXIT_FAILURE); + } + + for (const char *p = charset; *p != '\0'; p++) { + if (!fsm_addedge_literal(a, state, state, *p)) { + perror("fsm_addedge_literal"); + exit(EXIT_FAILURE); + } + } + + fsm_setend(a, state, true); + fsm_setstart(a, state); + } + + assert(fsm_all(a, fsm_isdfa)); + assert(fsm_all(b, fsm_isdfa)); + + size_t pre = fsm_countstates(b); + if (show_stats) { + fprintf(stderr, "pre-intersection dfa: %zu states\n", pre); + } + + /* + * Here I would call: + * + * q = fsm_walk2(a, b, FSM_WALK2_BOTH, FSM_WALK2_BOTH); + * + * This is equivalent to fsm_intersect(). fsm_intersect() asserts that + * both operands are DFA at runtime. But in this program we know this + * empirically from our callers. So we would call fsm_walk2() directly + * to avoid needlessly running the DFA predicate in DNDEBUG builds. + * + * This is intersection implemented by walking sets of states through + * both FSM simultaneously, as described by Hopcroft, Motwani and Ullman + * (2001, 2nd ed.) 4.2, Closure under Intersection. + * + * The intersection of two FSM consists of only those items which + * are present in _BOTH. + * + * Unfortunately fsm_walk2() isn't in the public API, so we don't do that. + */ + q = fsm_intersect(a, b); + if (q == NULL) { + perror("fsm_intersect"); + exit(EXIT_FAILURE); + } + + /* walking two DFA produces a DFA */ + assert(fsm_all(q, fsm_isdfa)); + + size_t post = fsm_countstates(q); + if (show_stats) { + fprintf(stderr, "post-intersection dfa: %zu states\n", post); + } + + /* intersecting with the charset FSM should never introduce new states */ + assert(post <= pre); + + return q; +} + +static void +categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags flags, + bool strict, bool verbose, + const char *charset, const char *reject, fsm_end_id_t id, const char *s) +{ + enum re_literal_category category; + struct re_err err; + char *lit_s; + size_t lit_n; + int r; + + /* + * -1: Error + * 1: Literal, *s may or may not be NULL (i.e. for an empty string), *n >= 0 + * and *category is set. + */ + const char *q = s; + r = re_is_literal(dialect, nlgetc, &q, opt, flags, &err, &category, + &lit_s, &lit_n); + + /* unsupported */ + if (r == -1 && err.e == RE_EUNSUPPORTED) { + if (verbose) { + fprintf(stderr, "declined (unsupported) #%u: /%.*s/\n", + id, (int) strcspn(s, "\n"), s); + } + + append_id(declined, &declined_count, id); + return; + } + + /* parse error */ +// we consider a parse error here a failure, rather than declining the pattern +// TOOD: pass cli flag to make this non-fatal, quietly decline. also for empty strings and unsatisfiable expressions + if (r == -1) { + if (!strict) { + if (verbose) { +// XXX: rephrase error. also re_perror() for this too + fprintf(stderr, "XXX: would decline (parse error) #%u: /%.*s/\n", + id, (int) strcspn(s, "\n"), s); + } + + append_id(declined, &declined_count, id); + return; + } + + const char *p; + + /* s is \n-terminated per pattern. + * We allocate here so we can \0-terminate for re_perror() */ + p = xstrndup(s, strcspn(s, "\n")); + re_perror(dialect, &err, NULL, p); + free((void *) p); + + exit(EXIT_FAILURE); + } + + /* empty string */ + if (r == 1 && lit_n == 0) { + fprintf(stdout, "re_is_literal: /%.*s/ regex matches empty string only\n", + (int) strcspn(s, "\n"), s); +// TODO: cli option to decline instead + exit(EXIT_FAILURE); + } + + /* unsatisfiable */ + if (r == 1 && category == RE_LITERAL_UNSATISFIABLE) { + fprintf(stdout, "re_is_literal: /%.*s/ regex is unsatisfiable\n", + (int) strcspn(s, "\n"), s); +// TODO: cli option to decline instead + exit(EXIT_FAILURE); + } + + /* is a literal */ + if (r == 1) { + assert(lit_n >= 0); + assert(lit_s != NULL); + + for (size_t i = 0; i < lit_n; i++) { + if (lit_s[i] == '\0') { +// TODO: cli option to decline instead + fprintf(stderr, "re_is_literal: '%.*s' literal contains \\0\n", + (int) strcspn(s, "\n"), s); + exit(EXIT_FAILURE); + } + } + + if (category == RE_LITERAL_ANCHOR_END || category == RE_LITERAL_ANCHOR_BOTH) { + /* + * This newline is appended in ast_is_literal() as an attempt to + * account for the PCRE2_DOTALL behaviour. I'm not interested in + * newlines for this program (optional or legitimate), + * so I'm just cutting it off. + */ +// TODO: maybe only if it's not in the character set? + if (lit_n > 0 && lit_s[lit_n - 1] == '\n') { + lit_n--; + } + } + + if (verbose) { + fprintf(stderr, "literal #%u '%.*s'\n", id, (int) lit_n, lit_s); + } + +// TODO: explain reject does not apply + if (!permitted_chars(lit_s, lit_n, charset, NULL)) { +// TODO: cli option to decline instead +//fprintf(stderr, "declined literal (charset) #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); + append_id(declined, &declined_count, id); + return; + } + +//fprintf(stderr, "re_is_literal found: '%.*s'\n", (int) lit_n, lit_s); + + switch (category) { + case RE_LITERAL_UNANCHORED: + append_literal(literals_none, &literals_none_count, id, lit_s, lit_n); + break; + + case RE_LITERAL_ANCHOR_START: + append_literal(literals_left, &literals_left_count, id, lit_s, lit_n); + break; + + case RE_LITERAL_ANCHOR_END: + append_literal(literals_right, &literals_right_count, id, lit_s, lit_n); + break; + + case RE_LITERAL_ANCHOR_BOTH: + append_literal(literals_both, &literals_both_count, id, lit_s, lit_n); + break; + + default: + assert(!"unreached"); + abort(); + } + + return; + } + + /* + * Reject regexps that contain characters that may be expensive. + * + * If the regex contains a character that's not in the charset, + * we can also save some compilation time and decline it up front. + * + * This isn't perfect, for example consider hex encoded values. + * My design assumption here is that it's okay to pre-filter strings + * down to a character set. This means /./ doesn't match any character, + * it means any character within the charset. + * + * So we're relying on that here, and this is just a best-effort help. + */ + if (!permitted_chars(s, strcspn(s, "\n"), charset, reject)) { +//fprintf(stderr, "declined general (charset or reject) #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); + append_id(declined, &declined_count, id); + return; + } + +//fprintf(stderr, "general #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); + /* not a literal */ + append_id(general, &general_count, id); +} + +/* + * This is re_strings() but for an array of struct literal + */ +static struct fsm * +literal_strings(const struct fsm_options *opt, + const struct literal a[], size_t n, enum re_strings_flags flags) +{ + struct re_strings *g; + struct fsm *fsm; + + assert(a != NULL); + + g = re_strings_new(); + if (g == NULL) { + return NULL; + perror("re_strings_new"); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < n; i++) { + if (!re_strings_add_raw(g, a[i].p, a[i].n, & (fsm_end_id_t) { a[i].id })) { + perror("re_strings_add_raw"); + exit(EXIT_FAILURE); + } + } + + fsm = re_strings_build(g, opt, flags); + re_strings_free(g); + +// XXX: this fsm has unreachable states boo +//fsm_print_dot(stderr, fsm); + + /* this is already a DFA, courtesy of re_strings() */ + assert(fsm_all(fsm, fsm_isdfa)); + + return fsm; +} + +static struct fsm * +build_literals_fsm(const struct fsm_options *opt, bool show_stats, + const char *charset, + struct literal *a, size_t n, enum re_strings_flags flags) +{ + struct fsm *fsm; + + fsm = literal_strings(opt, a, n, flags); +// XXX: we do produce an empty fsm for 0 literals assert(!fsm_empty(fsm)); + + for (size_t i = 0; i < n; i++) { +// TODO: when we switch to libfsm alloc hooks, check if re_is_literal() calls malloc proper or not for *lit_s + free((void *) a[i].p); + } +// TODO: also free a[j].a when we malloc for the 20k-element arrays + + fsm = intersect_charset(opt, show_stats, charset, fsm); + if (fsm == NULL) { + perror("intersect_charset"); + exit(EXIT_FAILURE); + } + + /* We don't minimise here because this fsm has multiple endids, + * and the resulting FSM would be very similar to the current DFA */ + + if (fsm_countstates(fsm) == 0) { + fprintf(stderr, "literals produced empty intersection\n"); + exit(EXIT_FAILURE); + } + +#ifndef NDEBUG +// TODO: could test to see that the fsm isn't any different. for general regexps, lots will be, because of /./ +// TODO: but literal strings should be no different. except for the anchoring +#endif + + return fsm; +} + +static struct fsm * +build_regex_fsm(const struct fsm_options *opt, bool show_stats, + const char *charset, + enum re_dialect dialect, bool strict, + const char *s, enum re_flags flags, fsm_end_id_t id) +{ + struct fsm *fsm; + struct re_err err; + + assert(s != NULL); + +// TODO: want to compile these as if anchored, then do the unanchored parts during unioning +// then we can also construct the unanchored parts from the charset + + const char *q = s; +// TODO: RE_SINGLE here? i think it shouldn't have any effect when \n isn't present in the charset + fsm = re_comp(dialect, nlgetc, &q, opt, flags, &err); + + if (strict && fsm == NULL) { + const char *p; + + /* patterns[] is \n-terminated per pattern. + * We allocate here so we can \0-terminate for re_perror() */ + p = xstrndup(s, strcspn(s, "\n")); + re_perror(dialect, &err, NULL, p); + exit(EXIT_FAILURE); + } + + if (fsm == NULL && err.e == RE_EUNSUPPORTED) { + return NULL; + } + + /* + * An unsatisfiable regex compiles to an FSM that matches nothing, + * so that unioning it with other regexes will still work. + */ + if (strict && fsm_empty(fsm)) { + fprintf(stdout, "re_comp: /%.*s/ regex is unsatisfiable\n", + (int) strcspn(s, "\n"), s); + exit(EXIT_FAILURE); + } + +// TODO: note we don't minimise rather than determinise, explain why + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(EXIT_FAILURE); + } + + if (!fsm_setendid(fsm, id)) { + perror("fsm_setendid"); + exit(EXIT_FAILURE); + } + + fsm = intersect_charset(opt, show_stats, charset, fsm); + if (fsm == NULL) { + perror("intersect_charset"); + exit(EXIT_FAILURE); + } + +// XXX: unsure why this gives 0 states, even with endids +// TODO: explain we minimise again after intersect_charset because this fsm has a single endid, so it's (potentially) a worthwhile difference + +// TODO: we do minimise now after intersection, explain it's because only one end id + if (!fsm_minimise(fsm)) { + perror("fsm_minimise"); + exit(EXIT_FAILURE); + } + + return fsm; +} + +static int +endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, + const void *endleaf_opaque) +{ + assert(endleaf_opaque == NULL); + assert(endleaf_opaque == NULL); + + (void) f; + (void) endleaf_opaque; + + /* morally an assertion, but I feel better leaving this in for various user data */ + if (count == 0) { + fprintf(stderr, "no IDs attached to one accepting state\n"); + exit(EXIT_FAILURE); + } + + /* exactly one end id means no ambiguious patterns */ +// TODO: want to ensure at compile time that the endids set only has one element, reject ambigious patterns +// XXX: i'm not convinced we do. we could just as well emit code that returns a list of IDs +// and have a cli option to error about ambiguities +// TODO: this isn't strict checking. this is the opposite to -u +// TODO: give examples, lx style +#if 0 + if (count > 1) { +// TODO: explain this more clearly + fprintf(stderr, "ambigious patterns:"); + + for (fsm_end_id_t i = 0; i < ids->count; i++) { + fprintf(stderr, " %u", ids->ids[i]); + } + + fprintf(stderr, "\n"); + +// we consider this a compilation error for the purposes of this program + exit(EXIT_FAILURE); + } +#endif + + /* + * Here I would like to emit (static unsigned []) { 1, 2, 3 } + * but specifying a storage duration for compound literals + * is a compiler extension. + * So I'm emitting a static const variable declaration instead. + */ + + fprintf(f, "{\n"); + fprintf(f, "\t\tstatic const unsigned a[] = { "); + for (fsm_end_id_t i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, " };\n"); + fprintf(f, "\t\t*ids = a;\n"); + fprintf(f, "\t\t*count = %zu;\n", count); + fprintf(f, "\t\treturn 0;\n"); + fprintf(f, "\t}"); + +// TODO: override return -1 + + return 0; +} + +void +print_fsm_c(struct fsm *fsm, const char *prefix) +{ + const struct fsm_options *old; + struct fsm_options tmp; + + if (prefix == NULL) { + prefix = "fsm"; + } + + fprintf(stdout, "/* generated */\n"); + fprintf(stdout, "\n"); + + fprintf(stdout, "#include \n"); + fprintf(stdout, "#include \n"); + fprintf(stdout, "\n"); + + fprintf(stdout, "bool\n"); + fprintf(stdout, "%s_main(const char *s, size_t n,\n", prefix); + fprintf(stdout, "\tconst unsigned **ids, size_t *count)\n"); + fprintf(stdout, "{\n"); + fprintf(stdout, "\tconst char *b = s, *e = s + n;\n"); + + old = fsm_getoptions(fsm); + assert(old != NULL); + + tmp = *old; + tmp.fragment = true, + tmp.endleaf_opaque = NULL, + tmp.endleaf = endleaf_c, + + fsm_setoptions(fsm, &tmp); + fsm_print_vmc(stdout, fsm); + fsm_setoptions(fsm, old); + + fprintf(stdout, "}\n"); +} + +static enum re_dialect +dialect_name(const char *name) +{ + size_t i; + + struct { + const char *name; + enum re_dialect dialect; + } a[] = { + { "like", RE_LIKE }, + { "literal", RE_LITERAL }, + { "glob", RE_GLOB }, + { "native", RE_NATIVE }, + { "pcre", RE_PCRE }, + { "sql", RE_SQL } + }; + + assert(name != NULL); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + if (0 == strcmp(a[i].name, name)) { + return a[i].dialect; + } + } + + fprintf(stderr, "unrecognised regexp dialect \"%s\"; valid dialects are: ", name); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + i + 1 < sizeof a / sizeof *a ? ", " : "\n"); + } + + exit(EXIT_FAILURE); +} + +static +void usage(const char *name) +{ + + if (name == NULL) { + name = ""; + } else { + const char *p = strrchr(name, '/'); + name = p != NULL ? p + 1 : name; + } + + printf("usage: %s: [-ciQqv] [-C charset] [-r dialect] [-R reject] input-file [declined-file]\n", name); + printf(" %s -h\n", name); +} + +int +main(int argc, char *argv[]) +{ + bool quiet = false; + bool strict = false; + bool verbose = false; + bool show_stats = false; + + size_t general_limit = 0; + + const char *charset = NULL; + const char *input_file = NULL; + const char *declined_file = NULL; + + enum re_dialect dialect = RE_PCRE; +// TODO: we need to provide a way to disable RE_END_NL +// TODO: RE_SINGLE. also be careful, flags disrupt re_is_literal() + const enum re_flags flags = 0; // TODO: notably not RE_END_NL + + // TODO: explain we decline these at compile time because NFA->DFA is too costly + const char *reject = "*+{"; + + const struct fsm_options opt = { + .anonymous_states = true, + .consolidate_edges = true, + .comments = false, + .case_ranges = true, + .always_hex = false, + .group_edges = true, + .io = FSM_IO_PAIR, +// TODO: cli option to set io api + }; + +// XXX: actual character set in practice +charset = + /* RFC9110 field-value stuff... */ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwzyx" + "0123456789" + "!#$%&'*+-.^_`|~" + + /* ...plus some extras found in practice */ + "'\"/()@,:; "; + +// TODO: -l cli option for codegen lang +// TODO: optional extra argv[1] to output declined patterns (we could iteratively munch down on the list) +// TODO: cli option for numeric limit for general_count to decline patterns +// TODO: cli option for memory limit to decline patterns (implement via allocation hooks) +// TODO: cli option for worker pool threads +// TODO: make this a whole tool, alongside re(1) and fsm(1) +// TODO: manpage +// TODO: probably handle each of fsms[] in a separate thread, use worker pool +// TODO: cli flag to set prefix + + { + const char *name = argv[0]; + int c; + + while (c = getopt(argc, argv, "h" "C:cin:r:R:Qqv"), c != -1) { + switch (c) { + case 'C': + charset = optarg; + break; + + case 'c': + /* + * https://www.rfc-editor.org/rfc/rfc9110#name-tokens + * + * tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + * / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + * / DIGIT / ALPHA + * ; any VCHAR, except delimiters + * + * https://www.rfc-editor.org/rfc/rfc9110#appendix-A + * field-name = token + * token = 1*tchar + * + * So a valid header field-name has at least one character. + * But we don't enforce that here. + */ +// TODO: although we could, with the charset FSM as [a-z]+ rather than [a-z]* + charset = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwzyx" + "0123456789" + "!#$%&'*+-.^_`|~"; + break; + + case 'h': + usage(name); + exit(EXIT_SUCCESS); + + case 'i': + strict = true; + break; + + case 'n': + general_limit = atoi(optarg); + break; + + case 'R': + reject = optarg; + break; + + case 'r': + dialect = dialect_name(optarg); + break; + + case 'Q': + /* one-off stats on resource consumption */ + show_stats = true; + break; + + case 'q': + /* "quiet" applies to the generated code only */ + quiet = true; + break; + + case 'v': + /* "verbose" means showing information about every pattern and FSM, + * this is O(n) output to the length of the input file */ + verbose = true; + break; + + case '?': + default: + usage(name); + exit(EXIT_FAILURE); + } + } + + argc -= optind; + argv += optind; + + switch (argc) { + case 2: + declined_file = argv[1]; + case 1: + input_file = argv[0]; + break; + + default: + usage(name); + exit(EXIT_FAILURE); + } + } + +// TODO: error if charset contains \n (it also can't contain \0) + + if (show_stats) { + fprintf(stderr, "charset: [%s]\n", charset); + fprintf(stderr, "reject: [%s]\n", reject); + } + + struct stat sb; + char *addr; + int fd; + + size_t patterns_count; + + /* + * Phases: + * + * 1. per byte, delimiting + * 2. per pattern, categorization + * 3. per set of literals/general regex to FSMs + * 4. union FSMs, codegen + */ + + { + fd = open(input_file, O_RDONLY); + if (fd == -1) { + perror(input_file); + exit(EXIT_FAILURE); + } + + if (fstat(fd, &sb) == -1) { + perror(input_file); + exit(EXIT_FAILURE); + } + + addr = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (addr == MAP_FAILED) { + perror("mmap"); + exit(EXIT_FAILURE); + } + + /* we assume a trailing newline per pattern in the getc callback */ + if (sb.st_size == 0 || addr[sb.st_size - 1] != '\n') { + fprintf(stderr, "%s: missing newline at end of file\n", input_file); + exit(EXIT_FAILURE); + } + + /* + * Here we iterate per byte, finding the \n delimiter for patterns. + * Henceforth we iterate per pattern id, not per byte. + */ + patterns_count = 0; + for (const char *p = addr; p - addr < sb.st_size; p++) { + if (*p == '\n') { + patterns_count++; + } + } + + if (show_stats) { + fprintf(stderr, "patterns: %zu\n", patterns_count); + } + } + + if (general_limit == 0) { + general_limit = patterns_count; + } + +// we're keying id -> spelling +// we don't use patterns[] just for debugging, we use it for passing to re_comp(). but not to re_strings(), we pass lit_s there +// note this is \n-terminated, not \0-terminated. need %.*s for printing +// TODO: explain the index here is an endid + + const char **patterns; + + patterns = xalloc(patterns_count * sizeof *patterns); + + // TODO: explain we use the worst case of patterns_count for all these, prior to categorization + declined = xalloc(patterns_count * sizeof *declined); + general = xalloc(patterns_count * sizeof *general); + +// XXX: xalloc()ing these conservatively is large, i don't like it + literals_none = xalloc(patterns_count * sizeof *literals_none); + literals_left = xalloc(patterns_count * sizeof *literals_left); + literals_right = xalloc(patterns_count * sizeof *literals_right); + literals_both = xalloc(patterns_count * sizeof *literals_both); + + /* + * On end id numbering: The UI must be one pattern per line, because + * it's the only format that doesn't introduce a syntax with escaping. + * So that means we need to track the line number associated with each + * regex. And that means the line number is neccessarily the end id. + * So patterns[] is indexed by the end id. + * + * We have four whole FSMs from re_strings(), each potentially contains + * a set of end ids. And each regex in general[i] is a single end id. + * So general[i] is the index into patterns[]. + */ + + /* + * Categorize patterns + */ + { + const char *q = addr; + + for (fsm_end_id_t id = 0; id < patterns_count; id++) { + patterns[id] = q; + if (verbose) { + fprintf(stderr, "pattern[%u]: /%.*s/\n", + id, (int) strcspn(q, "\n"), patterns[id]); + } + + categorize(&opt, dialect, flags, strict, verbose, + charset, reject, id, patterns[id]); + + /* + * On error the parser may not have consumed all the input. + * So we can't use the current value of q because it potentially + * points somewhere mid-way along the regex. + */ + q += strcspn(patterns[id], "\n") + 1; + } + } + + /* + * This is conservative allocation, the declined list may still grow yet. + * We track actual usage in fsm_count. + */ + struct fsm **fsms = xalloc((patterns_count - declined_count) * sizeof *fsms); + size_t fsm_count = 0; + + /* + * The order of processing literals and regexps here is arbitrary. + * If this were a threaded program, each set of literals and each + * individual general regex could be handled in parallel. + * + * Also, the order of FSMs in the fsms[] array is not significant, + * they could be populated in any order. They're just going to get + * unioned anyway. + */ + + /* sets of literals */ + { + const struct { + struct literal *a; + size_t n; + enum re_strings_flags flags; + } a[] = { + { literals_none, literals_none_count, 0 }, + { literals_left, literals_left_count, RE_STRINGS_ANCHOR_LEFT }, + { literals_right, literals_right_count, RE_STRINGS_ANCHOR_RIGHT }, + { literals_both, literals_both_count, RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT }, + }; + + for (size_t j = 0; j < sizeof a / sizeof *a; j++) { + if (show_stats) { + fprintf(stderr, "literals[%zu].n = %zu\n", j, a[j].n); + } + + struct fsm *fsm = build_literals_fsm(&opt, show_stats, charset, + a[j].a, a[j].n, a[j].flags); + assert(fsm != NULL); + + fsms[fsm_count] = fsm; + fsm_count++; + } + } + + free(literals_none); + free(literals_left); + free(literals_right); + free(literals_both); + + /* individual regexps */ + { + for (size_t i = 0; i < general_count; i++) { + if (i > general_limit) { + append_id(declined, &declined_count, i); + continue; + } + + if (verbose) { + fprintf(stderr, "general[%zu]: #%u /%.*s/\n", + i, general[i], (int) strcspn(patterns[general[i]], "\n"), patterns[general[i]]); + } + + struct fsm *fsm = build_regex_fsm(&opt, show_stats, charset, + dialect, strict, + patterns[general[i]], flags, general[i]); + if (fsm == NULL) { + append_id(declined, &declined_count, i); + continue; + } + + fsms[fsm_count] = fsm; + fsm_count++; + } + + if (general_count > general_limit) { + general_count = general_limit; + } + } + + free(general); + + if (show_stats) { + fprintf(stderr, "declined: %zu patterns\n", + declined_count); + fprintf(stderr, "literals_none: %zu patterns, %u states\n", + literals_none_count, fsm_countstates(fsms[0])); + fprintf(stderr, "literals_left: %zu patterns, %u states\n", + literals_left_count, fsm_countstates(fsms[1])); + fprintf(stderr, "literals_right: %zu patterns, %u states\n", + literals_right_count, fsm_countstates(fsms[2])); + fprintf(stderr, "literals_both: %zu patterns, %u states\n", + literals_both_count, fsm_countstates(fsms[3])); + fprintf(stderr, "general: %zu patterns (limit %zu)\n", + general_count, general_limit); + } + + if (verbose) { + for (size_t i = 0; i < declined_count; i++) { + fprintf(stderr, "declined[%zu]: #%u /%.*s/\n", + i, declined[i], + (int) strcspn(patterns[declined[i]], "\n"), patterns[declined[i]]); + } + } + +// TODO: dump to file, handle these with pcre or such +// TODO: explain you're expected to disambiguate by pattern spelling, the endids are not relevant here + if (declined_file != NULL) { + FILE *f; + + f = fopen(declined_file, "w"); + if (f == NULL) { + perror(declined_file); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < declined_count; i++) { + fprintf(f, "%.*s\n", (int) strcspn(patterns[declined[i]], "\n"), patterns[declined[i]]); + } + + fclose(f); + } + + free(declined); + free(patterns); + + if (-1 == munmap(addr, sb.st_size)) { + perror("munmap"); + exit(EXIT_FAILURE); + } + + if (-1 == close(fd)) { + perror("close"); + exit(EXIT_FAILURE); + } + + if (show_stats) { + fprintf(stderr, "fsm_count = %zu FSMs prior to union\n", fsm_count); + } + + if (verbose) { + for (size_t i = 0; i < fsm_count; i++) { + fprintf(stderr, "fsm[%zu] = %u states\n", i, fsm_countstates(fsms[i])); + } + } + + /* + * Union down and codegen. + */ + { + struct fsm *fsm; + + /* + * fsm_union_array introduces epsilons. + * If we construct anchored DFA and deal with ^.* ourselves here, + * we can union union without introducing epislons. + * This should reduce pressure on fsm_determinise() below. + */ + + fsm = fsm_union_array(fsm_count, fsms, NULL); + if (fsm == NULL) { + perror("fsm_union_array"); + exit(EXIT_FAILURE); + } + + free(fsms); + + if (show_stats) { + fprintf(stderr, "nfa: %u states\n", fsm_countstates(fsm)); + } + + /* + * We determinise but do not minimise the single DFA, like in lx. + * This keeps end states (and therefore end ids) unique. But unlike lx, + * end ids don't overlap between patterns (because they represent lines + * in the input file). So there isn't an advantage to minimising here. + */ + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(EXIT_FAILURE); + } + + if (show_stats) { + fprintf(stderr, "dfa: %u states\n", fsm_countstates(fsm)); + } + + if (!quiet) { + print_fsm_c(fsm, opt.prefix); + } + + fsm_free(fsm); + } + +#ifdef __linux__ + if (show_stats) { + struct rusage ru; + + if (getrusage(RUSAGE_SELF, &ru) != 0) { + perror("getrusage"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "rusage.utime: %g\n", (double) ru.ru_utime.tv_sec + (double) ru.ru_utime.tv_usec / 1000000); + fprintf(stderr, "rusage.stime: %g\n", (double) ru.ru_stime.tv_sec + (double) ru.ru_stime.tv_usec / 1000000); + fprintf(stderr, "rusage.maxrss: %zu MiB\n", (size_t) ru.ru_maxrss / 1024); + } +#endif + +// TODO: also free lit_s +} + From 994390ecda721530fb2f0bfbfb36dda5ce12471e Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 10 May 2024 15:43:14 +0100 Subject: [PATCH 154/373] Move to structs for literal and id sets. --- src/rx/main.c | 183 ++++++++++++++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 79 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 3bcd84ddf..811f52cd4 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -41,17 +41,15 @@ struct literal { fsm_end_id_t id; }; -// TODO: no need to be global now -static size_t declined_count, general_count; -static size_t literals_none_count, literals_left_count, literals_right_count, literals_both_count; -static fsm_end_id_t *declined; -static fsm_end_id_t *general; +struct literal_set { + size_t count; + struct literal *a; +}; -// XXX: xalloc()ing these conservatively is large, i don't like it -static struct literal *literals_none; -static struct literal *literals_left; -static struct literal *literals_right; -static struct literal *literals_both; +struct id_set { + size_t count; + fsm_end_id_t *a; +}; static void * xalloc(size_t size) @@ -130,24 +128,26 @@ nlgetc(void *opaque) } static void -append_id(fsm_end_id_t *a, size_t *count, fsm_end_id_t id) +append_id(struct id_set *set, fsm_end_id_t id) { - assert(a != NULL); + assert(set != NULL); + assert(set->a != NULL); - a[*count] = id; - (*count)++; + set->a[set->count] = id; + set->count++; } static void -append_literal(struct literal *a, size_t *count, fsm_end_id_t id, const char *p, size_t n) +append_literal(struct literal_set *set, const char *p, size_t n, fsm_end_id_t id) { - assert(a != NULL); + assert(set != NULL); + assert(set->a != NULL); assert(p != NULL); - a[*count].id = id; - a[*count].p = p; - a[*count].n = n; - (*count)++; + set->a[set->count].id = id; + set->a[set->count].p = p; + set->a[set->count].n = n; + set->count++; } /* ^[abc..]*$ */ @@ -243,7 +243,12 @@ intersect_charset(const struct fsm_options *opt, bool show_stats, static void categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags flags, bool strict, bool verbose, - const char *charset, const char *reject, fsm_end_id_t id, const char *s) + const char *charset, const char *reject, fsm_end_id_t id, const char *s, + struct id_set *general, struct id_set *declined, + struct literal_set *literals_none, + struct literal_set *literals_left, + struct literal_set *literals_right, + struct literal_set *literals_both) { enum re_literal_category category; struct re_err err; @@ -251,6 +256,9 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags size_t lit_n; int r; + assert(general != NULL); + assert(declined != NULL); + /* * -1: Error * 1: Literal, *s may or may not be NULL (i.e. for an empty string), *n >= 0 @@ -267,7 +275,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags id, (int) strcspn(s, "\n"), s); } - append_id(declined, &declined_count, id); + append_id(declined, id); return; } @@ -282,7 +290,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags id, (int) strcspn(s, "\n"), s); } - append_id(declined, &declined_count, id); + append_id(declined, id); return; } @@ -348,7 +356,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags if (!permitted_chars(lit_s, lit_n, charset, NULL)) { // TODO: cli option to decline instead //fprintf(stderr, "declined literal (charset) #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); - append_id(declined, &declined_count, id); + append_id(declined, id); return; } @@ -356,19 +364,19 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags switch (category) { case RE_LITERAL_UNANCHORED: - append_literal(literals_none, &literals_none_count, id, lit_s, lit_n); + append_literal(literals_none, lit_s, lit_n, id); break; case RE_LITERAL_ANCHOR_START: - append_literal(literals_left, &literals_left_count, id, lit_s, lit_n); + append_literal(literals_left, lit_s, lit_n, id); break; case RE_LITERAL_ANCHOR_END: - append_literal(literals_right, &literals_right_count, id, lit_s, lit_n); + append_literal(literals_right, lit_s, lit_n, id); break; case RE_LITERAL_ANCHOR_BOTH: - append_literal(literals_both, &literals_both_count, id, lit_s, lit_n); + append_literal(literals_both, lit_s, lit_n, id); break; default: @@ -394,13 +402,13 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags */ if (!permitted_chars(s, strcspn(s, "\n"), charset, reject)) { //fprintf(stderr, "declined general (charset or reject) #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); - append_id(declined, &declined_count, id); + append_id(declined, id); return; } //fprintf(stderr, "general #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); /* not a literal */ - append_id(general, &general_count, id); + append_id(general, id); } /* @@ -408,7 +416,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags */ static struct fsm * literal_strings(const struct fsm_options *opt, - const struct literal a[], size_t n, enum re_strings_flags flags) + const struct literal *a, size_t n, enum re_strings_flags flags) { struct re_strings *g; struct fsm *fsm; @@ -444,16 +452,18 @@ literal_strings(const struct fsm_options *opt, static struct fsm * build_literals_fsm(const struct fsm_options *opt, bool show_stats, const char *charset, - struct literal *a, size_t n, enum re_strings_flags flags) + struct literal_set *set, enum re_strings_flags flags) { struct fsm *fsm; - fsm = literal_strings(opt, a, n, flags); + assert(set != NULL); + + fsm = literal_strings(opt, set->a, set->count, flags); // XXX: we do produce an empty fsm for 0 literals assert(!fsm_empty(fsm)); - for (size_t i = 0; i < n; i++) { + for (size_t i = 0; i < set->count; i++) { // TODO: when we switch to libfsm alloc hooks, check if re_is_literal() calls malloc proper or not for *lit_s - free((void *) a[i].p); + free((void *) set->a[i].p); } // TODO: also free a[j].a when we malloc for the 20k-element arrays @@ -749,7 +759,7 @@ charset = // TODO: -l cli option for codegen lang // TODO: optional extra argv[1] to output declined patterns (we could iteratively munch down on the list) -// TODO: cli option for numeric limit for general_count to decline patterns +// TODO: cli option for numeric limit for general.count to decline patterns // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: cli option for worker pool threads // TODO: make this a whole tool, alongside re(1) and fsm(1) @@ -925,15 +935,29 @@ charset = patterns = xalloc(patterns_count * sizeof *patterns); + // TODO: patterns_set, fsm_set + struct id_set declined; + struct id_set general; + struct literal_set literals_none; + struct literal_set literals_left; + struct literal_set literals_right; + struct literal_set literals_both; + // TODO: explain we use the worst case of patterns_count for all these, prior to categorization - declined = xalloc(patterns_count * sizeof *declined); - general = xalloc(patterns_count * sizeof *general); + declined.a = xalloc(patterns_count * sizeof *declined.a); +declined.count = 0; + general.a = xalloc(patterns_count * sizeof *general.a); +general.count = 0; // XXX: xalloc()ing these conservatively is large, i don't like it - literals_none = xalloc(patterns_count * sizeof *literals_none); - literals_left = xalloc(patterns_count * sizeof *literals_left); - literals_right = xalloc(patterns_count * sizeof *literals_right); - literals_both = xalloc(patterns_count * sizeof *literals_both); + literals_none.a = xalloc(patterns_count * sizeof *literals_none.a); +literals_none.count = 0; + literals_left.a = xalloc(patterns_count * sizeof *literals_left.a); +literals_left.count = 0; + literals_right.a = xalloc(patterns_count * sizeof *literals_right.a); +literals_right.count = 0; + literals_both.a = xalloc(patterns_count * sizeof *literals_both.a); +literals_both.count = 0; /* * On end id numbering: The UI must be one pattern per line, because @@ -943,8 +967,8 @@ charset = * So patterns[] is indexed by the end id. * * We have four whole FSMs from re_strings(), each potentially contains - * a set of end ids. And each regex in general[i] is a single end id. - * So general[i] is the index into patterns[]. + * a set of end ids. And each regex in general.a[i] is a single end id. + * So general.a[i] is the index into patterns[]. */ /* @@ -961,7 +985,9 @@ charset = } categorize(&opt, dialect, flags, strict, verbose, - charset, reject, id, patterns[id]); + charset, reject, id, patterns[id], + &general, &declined, + &literals_none, &literals_left, &literals_right, &literals_both); /* * On error the parser may not have consumed all the input. @@ -976,7 +1002,7 @@ charset = * This is conservative allocation, the declined list may still grow yet. * We track actual usage in fsm_count. */ - struct fsm **fsms = xalloc((patterns_count - declined_count) * sizeof *fsms); + struct fsm **fsms = xalloc((patterns_count - declined.count) * sizeof *fsms); size_t fsm_count = 0; /* @@ -992,23 +1018,22 @@ charset = /* sets of literals */ { const struct { - struct literal *a; - size_t n; + struct literal_set *set; enum re_strings_flags flags; } a[] = { - { literals_none, literals_none_count, 0 }, - { literals_left, literals_left_count, RE_STRINGS_ANCHOR_LEFT }, - { literals_right, literals_right_count, RE_STRINGS_ANCHOR_RIGHT }, - { literals_both, literals_both_count, RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT }, + { &literals_none, 0 }, + { &literals_left, RE_STRINGS_ANCHOR_LEFT }, + { &literals_right, RE_STRINGS_ANCHOR_RIGHT }, + { &literals_both, RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT }, }; for (size_t j = 0; j < sizeof a / sizeof *a; j++) { if (show_stats) { - fprintf(stderr, "literals[%zu].n = %zu\n", j, a[j].n); + fprintf(stderr, "literals[%zu].count = %zu\n", j, a[j].set->count); } struct fsm *fsm = build_literals_fsm(&opt, show_stats, charset, - a[j].a, a[j].n, a[j].flags); + a[j].set, a[j].flags); assert(fsm != NULL); fsms[fsm_count] = fsm; @@ -1016,29 +1041,29 @@ charset = } } - free(literals_none); - free(literals_left); - free(literals_right); - free(literals_both); + free(literals_none.a); + free(literals_left.a); + free(literals_right.a); + free(literals_both.a); /* individual regexps */ { - for (size_t i = 0; i < general_count; i++) { + for (size_t i = 0; i < general.count; i++) { if (i > general_limit) { - append_id(declined, &declined_count, i); + append_id(&declined, i); continue; } if (verbose) { fprintf(stderr, "general[%zu]: #%u /%.*s/\n", - i, general[i], (int) strcspn(patterns[general[i]], "\n"), patterns[general[i]]); + i, general.a[i], (int) strcspn(patterns[general.a[i]], "\n"), patterns[general.a[i]]); } struct fsm *fsm = build_regex_fsm(&opt, show_stats, charset, dialect, strict, - patterns[general[i]], flags, general[i]); + patterns[general.a[i]], flags, general.a[i]); if (fsm == NULL) { - append_id(declined, &declined_count, i); + append_id(&declined, i); continue; } @@ -1046,33 +1071,33 @@ charset = fsm_count++; } - if (general_count > general_limit) { - general_count = general_limit; + if (general.count > general_limit) { + general.count = general_limit; } } - free(general); + free(general.a); if (show_stats) { fprintf(stderr, "declined: %zu patterns\n", - declined_count); + declined.count); fprintf(stderr, "literals_none: %zu patterns, %u states\n", - literals_none_count, fsm_countstates(fsms[0])); + literals_none.count, fsm_countstates(fsms[0])); fprintf(stderr, "literals_left: %zu patterns, %u states\n", - literals_left_count, fsm_countstates(fsms[1])); + literals_left.count, fsm_countstates(fsms[1])); fprintf(stderr, "literals_right: %zu patterns, %u states\n", - literals_right_count, fsm_countstates(fsms[2])); + literals_right.count, fsm_countstates(fsms[2])); fprintf(stderr, "literals_both: %zu patterns, %u states\n", - literals_both_count, fsm_countstates(fsms[3])); + literals_both.count, fsm_countstates(fsms[3])); fprintf(stderr, "general: %zu patterns (limit %zu)\n", - general_count, general_limit); + general.count, general_limit); } if (verbose) { - for (size_t i = 0; i < declined_count; i++) { - fprintf(stderr, "declined[%zu]: #%u /%.*s/\n", - i, declined[i], - (int) strcspn(patterns[declined[i]], "\n"), patterns[declined[i]]); + for (size_t i = 0; i < declined.count; i++) { + fprintf(stderr, "declined.a[%zu]: #%u /%.*s/\n", + i, declined.a[i], + (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); } } @@ -1087,14 +1112,14 @@ charset = exit(EXIT_FAILURE); } - for (size_t i = 0; i < declined_count; i++) { - fprintf(f, "%.*s\n", (int) strcspn(patterns[declined[i]], "\n"), patterns[declined[i]]); + for (size_t i = 0; i < declined.count; i++) { + fprintf(f, "%.*s\n", (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); } fclose(f); } - free(declined); + free(declined.a); free(patterns); if (-1 == munmap(addr, sb.st_size)) { From 976313f709c8962939012a2128aec3e4e62bd5b9 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 10 May 2024 19:26:05 +0100 Subject: [PATCH 155/373] Fold together literal sets into an array indexed by flags. --- src/rx/main.c | 120 +++++++++++++++++++++++--------------------------- 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 811f52cd4..2cd08a238 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -150,6 +150,28 @@ append_literal(struct literal_set *set, const char *p, size_t n, fsm_end_id_t id set->count++; } +static enum re_strings_flags +literal_flags(enum re_literal_category category) +{ + switch (category) { + case RE_LITERAL_UNANCHORED: + return 0; + + case RE_LITERAL_ANCHOR_START: + return RE_STRINGS_ANCHOR_LEFT; + + case RE_LITERAL_ANCHOR_END: + return RE_STRINGS_ANCHOR_RIGHT; + + case RE_LITERAL_ANCHOR_BOTH: + return RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT; + + default: + assert(!"unreached"); + abort(); + } +} + /* ^[abc..]*$ */ // XXX: this interface doesn't allow us to have \0 in the character set static struct fsm * @@ -245,10 +267,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags bool strict, bool verbose, const char *charset, const char *reject, fsm_end_id_t id, const char *s, struct id_set *general, struct id_set *declined, - struct literal_set *literals_none, - struct literal_set *literals_left, - struct literal_set *literals_right, - struct literal_set *literals_both) + struct literal_set *literals) { enum re_literal_category category; struct re_err err; @@ -258,6 +277,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags assert(general != NULL); assert(declined != NULL); + assert(literals != NULL); /* * -1: Error @@ -323,7 +343,6 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags /* is a literal */ if (r == 1) { - assert(lit_n >= 0); assert(lit_s != NULL); for (size_t i = 0; i < lit_n; i++) { @@ -362,28 +381,10 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags //fprintf(stderr, "re_is_literal found: '%.*s'\n", (int) lit_n, lit_s); - switch (category) { - case RE_LITERAL_UNANCHORED: - append_literal(literals_none, lit_s, lit_n, id); - break; - - case RE_LITERAL_ANCHOR_START: - append_literal(literals_left, lit_s, lit_n, id); - break; - - case RE_LITERAL_ANCHOR_END: - append_literal(literals_right, lit_s, lit_n, id); - break; - - case RE_LITERAL_ANCHOR_BOTH: - append_literal(literals_both, lit_s, lit_n, id); - break; - - default: - assert(!"unreached"); - abort(); - } + enum re_strings_flags flags = literal_flags(category); + assert(flags >= 0 && flags <= 3); + append_literal(&literals[flags], lit_s, lit_n, id); return; } @@ -850,6 +851,7 @@ charset = switch (argc) { case 2: declined_file = argv[1]; + /* fallthrough */ case 1: input_file = argv[0]; break; @@ -938,26 +940,19 @@ charset = // TODO: patterns_set, fsm_set struct id_set declined; struct id_set general; - struct literal_set literals_none; - struct literal_set literals_left; - struct literal_set literals_right; - struct literal_set literals_both; + struct literal_set literals[4]; /* unanchored, left, right, both */ // TODO: explain we use the worst case of patterns_count for all these, prior to categorization declined.a = xalloc(patterns_count * sizeof *declined.a); -declined.count = 0; + declined.count = 0; general.a = xalloc(patterns_count * sizeof *general.a); -general.count = 0; + general.count = 0; // XXX: xalloc()ing these conservatively is large, i don't like it - literals_none.a = xalloc(patterns_count * sizeof *literals_none.a); -literals_none.count = 0; - literals_left.a = xalloc(patterns_count * sizeof *literals_left.a); -literals_left.count = 0; - literals_right.a = xalloc(patterns_count * sizeof *literals_right.a); -literals_right.count = 0; - literals_both.a = xalloc(patterns_count * sizeof *literals_both.a); -literals_both.count = 0; + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + literals[i].a = xalloc(patterns_count * sizeof *literals[i].a); + literals[i].count = 0; + } /* * On end id numbering: The UI must be one pattern per line, because @@ -986,8 +981,7 @@ literals_both.count = 0; categorize(&opt, dialect, flags, strict, verbose, charset, reject, id, patterns[id], - &general, &declined, - &literals_none, &literals_left, &literals_right, &literals_both); + &general, &declined, literals); /* * On error the parser may not have consumed all the input. @@ -1017,23 +1011,20 @@ literals_both.count = 0; /* sets of literals */ { - const struct { - struct literal_set *set; - enum re_strings_flags flags; - } a[] = { - { &literals_none, 0 }, - { &literals_left, RE_STRINGS_ANCHOR_LEFT }, - { &literals_right, RE_STRINGS_ANCHOR_RIGHT }, - { &literals_both, RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT }, + enum re_strings_flags flags[] = { + 0, + RE_STRINGS_ANCHOR_LEFT, + RE_STRINGS_ANCHOR_RIGHT, + RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT }; - for (size_t j = 0; j < sizeof a / sizeof *a; j++) { + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { if (show_stats) { - fprintf(stderr, "literals[%zu].count = %zu\n", j, a[j].set->count); + fprintf(stderr, "literals[%zu].count = %zu\n", i, literals[i].count); } struct fsm *fsm = build_literals_fsm(&opt, show_stats, charset, - a[j].set, a[j].flags); + &literals[i], flags[i]); assert(fsm != NULL); fsms[fsm_count] = fsm; @@ -1041,10 +1032,9 @@ literals_both.count = 0; } } - free(literals_none.a); - free(literals_left.a); - free(literals_right.a); - free(literals_both.a); + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + free(literals[i].a); + } /* individual regexps */ { @@ -1081,14 +1071,14 @@ literals_both.count = 0; if (show_stats) { fprintf(stderr, "declined: %zu patterns\n", declined.count); - fprintf(stderr, "literals_none: %zu patterns, %u states\n", - literals_none.count, fsm_countstates(fsms[0])); - fprintf(stderr, "literals_left: %zu patterns, %u states\n", - literals_left.count, fsm_countstates(fsms[1])); - fprintf(stderr, "literals_right: %zu patterns, %u states\n", - literals_right.count, fsm_countstates(fsms[2])); - fprintf(stderr, "literals_both: %zu patterns, %u states\n", - literals_both.count, fsm_countstates(fsms[3])); + fprintf(stderr, "literals (unanchored): %zu patterns, %u states\n", + literals[0].count, fsm_countstates(fsms[0])); + fprintf(stderr, "literals (left): %zu patterns, %u states\n", + literals[1].count, fsm_countstates(fsms[1])); + fprintf(stderr, "literals (right): %zu patterns, %u states\n", + literals[2].count, fsm_countstates(fsms[2])); + fprintf(stderr, "literals (both): %zu patterns, %u states\n", + literals[3].count, fsm_countstates(fsms[3])); fprintf(stderr, "general: %zu patterns (limit %zu)\n", general.count, general_limit); } From f234a1c49d10f600711dff92b73ee2c90bd4dbe2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 10 May 2024 23:53:12 +0100 Subject: [PATCH 156/373] Clarification. --- src/rx/main.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 2cd08a238..ac2a33111 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -483,8 +483,17 @@ build_literals_fsm(const struct fsm_options *opt, bool show_stats, } #ifndef NDEBUG -// TODO: could test to see that the fsm isn't any different. for general regexps, lots will be, because of /./ -// TODO: but literal strings should be no different. except for the anchoring + /* + * We could test to see that the fsm isn't any different. + * For general regexps, lots will be different because of + * mapping down /./ edges to the character set. + * + * But literal strings should be no different after the mapping, + * except for the implicit .* at either end for anchoring. + * + * So in order to assert here, we'd depend on constructing + * those .* long-hand at union time. + */ #endif return fsm; From e74a9d449d5766e8cf086ece8a23e00d138999a0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 27 May 2024 10:44:54 +0100 Subject: [PATCH 157/373] Port to fsm_intersect_charset. --- src/rx/main.c | 73 +++++++-------------------------------------------- 1 file changed, 10 insertions(+), 63 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index ac2a33111..60b73dab0 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -172,79 +172,26 @@ literal_flags(enum re_literal_category category) } } -/* ^[abc..]*$ */ -// XXX: this interface doesn't allow us to have \0 in the character set +/* this interface doesn't allow us to have \0 in the character set */ static struct fsm * -intersect_charset(const struct fsm_options *opt, bool show_stats, - const char *charset, struct fsm *b) +intersect_charset(bool show_stats, const char *charset, struct fsm *fsm) { - struct fsm *a, *q; + struct fsm *q; if (charset == NULL) { - return b; + return fsm; } - /* - * Since intersection is destructive, there's no point in making - * the charset FSM in advance and then fsm_clone()ing it. - * We may as well just make a new one each time. - */ - { - fsm_state_t state; - - // TODO: pass .statealloc explicitly, we know it's 1. the default is overkill - a = fsm_new(opt); - if (a == NULL) { - perror("fsm_new"); - exit(EXIT_FAILURE); - } - - if (!fsm_addstate(a, &state)) { - perror("fsm_addstate"); - exit(EXIT_FAILURE); - } - - for (const char *p = charset; *p != '\0'; p++) { - if (!fsm_addedge_literal(a, state, state, *p)) { - perror("fsm_addedge_literal"); - exit(EXIT_FAILURE); - } - } - - fsm_setend(a, state, true); - fsm_setstart(a, state); - } - - assert(fsm_all(a, fsm_isdfa)); - assert(fsm_all(b, fsm_isdfa)); + assert(fsm_all(fsm, fsm_isdfa)); - size_t pre = fsm_countstates(b); + size_t pre = fsm_countstates(fsm); if (show_stats) { fprintf(stderr, "pre-intersection dfa: %zu states\n", pre); } - /* - * Here I would call: - * - * q = fsm_walk2(a, b, FSM_WALK2_BOTH, FSM_WALK2_BOTH); - * - * This is equivalent to fsm_intersect(). fsm_intersect() asserts that - * both operands are DFA at runtime. But in this program we know this - * empirically from our callers. So we would call fsm_walk2() directly - * to avoid needlessly running the DFA predicate in DNDEBUG builds. - * - * This is intersection implemented by walking sets of states through - * both FSM simultaneously, as described by Hopcroft, Motwani and Ullman - * (2001, 2nd ed.) 4.2, Closure under Intersection. - * - * The intersection of two FSM consists of only those items which - * are present in _BOTH. - * - * Unfortunately fsm_walk2() isn't in the public API, so we don't do that. - */ - q = fsm_intersect(a, b); + q = fsm_intersect_charset(fsm, strlen(charset), charset); if (q == NULL) { - perror("fsm_intersect"); + perror("fsm_intersect_charset"); exit(EXIT_FAILURE); } @@ -468,7 +415,7 @@ build_literals_fsm(const struct fsm_options *opt, bool show_stats, } // TODO: also free a[j].a when we malloc for the 20k-element arrays - fsm = intersect_charset(opt, show_stats, charset, fsm); + fsm = intersect_charset(show_stats, charset, fsm); if (fsm == NULL) { perror("intersect_charset"); exit(EXIT_FAILURE); @@ -552,7 +499,7 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } - fsm = intersect_charset(opt, show_stats, charset, fsm); + fsm = intersect_charset(show_stats, charset, fsm); if (fsm == NULL) { perror("intersect_charset"); exit(EXIT_FAILURE); From ee5773913ad0b4bcf6313e85af607108e2c1450b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 24 May 2024 19:09:13 +0100 Subject: [PATCH 158/373] Rework categorisation for (hopefully) clearer outputs and verbose messages. --- src/rx/main.c | 355 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 233 insertions(+), 122 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 60b73dab0..786db12b7 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,16 @@ #include #include +enum category { + CATEGORY_LITERAL, + CATEGORY_GENERAL, + CATEGORY_ERROR, + CATEGORY_UNSUPPORTED, + CATEGORY_UNSATISFIABLE, + CATEGORY_ZERO, + CATEGORY_PERMITTED +}; + struct literal { const void *p; size_t n; @@ -51,6 +62,20 @@ struct id_set { fsm_end_id_t *a; }; +/* +TODO: flag to have the generated api only ever return a single endid, and resolve ambiguity by one of several ways: +1. earlier line (lower endid) wins +2. longest match/most specific regex wins (this doesn't work for dfa) +3. error about it at compile time (in particular use rx -q as a lint to find conflicts). this is the opposite to -u. give examples, lx style +- unsure if there are any other strategies for allowing ambiguities +4. allow ambiguious patterns and have the generated api return a set +*/ +enum ambig { + AMBIG_ERROR, + AMBIG_EARLIEST, + AMBIG_MULTIPLE +}; + static void * xalloc(size_t size) { @@ -150,6 +175,24 @@ append_literal(struct literal_set *set, const char *p, size_t n, fsm_end_id_t id set->count++; } +static fsm_end_id_t +min_id(const fsm_end_id_t *ids, size_t count) +{ + fsm_end_id_t min; + + assert(ids != NULL); + assert(count > 0); + + min = ids[0]; + for (fsm_end_id_t i = 1; i < count; i++) { + if (ids[i] < min) { + min = ids[i]; + } + } + + return min; +} + static enum re_strings_flags literal_flags(enum re_literal_category category) { @@ -209,22 +252,39 @@ intersect_charset(bool show_stats, const char *charset, struct fsm *fsm) return q; } -static void +const char * +category_reason(enum category r) +{ + switch (r) { + case CATEGORY_LITERAL: return "literal"; + case CATEGORY_GENERAL: return "general"; + case CATEGORY_ERROR: return "parse error"; + case CATEGORY_UNSUPPORTED: return "unsupported"; + case CATEGORY_UNSATISFIABLE: return "unsatisfiable"; + case CATEGORY_ZERO: return "contains \\0, which a charset cannot express"; + case CATEGORY_PERMITTED: return "charset/reject"; + + default: + assert(!"unreached"); + abort(); + } +} + +/* return an error reason if a pattern was not successfully appended to a set */ +static enum category categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags flags, - bool strict, bool verbose, - const char *charset, const char *reject, fsm_end_id_t id, const char *s, - struct id_set *general, struct id_set *declined, - struct literal_set *literals) + const char *charset, const char *reject, const char *s, + struct re_err *err, + char **lit_s_out, size_t *lit_n_out) { enum re_literal_category category; - struct re_err err; char *lit_s; size_t lit_n; int r; - assert(general != NULL); - assert(declined != NULL); - assert(literals != NULL); + assert(err != NULL); + assert(lit_s_out != NULL); + assert(lit_n_out != NULL); /* * -1: Error @@ -232,60 +292,27 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags * and *category is set. */ const char *q = s; - r = re_is_literal(dialect, nlgetc, &q, opt, flags, &err, &category, + r = re_is_literal(dialect, nlgetc, &q, opt, flags, err, &category, &lit_s, &lit_n); /* unsupported */ - if (r == -1 && err.e == RE_EUNSUPPORTED) { - if (verbose) { - fprintf(stderr, "declined (unsupported) #%u: /%.*s/\n", - id, (int) strcspn(s, "\n"), s); - } - - append_id(declined, id); - return; + if (r == -1 && err->e == RE_EUNSUPPORTED) { + return CATEGORY_UNSUPPORTED; } /* parse error */ -// we consider a parse error here a failure, rather than declining the pattern -// TOOD: pass cli flag to make this non-fatal, quietly decline. also for empty strings and unsatisfiable expressions if (r == -1) { - if (!strict) { - if (verbose) { -// XXX: rephrase error. also re_perror() for this too - fprintf(stderr, "XXX: would decline (parse error) #%u: /%.*s/\n", - id, (int) strcspn(s, "\n"), s); - } - - append_id(declined, id); - return; - } - - const char *p; - - /* s is \n-terminated per pattern. - * We allocate here so we can \0-terminate for re_perror() */ - p = xstrndup(s, strcspn(s, "\n")); - re_perror(dialect, &err, NULL, p); - free((void *) p); - - exit(EXIT_FAILURE); + return CATEGORY_ERROR; } /* empty string */ if (r == 1 && lit_n == 0) { - fprintf(stdout, "re_is_literal: /%.*s/ regex matches empty string only\n", - (int) strcspn(s, "\n"), s); -// TODO: cli option to decline instead - exit(EXIT_FAILURE); + /* this is acceptable */ } /* unsatisfiable */ if (r == 1 && category == RE_LITERAL_UNSATISFIABLE) { - fprintf(stdout, "re_is_literal: /%.*s/ regex is unsatisfiable\n", - (int) strcspn(s, "\n"), s); -// TODO: cli option to decline instead - exit(EXIT_FAILURE); + return CATEGORY_UNSATISFIABLE; } /* is a literal */ @@ -293,11 +320,8 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags assert(lit_s != NULL); for (size_t i = 0; i < lit_n; i++) { - if (lit_s[i] == '\0') { -// TODO: cli option to decline instead - fprintf(stderr, "re_is_literal: '%.*s' literal contains \\0\n", - (int) strcspn(s, "\n"), s); - exit(EXIT_FAILURE); + if (charset != NULL && lit_s[i] == '\0') { + return CATEGORY_ZERO; } } @@ -314,16 +338,9 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags } } - if (verbose) { - fprintf(stderr, "literal #%u '%.*s'\n", id, (int) lit_n, lit_s); - } - // TODO: explain reject does not apply if (!permitted_chars(lit_s, lit_n, charset, NULL)) { -// TODO: cli option to decline instead -//fprintf(stderr, "declined literal (charset) #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); - append_id(declined, id); - return; + return CATEGORY_PERMITTED; } //fprintf(stderr, "re_is_literal found: '%.*s'\n", (int) lit_n, lit_s); @@ -331,8 +348,10 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags enum re_strings_flags flags = literal_flags(category); assert(flags >= 0 && flags <= 3); - append_literal(&literals[flags], lit_s, lit_n, id); - return; + *lit_s_out = lit_s; + *lit_n_out = lit_n; + + return CATEGORY_LITERAL; } /* @@ -348,15 +367,15 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags * * So we're relying on that here, and this is just a best-effort help. */ - if (!permitted_chars(s, strcspn(s, "\n"), charset, reject)) { -//fprintf(stderr, "declined general (charset or reject) #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); - append_id(declined, id); - return; +// TODO: would need to pass charset here *plus all the regex syntax operators* +// we shouldn't enforce positive charset pre-lexing here, this would need to be done at AST time + if (!permitted_chars(s, strcspn(s, "\n"), NULL, reject)) { +// XXX: this accidentally includes regex operators, we need those in the charset too + return CATEGORY_PERMITTED; } -//fprintf(stderr, "general #%u: /%.*s/\n", id, (int) strcspn(s, "\n"), s); /* not a literal */ - append_id(general, id); + return CATEGORY_GENERAL; } /* @@ -509,10 +528,13 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, // TODO: explain we minimise again after intersect_charset because this fsm has a single endid, so it's (potentially) a worthwhile difference // TODO: we do minimise now after intersection, explain it's because only one end id +fsm_print_dot(stderr, fsm); +#if 0 if (!fsm_minimise(fsm)) { perror("fsm_minimise"); exit(EXIT_FAILURE); } +#endif return fsm; } @@ -521,11 +543,10 @@ static int endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque) { - assert(endleaf_opaque == NULL); - assert(endleaf_opaque == NULL); + assert(ids != NULL); + assert(endleaf_opaque != NULL); - (void) f; - (void) endleaf_opaque; + enum ambig ambig = * (const enum ambig *) endleaf_opaque; /* morally an assertion, but I feel better leaving this in for various user data */ if (count == 0) { @@ -533,48 +554,67 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, exit(EXIT_FAILURE); } + for (fsm_end_id_t i = 0; i < count; i++) { + assert(ids[i] <= INT_MAX); + } + /* exactly one end id means no ambiguious patterns */ -// TODO: want to ensure at compile time that the endids set only has one element, reject ambigious patterns -// XXX: i'm not convinced we do. we could just as well emit code that returns a list of IDs -// and have a cli option to error about ambiguities -// TODO: this isn't strict checking. this is the opposite to -u -// TODO: give examples, lx style -#if 0 - if (count > 1) { + + switch (ambig) { + case AMBIG_ERROR: + if (count > 1) { // TODO: explain this more clearly - fprintf(stderr, "ambigious patterns:"); + fprintf(stderr, "ambigious patterns:"); - for (fsm_end_id_t i = 0; i < ids->count; i++) { - fprintf(stderr, " %u", ids->ids[i]); + for (fsm_end_id_t i = 0; i < count; i++) { + fprintf(stderr, " %u", ids[i]); + } + +// TODO: give example, lx-style + + fprintf(stderr, "\n"); + + exit(EXIT_FAILURE); } - fprintf(stderr, "\n"); + fprintf(f, "return %u;", ids[0]); + break; -// we consider this a compilation error for the purposes of this program - exit(EXIT_FAILURE); - } -#endif + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, but are not sorted. + */ + fprintf(f, "return %u;", min_id(ids, count)); + break; - /* - * Here I would like to emit (static unsigned []) { 1, 2, 3 } - * but specifying a storage duration for compound literals - * is a compiler extension. - * So I'm emitting a static const variable declaration instead. - */ + case AMBIG_MULTIPLE: + /* + * Here I would like to emit (static unsigned []) { 1, 2, 3 } + * but specifying a storage duration for compound literals + * is a compiler extension. + * So I'm emitting a static const variable declaration instead. + */ - fprintf(f, "{\n"); - fprintf(f, "\t\tstatic const unsigned a[] = { "); - for (fsm_end_id_t i = 0; i < count; i++) { - fprintf(f, "%u", ids[i]); - if (i + 1 < count) { - fprintf(f, ", "); + fprintf(f, "{\n"); + fprintf(f, "\t\tstatic const unsigned a[] = { "); + for (fsm_end_id_t i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } } + fprintf(f, " };\n"); + fprintf(f, "\t\t*ids = a;\n"); + fprintf(f, "\t\t*count = %zu;\n", count); + fprintf(f, "\t\treturn 0;\n"); + fprintf(f, "\t}"); + break; + + default: + assert(!"unreached"); + abort(); } - fprintf(f, " };\n"); - fprintf(f, "\t\t*ids = a;\n"); - fprintf(f, "\t\t*count = %zu;\n", count); - fprintf(f, "\t\treturn 0;\n"); - fprintf(f, "\t}"); // TODO: override return -1 @@ -582,7 +622,7 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, } void -print_fsm_c(struct fsm *fsm, const char *prefix) +print_fsm_c(struct fsm *fsm, const char *prefix, enum ambig ambig) { const struct fsm_options *old; struct fsm_options tmp; @@ -598,9 +638,16 @@ print_fsm_c(struct fsm *fsm, const char *prefix) fprintf(stdout, "#include \n"); fprintf(stdout, "\n"); - fprintf(stdout, "bool\n"); - fprintf(stdout, "%s_main(const char *s, size_t n,\n", prefix); - fprintf(stdout, "\tconst unsigned **ids, size_t *count)\n"); +// TODO: not bool? +// TODO: return type depends on enum ambig +// TODO: but also override return -1 + fprintf(stdout, "int\n"); + fprintf(stdout, "%s_main(const char *s, size_t n", prefix); + if (ambig == AMBIG_MULTIPLE) { + fprintf(stdout, ",\n"); + fprintf(stdout, "\tconst unsigned **ids, size_t *count"); + } + fprintf(stdout, ")\n"); fprintf(stdout, "{\n"); fprintf(stdout, "\tconst char *b = s, *e = s + n;\n"); @@ -609,7 +656,7 @@ print_fsm_c(struct fsm *fsm, const char *prefix) tmp = *old; tmp.fragment = true, - tmp.endleaf_opaque = NULL, + tmp.endleaf_opaque = &ambig, tmp.endleaf = endleaf_c, fsm_setoptions(fsm, &tmp); @@ -666,7 +713,7 @@ void usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQqv] [-C charset] [-r dialect] [-R reject] input-file [declined-file]\n", name); + printf("usage: %s: [-ciQquv] [-C charset] [-r dialect] [-R reject] input-file [declined-file]\n", name); printf(" %s -h\n", name); } @@ -677,6 +724,7 @@ main(int argc, char *argv[]) bool strict = false; bool verbose = false; bool show_stats = false; + enum ambig ambig = AMBIG_ERROR; size_t general_limit = 0; @@ -718,9 +766,8 @@ charset = // TODO: optional extra argv[1] to output declined patterns (we could iteratively munch down on the list) // TODO: cli option for numeric limit for general.count to decline patterns // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) -// TODO: cli option for worker pool threads -// TODO: make this a whole tool, alongside re(1) and fsm(1) // TODO: manpage +// TODO: cli option for worker pool threads // TODO: probably handle each of fsms[] in a separate thread, use worker pool // TODO: cli flag to set prefix @@ -728,7 +775,7 @@ charset = const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cin:r:R:Qqv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cin:r:R:Qquv"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -788,6 +835,10 @@ charset = quiet = true; break; + case 'u': + ambig = AMBIG_MULTIPLE; + break; + case 'v': /* "verbose" means showing information about every pattern and FSM, * this is O(n) output to the length of the input file */ @@ -875,6 +926,11 @@ charset = } } + if (patterns_count > INT_MAX) { +// TODO: error about it +// TODO: explain this is so we can fit the endid value in an int for the return value of the generated code + } + if (show_stats) { fprintf(stderr, "patterns: %zu\n", patterns_count); } @@ -929,15 +985,20 @@ charset = const char *q = addr; for (fsm_end_id_t id = 0; id < patterns_count; id++) { + struct re_err err; + char *lit_s; + size_t lit_n; + enum category r; + patterns[id] = q; if (verbose) { - fprintf(stderr, "pattern[%u]: /%.*s/\n", + fprintf(stderr, "pattern #%u: /%.*s/\n", id, (int) strcspn(q, "\n"), patterns[id]); } - categorize(&opt, dialect, flags, strict, verbose, - charset, reject, id, patterns[id], - &general, &declined, literals); + r = categorize(&opt, dialect, flags, + charset, reject, patterns[id], + &err, &lit_s, &lit_n); /* * On error the parser may not have consumed all the input. @@ -945,6 +1006,50 @@ charset = * points somewhere mid-way along the regex. */ q += strcspn(patterns[id], "\n") + 1; + + if (verbose) { + switch (r) { + case CATEGORY_LITERAL: + fprintf(stderr, "literal #%u '%.*s'\n", + id, (int) lit_n, lit_s); + break; + + case CATEGORY_GENERAL: + fprintf(stderr, "general #%u: /%.*s/\n", + id, (int) strcspn(patterns[id], "\n"), patterns[id]); + break; + + case CATEGORY_ERROR: + fprintf(stderr, "declined (%s) #%u: /%.*s/", + category_reason(r), id, + (int) strcspn(patterns[id], "\n"), patterns[id]); + re_perror(dialect, &err, NULL, NULL); + break; + + default: + fprintf(stderr, "declined (%s) #%u: /%.*s/\n", + category_reason(r), id, + (int) strcspn(patterns[id], "\n"), patterns[id]); + break; + } + } + + switch (r) { + case CATEGORY_LITERAL: + append_literal(&literals[flags], lit_s, lit_n, id); + break; + + case CATEGORY_GENERAL: + append_id(&general, id); + break; + + default: + if (strict) { + exit(EXIT_FAILURE); + } + + append_id(&declined, id); + } } } @@ -952,7 +1057,8 @@ charset = * This is conservative allocation, the declined list may still grow yet. * We track actual usage in fsm_count. */ - struct fsm **fsms = xalloc((patterns_count - declined.count) * sizeof *fsms); +#define LITERALS_COUNT 4 + struct fsm **fsms = xalloc((LITERALS_COUNT + patterns_count - declined.count) * sizeof *fsms); size_t fsm_count = 0; /* @@ -965,9 +1071,12 @@ charset = * unioned anyway. */ +// TODO: also have strict mode error for empty strings and unsatisfiable expressions +// TODO: better to do this when looking at the DFA for both literals and for regexps + /* sets of literals */ { - enum re_strings_flags flags[] = { + enum re_strings_flags flags[LITERALS_COUNT] = { 0, RE_STRINGS_ANCHOR_LEFT, RE_STRINGS_ANCHOR_RIGHT, @@ -983,6 +1092,7 @@ charset = &literals[i], flags[i]); assert(fsm != NULL); +fprintf(stderr, "fsms[%zu] <- literal[%zu]\n", fsm_count, i); fsms[fsm_count] = fsm; fsm_count++; } @@ -1013,6 +1123,7 @@ charset = continue; } +fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); fsms[fsm_count] = fsm; fsm_count++; } @@ -1041,7 +1152,7 @@ charset = if (verbose) { for (size_t i = 0; i < declined.count; i++) { - fprintf(stderr, "declined.a[%zu]: #%u /%.*s/\n", + fprintf(stderr, "declined[%zu]: #%u /%.*s/\n", i, declined.a[i], (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); } @@ -1129,7 +1240,7 @@ charset = } if (!quiet) { - print_fsm_c(fsm, opt.prefix); + print_fsm_c(fsm, opt.prefix, ambig); } fsm_free(fsm); From 237544a41151c5d8db226d04f41e52b669f3e2a7 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 31 May 2024 09:12:40 -0700 Subject: [PATCH 159/373] Back to POSIX for this, I messed up strndup. --- src/adt/xalloc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/adt/xalloc.c b/src/adt/xalloc.c index 91ee82431..3c9f0dd98 100644 --- a/src/adt/xalloc.c +++ b/src/adt/xalloc.c @@ -4,6 +4,8 @@ * See LICENCE for the full copyright terms. */ +#define _POSIX_C_SOURCE 200809L + #include #include #include @@ -16,12 +18,15 @@ xstrdup(const char *s) { char *new; - new = malloc(strlen(s) + 1); + assert(s != NULL); + + new = strdup(s); if (new == NULL) { + perror("strdup"); return NULL; } - return strcpy(new, s); + return new; } char * @@ -29,12 +34,15 @@ xstrndup(const char *s, size_t n) { char *new; - new = malloc(n); + assert(s != NULL); + + new = strndup(s, n); if (new == NULL) { + perror("strndup"); return NULL; } - return memcpy(new, s, n); + return new; } void * From 708f7f8b90384e9b6242fa6d053cbda3f762adb4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 31 May 2024 09:29:57 -0700 Subject: [PATCH 160/373] Tidying up. --- src/rx/main.c | 134 +++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 786db12b7..1af2d338d 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -25,6 +25,8 @@ #include #include +#include + #include #include #include @@ -37,6 +39,7 @@ #include enum category { + CATEGORY_EMPTY, CATEGORY_LITERAL, CATEGORY_GENERAL, CATEGORY_ERROR, @@ -76,31 +79,6 @@ enum ambig { AMBIG_MULTIPLE }; -static void * -xalloc(size_t size) -{ - void *p; - - p = malloc(size); - if (p == NULL) { - perror("malloc"); - exit(EXIT_FAILURE); - } - - return p; -} -static char * -xstrndup(const char *s, size_t n) -{ - char *new = strndup(s, n); - if (new == NULL) { - perror("strndup"); - exit(EXIT_FAILURE); - } - - return new; -} - static bool permitted_chars(const char *p, size_t n, const char *accept, const char *reject) { @@ -256,6 +234,7 @@ const char * category_reason(enum category r) { switch (r) { + case CATEGORY_EMPTY: return "empty string"; case CATEGORY_LITERAL: return "literal"; case CATEGORY_GENERAL: return "general"; case CATEGORY_ERROR: return "parse error"; @@ -307,7 +286,8 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags /* empty string */ if (r == 1 && lit_n == 0) { - /* this is acceptable */ + assert(lit_s == NULL); + return CATEGORY_EMPTY; } /* unsatisfiable */ @@ -343,8 +323,6 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags return CATEGORY_PERMITTED; } -//fprintf(stderr, "re_is_literal found: '%.*s'\n", (int) lit_n, lit_s); - enum re_strings_flags flags = literal_flags(category); assert(flags >= 0 && flags <= 3); @@ -388,7 +366,7 @@ literal_strings(const struct fsm_options *opt, struct re_strings *g; struct fsm *fsm; - assert(a != NULL); + assert(a != NULL || n == 0); g = re_strings_new(); if (g == NULL) { @@ -407,9 +385,6 @@ literal_strings(const struct fsm_options *opt, fsm = re_strings_build(g, opt, flags); re_strings_free(g); -// XXX: this fsm has unreachable states boo -//fsm_print_dot(stderr, fsm); - /* this is already a DFA, courtesy of re_strings() */ assert(fsm_all(fsm, fsm_isdfa)); @@ -428,12 +403,6 @@ build_literals_fsm(const struct fsm_options *opt, bool show_stats, fsm = literal_strings(opt, set->a, set->count, flags); // XXX: we do produce an empty fsm for 0 literals assert(!fsm_empty(fsm)); - for (size_t i = 0; i < set->count; i++) { -// TODO: when we switch to libfsm alloc hooks, check if re_is_literal() calls malloc proper or not for *lit_s - free((void *) set->a[i].p); - } -// TODO: also free a[j].a when we malloc for the 20k-element arrays - fsm = intersect_charset(show_stats, charset, fsm); if (fsm == NULL) { perror("intersect_charset"); @@ -507,7 +476,10 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } -// TODO: note we don't minimise rather than determinise, explain why + /* + * There's no need to minimise here, we only need a DFA in order + * to fsm_intersect_charset(). We'd want to minimise again anyway. + */ if (!fsm_determinise(fsm)) { perror("fsm_determinise"); exit(EXIT_FAILURE); @@ -524,17 +496,15 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } -// XXX: unsure why this gives 0 states, even with endids -// TODO: explain we minimise again after intersect_charset because this fsm has a single endid, so it's (potentially) a worthwhile difference - -// TODO: we do minimise now after intersection, explain it's because only one end id -fsm_print_dot(stderr, fsm); -#if 0 + /* + * This FSM has only one endid. fsm_minimise() does now maintain + * uniqueness for endids, but here we get the same FSM as if they + * didn't exist at all. + */ if (!fsm_minimise(fsm)) { perror("fsm_minimise"); exit(EXIT_FAILURE); } -#endif return fsm; } @@ -926,9 +896,13 @@ charset = } } + /* + * This is so we can fit the endid value in an int for the + * return value of the generated code. + */ if (patterns_count > INT_MAX) { -// TODO: error about it -// TODO: explain this is so we can fit the endid value in an int for the return value of the generated code + fprintf(stderr, "pattern count overflow\n"); + exit(EXIT_FAILURE); } if (show_stats) { @@ -947,7 +921,7 @@ charset = const char **patterns; - patterns = xalloc(patterns_count * sizeof *patterns); + patterns = xmalloc(patterns_count * sizeof *patterns); // TODO: patterns_set, fsm_set struct id_set declined; @@ -955,14 +929,14 @@ charset = struct literal_set literals[4]; /* unanchored, left, right, both */ // TODO: explain we use the worst case of patterns_count for all these, prior to categorization - declined.a = xalloc(patterns_count * sizeof *declined.a); + declined.a = xmalloc(patterns_count * sizeof *declined.a); declined.count = 0; - general.a = xalloc(patterns_count * sizeof *general.a); + general.a = xmalloc(patterns_count * sizeof *general.a); general.count = 0; -// XXX: xalloc()ing these conservatively is large, i don't like it +// XXX: malloc()ing these conservatively is large, i don't like it for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { - literals[i].a = xalloc(patterns_count * sizeof *literals[i].a); + literals[i].a = xmalloc(patterns_count * sizeof *literals[i].a); literals[i].count = 0; } @@ -1014,6 +988,8 @@ charset = id, (int) lit_n, lit_s); break; + case CATEGORY_EMPTY: + /* fallthrough */ case CATEGORY_GENERAL: fprintf(stderr, "general #%u: /%.*s/\n", id, (int) strcspn(patterns[id], "\n"), patterns[id]); @@ -1039,6 +1015,8 @@ charset = append_literal(&literals[flags], lit_s, lit_n, id); break; + case CATEGORY_EMPTY: + /* fallthrough */ case CATEGORY_GENERAL: append_id(&general, id); break; @@ -1051,14 +1029,24 @@ charset = append_id(&declined, id); } } + + /* + * realloc down to size, note this leaves some .a arrays NULL + * when its count is 0. + */ + declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); + general.a = xrealloc(general.a, general.count * sizeof *general.a); + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + literals[i].a = xrealloc(literals[i].a, literals[i].count * sizeof *literals[i].a); + } } /* * This is conservative allocation, the declined list may still grow yet. * We track actual usage in fsm_count. */ -#define LITERALS_COUNT 4 - struct fsm **fsms = xalloc((LITERALS_COUNT + patterns_count - declined.count) * sizeof *fsms); + struct fsm **fsms = xmalloc(((sizeof literals / sizeof *literals) + + patterns_count - declined.count) * sizeof *fsms); size_t fsm_count = 0; /* @@ -1076,7 +1064,7 @@ charset = /* sets of literals */ { - enum re_strings_flags flags[LITERALS_COUNT] = { + enum re_strings_flags flags[sizeof literals / sizeof *literals] = { 0, RE_STRINGS_ANCHOR_LEFT, RE_STRINGS_ANCHOR_RIGHT, @@ -1092,13 +1080,19 @@ charset = &literals[i], flags[i]); assert(fsm != NULL); -fprintf(stderr, "fsms[%zu] <- literal[%zu]\n", fsm_count, i); + if (verbose) { + fprintf(stderr, "fsms[%zu] <- literal[%zu]\n", fsm_count, i); + } + fsms[fsm_count] = fsm; fsm_count++; } } for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + for (size_t j = 0; j < literals[i].count; j++) { + free((void *) literals[i].a[j].p); /* allocated by re_is_literal() */ + } free(literals[i].a); } @@ -1112,7 +1106,8 @@ fprintf(stderr, "fsms[%zu] <- literal[%zu]\n", fsm_count, i); if (verbose) { fprintf(stderr, "general[%zu]: #%u /%.*s/\n", - i, general.a[i], (int) strcspn(patterns[general.a[i]], "\n"), patterns[general.a[i]]); + i, general.a[i], + (int) strcspn(patterns[general.a[i]], "\n"), patterns[general.a[i]]); } struct fsm *fsm = build_regex_fsm(&opt, show_stats, charset, @@ -1123,7 +1118,10 @@ fprintf(stderr, "fsms[%zu] <- literal[%zu]\n", fsm_count, i); continue; } -fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); + if (verbose) { + fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); + } + fsms[fsm_count] = fsm; fsm_count++; } @@ -1140,11 +1138,11 @@ fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); declined.count); fprintf(stderr, "literals (unanchored): %zu patterns, %u states\n", literals[0].count, fsm_countstates(fsms[0])); - fprintf(stderr, "literals (left): %zu patterns, %u states\n", + fprintf(stderr, "literals (^left): %zu patterns, %u states\n", literals[1].count, fsm_countstates(fsms[1])); - fprintf(stderr, "literals (right): %zu patterns, %u states\n", + fprintf(stderr, "literals (right$): %zu patterns, %u states\n", literals[2].count, fsm_countstates(fsms[2])); - fprintf(stderr, "literals (both): %zu patterns, %u states\n", + fprintf(stderr, "literals (^both$): %zu patterns, %u states\n", literals[3].count, fsm_countstates(fsms[3])); fprintf(stderr, "general: %zu patterns (limit %zu)\n", general.count, general_limit); @@ -1158,8 +1156,11 @@ fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); } } -// TODO: dump to file, handle these with pcre or such -// TODO: explain you're expected to disambiguate by pattern spelling, the endids are not relevant here + /* + * The declined patterns are intended to be handled by some other regex + * engine. The user is expected to disambiguate these by pattern spelling. + * This gives a way to map back to the per-line ID from the original list. + */ if (declined_file != NULL) { FILE *f; @@ -1170,7 +1171,8 @@ fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); } for (size_t i = 0; i < declined.count; i++) { - fprintf(f, "%.*s\n", (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); + fprintf(f, "%.*s\n", + (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); } fclose(f); @@ -1260,7 +1262,5 @@ fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); fprintf(stderr, "rusage.maxrss: %zu MiB\n", (size_t) ru.ru_maxrss / 1024); } #endif - -// TODO: also free lit_s } From 3ab20294b6a7c601ea4b109cce846300f6d68669 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 7 Jun 2024 00:57:04 +0100 Subject: [PATCH 161/373] Clarification, resolving a few TODOs, smaller CLI options etc. --- src/rx/main.c | 222 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 166 insertions(+), 56 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 1af2d338d..9e333730c 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -46,7 +46,7 @@ enum category { CATEGORY_UNSUPPORTED, CATEGORY_UNSATISFIABLE, CATEGORY_ZERO, - CATEGORY_PERMITTED + CATEGORY_NOT_PERMITTED }; struct literal { @@ -65,20 +65,14 @@ struct id_set { fsm_end_id_t *a; }; -/* -TODO: flag to have the generated api only ever return a single endid, and resolve ambiguity by one of several ways: -1. earlier line (lower endid) wins -2. longest match/most specific regex wins (this doesn't work for dfa) -3. error about it at compile time (in particular use rx -q as a lint to find conflicts). this is the opposite to -u. give examples, lx style -- unsure if there are any other strategies for allowing ambiguities -4. allow ambiguious patterns and have the generated api return a set -*/ enum ambig { AMBIG_ERROR, AMBIG_EARLIEST, AMBIG_MULTIPLE }; +typedef void (print_fsm)(struct fsm *, const char *, enum ambig); + static bool permitted_chars(const char *p, size_t n, const char *accept, const char *reject) { @@ -86,7 +80,6 @@ permitted_chars(const char *p, size_t n, const char *accept, const char *reject) bool r = true; assert(p != NULL); - assert(accept != NULL || reject != NULL); /* * p,n is a slice into a \n-terminated buffer, so we can't strcspn() here. @@ -94,6 +87,7 @@ permitted_chars(const char *p, size_t n, const char *accept, const char *reject) * actually a problem I'd prefer to allocate and \0-terminate here just * for the sake of using strcspn(). */ +// XXX: in this program xstrndup is enough after all. strndup will look past the end for a \0, but n < the position of \n, and all lines end with \n. so it'll just look at the \n s = xstrndup(p, n); if (accept != NULL) { @@ -241,7 +235,7 @@ category_reason(enum category r) case CATEGORY_UNSUPPORTED: return "unsupported"; case CATEGORY_UNSATISFIABLE: return "unsatisfiable"; case CATEGORY_ZERO: return "contains \\0, which a charset cannot express"; - case CATEGORY_PERMITTED: return "charset/reject"; + case CATEGORY_NOT_PERMITTED: return "charset/reject"; default: assert(!"unreached"); @@ -292,6 +286,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags /* unsatisfiable */ if (r == 1 && category == RE_LITERAL_UNSATISFIABLE) { + assert(lit_s == NULL); return CATEGORY_UNSATISFIABLE; } @@ -318,9 +313,11 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags } } -// TODO: explain reject does not apply + /* The reject set does not apply for literals, + * it's intended only for regex operators */ if (!permitted_chars(lit_s, lit_n, charset, NULL)) { - return CATEGORY_PERMITTED; + free(lit_s); + return CATEGORY_NOT_PERMITTED; } enum re_strings_flags flags = literal_flags(category); @@ -332,6 +329,18 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags return CATEGORY_LITERAL; } + /* + * XXX: The charset accidentally includes regex operators, + * because we're cheesily looking at the string pre-parse, + * we need those in the charset too, in addition to the + * charset matching literals within the regex. + * + * So we shouldn't enforce positive charset pre-lexing here, + * this would need to be done at AST time. Until this is done, + * I'm just skipping the reject set. + */ + (void) reject; +#if 0 /* * Reject regexps that contain characters that may be expensive. * @@ -345,12 +354,10 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags * * So we're relying on that here, and this is just a best-effort help. */ -// TODO: would need to pass charset here *plus all the regex syntax operators* -// we shouldn't enforce positive charset pre-lexing here, this would need to be done at AST time if (!permitted_chars(s, strcspn(s, "\n"), NULL, reject)) { -// XXX: this accidentally includes regex operators, we need those in the charset too return CATEGORY_PERMITTED; } +#endif /* not a literal */ return CATEGORY_GENERAL; @@ -401,7 +408,13 @@ build_literals_fsm(const struct fsm_options *opt, bool show_stats, assert(set != NULL); fsm = literal_strings(opt, set->a, set->count, flags); -// XXX: we do produce an empty fsm for 0 literals assert(!fsm_empty(fsm)); + + /* we do produce an empty fsm for 0 literals */ + if (set->count == 0) { + assert(fsm_empty(fsm)); + } else { + assert(!fsm_empty(fsm)); + } fsm = intersect_charset(show_stats, charset, fsm); if (fsm == NULL) { @@ -485,16 +498,29 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } + assert(!fsm_empty(fsm)); + if (!fsm_setendid(fsm, id)) { perror("fsm_setendid"); exit(EXIT_FAILURE); } +// XXX: collapses down to nothing because for ^.*abc.*$ the middle is a subset of the .* +// XXX: not sure at all +// XXX: this is the wrong operation! +// instead of .* we should go edge-by-edge and remove anything not in the charset +// but that would give the same result +// i want to intersect_charset but without folding the fsm down to nothing +// XXX: i *think* intersect_charset must be doing the wrong thing. somehow + (void) charset; + (void) show_stats; +/* fsm = intersect_charset(show_stats, charset, fsm); if (fsm == NULL) { perror("intersect_charset"); exit(EXIT_FAILURE); } +*/ /* * This FSM has only one endid. fsm_minimise() does now maintain @@ -506,6 +532,8 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } + assert(!fsm_empty(fsm)); + return fsm; } @@ -591,8 +619,8 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, return 0; } -void -print_fsm_c(struct fsm *fsm, const char *prefix, enum ambig ambig) +static void +print_fsm_vmc(struct fsm *fsm, const char *prefix, enum ambig ambig) { const struct fsm_options *old; struct fsm_options tmp; @@ -672,6 +700,70 @@ dialect_name(const char *name) exit(EXIT_FAILURE); } +static enum fsm_io +io(const char *name) +{ + size_t i; + + struct { + const char *name; + enum fsm_io io; + } a[] = { + { "getc", FSM_IO_GETC }, + { "str", FSM_IO_STR }, + { "pair", FSM_IO_PAIR } + }; + + assert(name != NULL); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + if (0 == strcmp(a[i].name, name)) { + return a[i].io; + } + } + + fprintf(stderr, "unrecognised IO API; valid IO APIs are: "); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + i + 1 < sizeof a / sizeof *a ? ", " : "\n"); + } + + exit(EXIT_FAILURE); +} + +static print_fsm * +print_name(const char *name) +{ + size_t i; + + struct { + const char *name; + print_fsm *print; + } a[] = { + { "vmc", print_fsm_vmc } + }; + + assert(name != NULL); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + if (0 == strcmp(a[i].name, name)) { + return a[i].print; + } + } + + fprintf(stderr, "unrecognised output language; valid languages are: "); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + i + 1 < sizeof a / sizeof *a ? ", " : "\n"); + } + + exit(EXIT_FAILURE); +} + static void usage(const char *name) { @@ -683,7 +775,7 @@ void usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQquv] [-C charset] [-r dialect] [-R reject] input-file [declined-file]\n", name); + printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] input-file [declined-file]\n", name); printf(" %s -h\n", name); } @@ -695,6 +787,7 @@ main(int argc, char *argv[]) bool verbose = false; bool show_stats = false; enum ambig ambig = AMBIG_ERROR; + print_fsm *print = print_fsm_vmc; size_t general_limit = 0; @@ -703,49 +796,37 @@ main(int argc, char *argv[]) const char *declined_file = NULL; enum re_dialect dialect = RE_PCRE; -// TODO: we need to provide a way to disable RE_END_NL -// TODO: RE_SINGLE. also be careful, flags disrupt re_is_literal() + + /* + * TODO: we need to provide a way to disable RE_END_NL + * TODO: RE_SINGLE. also be careful, flags disrupt re_is_literal() + */ const enum re_flags flags = 0; // TODO: notably not RE_END_NL - // TODO: explain we decline these at compile time because NFA->DFA is too costly - const char *reject = "*+{"; + /* regex syntax characters we decline at compile time + * because NFA->DFA is too costly */ + const char *reject = ""; - const struct fsm_options opt = { + struct fsm_options opt = { .anonymous_states = true, .consolidate_edges = true, .comments = false, .case_ranges = true, .always_hex = false, .group_edges = true, - .io = FSM_IO_PAIR, -// TODO: cli option to set io api + .io = FSM_IO_PAIR }; -// XXX: actual character set in practice -charset = - /* RFC9110 field-value stuff... */ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwzyx" - "0123456789" - "!#$%&'*+-.^_`|~" - - /* ...plus some extras found in practice */ - "'\"/()@,:; "; - -// TODO: -l cli option for codegen lang -// TODO: optional extra argv[1] to output declined patterns (we could iteratively munch down on the list) // TODO: cli option for numeric limit for general.count to decline patterns // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: manpage -// TODO: cli option for worker pool threads -// TODO: probably handle each of fsms[] in a separate thread, use worker pool // TODO: cli flag to set prefix { const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cin:r:R:Qquv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cikl::n:r:R:Qquv"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -783,12 +864,24 @@ charset = strict = true; break; + case 'k': + opt.io = io(optarg); + break; + + case 'l': + print = print_name(optarg); + break; + case 'n': general_limit = atoi(optarg); break; case 'R': reject = optarg; + + /* XXX: see categorize() */ + fprintf(stderr, "reject charset not implemented\n"); + exit(EXIT_FAILURE); break; case 'r': @@ -839,10 +932,15 @@ charset = } } -// TODO: error if charset contains \n (it also can't contain \0) + if (charset != NULL && strchr(charset, '\n')) { + fprintf(stderr, "charset cannot contain \\n " + "(because regexes are delimited by newlines)\n"); + exit(EXIT_FAILURE); + } if (show_stats) { - fprintf(stderr, "charset: [%s]\n", charset); + fprintf(stderr, "charset: [%s]\n", + charset == NULL ? "(none)" : charset); fprintf(stderr, "reject: [%s]\n", reject); } @@ -914,10 +1012,17 @@ charset = general_limit = patterns_count; } -// we're keying id -> spelling -// we don't use patterns[] just for debugging, we use it for passing to re_comp(). but not to re_strings(), we pass lit_s there -// note this is \n-terminated, not \0-terminated. need %.*s for printing -// TODO: explain the index here is an endid + /* + * We're keying unique id to regex pattern spellings. The index + * here is used as an fsm_end_id_t. + * + * We don't use patterns[] just for debugging, we use it for passing + * to re_comp(). But not to re_strings(), we pass lit_s there. + * + * Note patterns[] points into the mmaped file, so each pattern is + * \n-terminated, not \0-terminated. So we use %.*s for printing. + * XXX: this is ridiculous, we should just malloc() and \0-terminate. + */ const char **patterns; @@ -928,13 +1033,15 @@ charset = struct id_set general; struct literal_set literals[4]; /* unanchored, left, right, both */ - // TODO: explain we use the worst case of patterns_count for all these, prior to categorization + /* + * We allocate the worse case of patterns_count for these, + * prior to categorization. And then realloc down later. + */ declined.a = xmalloc(patterns_count * sizeof *declined.a); declined.count = 0; general.a = xmalloc(patterns_count * sizeof *general.a); general.count = 0; -// XXX: malloc()ing these conservatively is large, i don't like it for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { literals[i].a = xmalloc(patterns_count * sizeof *literals[i].a); literals[i].count = 0; @@ -965,10 +1072,6 @@ charset = enum category r; patterns[id] = q; - if (verbose) { - fprintf(stderr, "pattern #%u: /%.*s/\n", - id, (int) strcspn(q, "\n"), patterns[id]); - } r = categorize(&opt, dialect, flags, charset, reject, patterns[id], @@ -1034,6 +1137,7 @@ charset = * realloc down to size, note this leaves some .a arrays NULL * when its count is 0. */ + patterns = xrealloc(patterns, patterns_count * sizeof *patterns); declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); general.a = xrealloc(general.a, general.count * sizeof *general.a); for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { @@ -1057,6 +1161,10 @@ charset = * Also, the order of FSMs in the fsms[] array is not significant, * they could be populated in any order. They're just going to get * unioned anyway. + * + * We'd probably have a CLI option for the size of the worker pool. + * However the bottleneck is fsm_determinise() after union, not here. + * So this doesn't help until fsm_determinise() also runs in parallel. */ // TODO: also have strict mode error for empty strings and unsatisfiable expressions @@ -1133,6 +1241,8 @@ charset = free(general.a); + fsms = xrealloc(fsms, fsm_count * sizeof *fsms); + if (show_stats) { fprintf(stderr, "declined: %zu patterns\n", declined.count); @@ -1242,7 +1352,7 @@ charset = } if (!quiet) { - print_fsm_c(fsm, opt.prefix, ambig); + print(fsm, opt.prefix, ambig); } fsm_free(fsm); From 37d34c858705a8d521b12f9acee95c61752e794c Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 7 Jun 2024 01:48:56 +0100 Subject: [PATCH 162/373] Allocate and duplicate for \0-terminated patterns, rather than pointing at \n-terminated patterns in-situ. This is just so much less complicated. --- src/rx/main.c | 173 ++++++++++++++++---------------------------------- 1 file changed, 55 insertions(+), 118 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 9e333730c..477925a4c 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -45,7 +45,6 @@ enum category { CATEGORY_ERROR, CATEGORY_UNSUPPORTED, CATEGORY_UNSATISFIABLE, - CATEGORY_ZERO, CATEGORY_NOT_PERMITTED }; @@ -73,57 +72,6 @@ enum ambig { typedef void (print_fsm)(struct fsm *, const char *, enum ambig); -static bool -permitted_chars(const char *p, size_t n, const char *accept, const char *reject) -{ - char *s; - bool r = true; - - assert(p != NULL); - - /* - * p,n is a slice into a \n-terminated buffer, so we can't strcspn() here. - * We could provide our own implementation (i.e. memcspn), but until it's - * actually a problem I'd prefer to allocate and \0-terminate here just - * for the sake of using strcspn(). - */ -// XXX: in this program xstrndup is enough after all. strndup will look past the end for a \0, but n < the position of \n, and all lines end with \n. so it'll just look at the \n - s = xstrndup(p, n); - - if (accept != NULL) { - r &= s[strspn(s, accept)] == '\0'; - } - - if (reject != NULL) { - r &= s[strcspn(s, reject)] == '\0'; - } - - free(s); - - return r; -} - -int -nlgetc(void *opaque) -{ - const char **s = opaque; - char c; - - assert(s != NULL); - assert(*s != NULL); - - c = **s; - - /* skip the newline too */ - (*s)++; - - if (c == '\n') { - return EOF; - } - - return (unsigned char) c; -} - static void append_id(struct id_set *set, fsm_end_id_t id) { @@ -234,7 +182,6 @@ category_reason(enum category r) case CATEGORY_ERROR: return "parse error"; case CATEGORY_UNSUPPORTED: return "unsupported"; case CATEGORY_UNSATISFIABLE: return "unsatisfiable"; - case CATEGORY_ZERO: return "contains \\0, which a charset cannot express"; case CATEGORY_NOT_PERMITTED: return "charset/reject"; default: @@ -265,7 +212,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags * and *category is set. */ const char *q = s; - r = re_is_literal(dialect, nlgetc, &q, opt, flags, err, &category, + r = re_is_literal(dialect, fsm_sgetc, &q, opt, flags, err, &category, &lit_s, &lit_n); /* unsupported */ @@ -294,12 +241,6 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags if (r == 1) { assert(lit_s != NULL); - for (size_t i = 0; i < lit_n; i++) { - if (charset != NULL && lit_s[i] == '\0') { - return CATEGORY_ZERO; - } - } - if (category == RE_LITERAL_ANCHOR_END || category == RE_LITERAL_ANCHOR_BOTH) { /* * This newline is appended in ast_is_literal() as an attempt to @@ -313,11 +254,20 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags } } - /* The reject set does not apply for literals, - * it's intended only for regex operators */ - if (!permitted_chars(lit_s, lit_n, charset, NULL)) { - free(lit_s); - return CATEGORY_NOT_PERMITTED; + /* + * The reject set does not apply for literals, + * it's intended only for regex operators. + * + * We're walking lit_s longhand here (rather than strcspn) + * because it isn't a \0-terminated string. + */ + if (charset != NULL) { + for (size_t i = 0; i < lit_n; i++) { + if (!strchr(charset, lit_s[i])) { + free(lit_s); + return CATEGORY_NOT_PERMITTED; + } + } } enum re_strings_flags flags = literal_flags(category); @@ -354,8 +304,8 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags * * So we're relying on that here, and this is just a best-effort help. */ - if (!permitted_chars(s, strcspn(s, "\n"), NULL, reject)) { - return CATEGORY_PERMITTED; + if (reject != NULL && s[strcspn(s, reject)]) { + return CATEGORY_NOT_PERMITTED; } #endif @@ -463,15 +413,10 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, const char *q = s; // TODO: RE_SINGLE here? i think it shouldn't have any effect when \n isn't present in the charset - fsm = re_comp(dialect, nlgetc, &q, opt, flags, &err); + fsm = re_comp(dialect, fsm_sgetc, &q, opt, flags, &err); if (strict && fsm == NULL) { - const char *p; - - /* patterns[] is \n-terminated per pattern. - * We allocate here so we can \0-terminate for re_perror() */ - p = xstrndup(s, strcspn(s, "\n")); - re_perror(dialect, &err, NULL, p); + re_perror(dialect, &err, NULL, s); exit(EXIT_FAILURE); } @@ -484,8 +429,7 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, * so that unioning it with other regexes will still work. */ if (strict && fsm_empty(fsm)) { - fprintf(stdout, "re_comp: /%.*s/ regex is unsatisfiable\n", - (int) strcspn(s, "\n"), s); + fprintf(stdout, "re_comp: /%s/ regex is unsatisfiable\n", s); exit(EXIT_FAILURE); } @@ -764,8 +708,8 @@ print_name(const char *name) exit(EXIT_FAILURE); } -static -void usage(const char *name) +static void +usage(const char *name) { if (name == NULL) { @@ -1018,15 +962,9 @@ main(int argc, char *argv[]) * * We don't use patterns[] just for debugging, we use it for passing * to re_comp(). But not to re_strings(), we pass lit_s there. - * - * Note patterns[] points into the mmaped file, so each pattern is - * \n-terminated, not \0-terminated. So we use %.*s for printing. - * XXX: this is ridiculous, we should just malloc() and \0-terminate. */ - const char **patterns; - - patterns = xmalloc(patterns_count * sizeof *patterns); + char **patterns = xmalloc(patterns_count * sizeof *patterns); // TODO: patterns_set, fsm_set struct id_set declined; @@ -1070,19 +1008,19 @@ main(int argc, char *argv[]) char *lit_s; size_t lit_n; enum category r; + size_t pattern_len; + + pattern_len = strcspn(q, "\n"); - patterns[id] = q; + patterns[id] = xmalloc(pattern_len + 1); + memcpy(patterns[id], q, pattern_len); + patterns[id][pattern_len] = '\0'; r = categorize(&opt, dialect, flags, charset, reject, patterns[id], &err, &lit_s, &lit_n); - /* - * On error the parser may not have consumed all the input. - * So we can't use the current value of q because it potentially - * points somewhere mid-way along the regex. - */ - q += strcspn(patterns[id], "\n") + 1; + q += pattern_len + 1; if (verbose) { switch (r) { @@ -1094,21 +1032,19 @@ main(int argc, char *argv[]) case CATEGORY_EMPTY: /* fallthrough */ case CATEGORY_GENERAL: - fprintf(stderr, "general #%u: /%.*s/\n", - id, (int) strcspn(patterns[id], "\n"), patterns[id]); + fprintf(stderr, "general #%u: /%s/\n", + id, patterns[id]); break; case CATEGORY_ERROR: - fprintf(stderr, "declined (%s) #%u: /%.*s/", - category_reason(r), id, - (int) strcspn(patterns[id], "\n"), patterns[id]); + fprintf(stderr, "declined (%s) #%u: /%s/", + category_reason(r), id, patterns[id]); re_perror(dialect, &err, NULL, NULL); break; default: - fprintf(stderr, "declined (%s) #%u: /%.*s/\n", - category_reason(r), id, - (int) strcspn(patterns[id], "\n"), patterns[id]); + fprintf(stderr, "declined (%s) #%u: /%s/\n", + category_reason(r), id, patterns[id]); break; } } @@ -1145,6 +1081,16 @@ main(int argc, char *argv[]) } } + if (-1 == munmap(addr, sb.st_size)) { + perror("munmap"); + exit(EXIT_FAILURE); + } + + if (-1 == close(fd)) { + perror("close"); + exit(EXIT_FAILURE); + } + /* * This is conservative allocation, the declined list may still grow yet. * We track actual usage in fsm_count. @@ -1213,9 +1159,8 @@ main(int argc, char *argv[]) } if (verbose) { - fprintf(stderr, "general[%zu]: #%u /%.*s/\n", - i, general.a[i], - (int) strcspn(patterns[general.a[i]], "\n"), patterns[general.a[i]]); + fprintf(stderr, "general[%zu]: #%u /%s/\n", + i, general.a[i], patterns[general.a[i]]); } struct fsm *fsm = build_regex_fsm(&opt, show_stats, charset, @@ -1260,9 +1205,8 @@ main(int argc, char *argv[]) if (verbose) { for (size_t i = 0; i < declined.count; i++) { - fprintf(stderr, "declined[%zu]: #%u /%.*s/\n", - i, declined.a[i], - (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); + fprintf(stderr, "declined[%zu]: #%u /%s/\n", + i, declined.a[i], patterns[declined.a[i]]); } } @@ -1281,25 +1225,18 @@ main(int argc, char *argv[]) } for (size_t i = 0; i < declined.count; i++) { - fprintf(f, "%.*s\n", - (int) strcspn(patterns[declined.a[i]], "\n"), patterns[declined.a[i]]); + fprintf(f, "%s\n", patterns[declined.a[i]]); } fclose(f); } - free(declined.a); - free(patterns); - - if (-1 == munmap(addr, sb.st_size)) { - perror("munmap"); - exit(EXIT_FAILURE); + for (size_t i = 0; i < patterns_count; i++) { + free(patterns[i]); } - if (-1 == close(fd)) { - perror("close"); - exit(EXIT_FAILURE); - } + free(declined.a); + free(patterns); if (show_stats) { fprintf(stderr, "fsm_count = %zu FSMs prior to union\n", fsm_count); From aa0eca164bb814631fec4f24d2e126e27e836d7d Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 7 Jun 2024 13:05:56 +0100 Subject: [PATCH 163/373] Simplification, deal with -n earlier. --- src/rx/main.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 477925a4c..eb67e785e 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -761,7 +761,6 @@ main(int argc, char *argv[]) .io = FSM_IO_PAIR }; -// TODO: cli option for numeric limit for general.count to decline patterns // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: manpage // TODO: cli flag to set prefix @@ -1057,7 +1056,7 @@ main(int argc, char *argv[]) case CATEGORY_EMPTY: /* fallthrough */ case CATEGORY_GENERAL: - append_id(&general, id); + append_id(general.count < general_limit ? &general : &declined, id); break; default: @@ -1153,11 +1152,6 @@ main(int argc, char *argv[]) /* individual regexps */ { for (size_t i = 0; i < general.count; i++) { - if (i > general_limit) { - append_id(&declined, i); - continue; - } - if (verbose) { fprintf(stderr, "general[%zu]: #%u /%s/\n", i, general.a[i], patterns[general.a[i]]); From b6fe2849d11ded95af930415fb1b0817e32b1288 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 19 Jun 2024 18:27:55 +0100 Subject: [PATCH 164/373] Give an example for AMBIG_ERROR. I think at the time of writing, fsm_example() is broken, because I get no output here. Possibly related to #438 --- src/rx/main.c | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index eb67e785e..d7df9b28b 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -38,6 +38,12 @@ #include #include +enum ambig { + AMBIG_ERROR, + AMBIG_EARLIEST, + AMBIG_MULTIPLE +}; + enum category { CATEGORY_EMPTY, CATEGORY_LITERAL, @@ -64,10 +70,9 @@ struct id_set { fsm_end_id_t *a; }; -enum ambig { - AMBIG_ERROR, - AMBIG_EARLIEST, - AMBIG_MULTIPLE +struct endleaf_env { + enum ambig ambig; + const struct fsm *fsm; }; typedef void (print_fsm)(struct fsm *, const char *, enum ambig); @@ -485,10 +490,11 @@ static int endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque) { - assert(ids != NULL); - assert(endleaf_opaque != NULL); + const struct endleaf_env *env = endleaf_opaque; - enum ambig ambig = * (const enum ambig *) endleaf_opaque; + assert(ids != NULL); + assert(env != NULL); + assert(env->fsm != NULL); /* morally an assertion, but I feel better leaving this in for various user data */ if (count == 0) { @@ -502,9 +508,18 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, /* exactly one end id means no ambiguious patterns */ - switch (ambig) { + switch (env->ambig) { case AMBIG_ERROR: if (count > 1) { + char buf[50]; /* 50 looks reasonable for an on-screen limit */ + int n; + + n = fsm_example(env->fsm, ids[0], buf, sizeof buf); + if (-1 == n) { + perror("fsm_example"); + exit(EXIT_FAILURE); + } + // TODO: explain this more clearly fprintf(stderr, "ambigious patterns:"); @@ -512,9 +527,8 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, fprintf(stderr, " %u", ids[i]); } -// TODO: give example, lx-style - - fprintf(stderr, "\n"); + fprintf(f, "; for example on input '%s%s'\n", buf, + n >= (int) sizeof buf - 1 ? "..." : ""); exit(EXIT_FAILURE); } @@ -598,7 +612,7 @@ print_fsm_vmc(struct fsm *fsm, const char *prefix, enum ambig ambig) tmp = *old; tmp.fragment = true, - tmp.endleaf_opaque = &ambig, + tmp.endleaf_opaque = & (struct endleaf_env) { ambig, fsm }; tmp.endleaf = endleaf_c, fsm_setoptions(fsm, &tmp); From 7290f010abbf4ecf15cc4166bfa2b393279c8305 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 19 Jun 2024 19:08:04 +0100 Subject: [PATCH 165/373] Move declined-file to an optarg, we need to keep argv[] for multiple input files. --- src/rx/main.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index d7df9b28b..48b6b4318 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -733,7 +733,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] input-file [declined-file]\n", name); + printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] input-file\n", name); printf(" %s -h\n", name); } @@ -778,12 +778,13 @@ main(int argc, char *argv[]) // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: manpage // TODO: cli flag to set prefix +// TODO: argv[] multiple files, one per id { const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cikl::n:r:R:Qquv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:ikl::n:r:R:Qquv"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -813,6 +814,10 @@ main(int argc, char *argv[]) "!#$%&'*+-.^_`|~"; break; + case 'd': + declined_file = optarg; + break; + case 'h': usage(name); exit(EXIT_SUCCESS); @@ -875,18 +880,12 @@ main(int argc, char *argv[]) argc -= optind; argv += optind; - switch (argc) { - case 2: - declined_file = argv[1]; - /* fallthrough */ - case 1: - input_file = argv[0]; - break; - - default: + if (argc != 1) { usage(name); exit(EXIT_FAILURE); } + + input_file = argv[0]; } if (charset != NULL && strchr(charset, '\n')) { From 798f8e74c56a24dafb94f20c37bac7f926d2e670 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 20 Jun 2024 15:12:00 +0100 Subject: [PATCH 166/373] Store patterns per set, rather than indexing into a global pattern array. This sets the scene for multi-file pattern lists, later on. But also seems to keep scopes tighter and make things simpler as a byproduct. --- src/rx/main.c | 97 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 48b6b4318..08db13117 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -54,10 +54,16 @@ enum category { CATEGORY_NOT_PERMITTED }; +struct pattern { + fsm_end_id_t id; + enum re_dialect dialect; + const char *s; +}; + struct literal { + fsm_end_id_t id; const void *p; size_t n; - fsm_end_id_t id; }; struct literal_set { @@ -65,9 +71,9 @@ struct literal_set { struct literal *a; }; -struct id_set { +struct pattern_set { size_t count; - fsm_end_id_t *a; + struct pattern *a; }; struct endleaf_env { @@ -78,12 +84,14 @@ struct endleaf_env { typedef void (print_fsm)(struct fsm *, const char *, enum ambig); static void -append_id(struct id_set *set, fsm_end_id_t id) +append_pattern(struct pattern_set *set, fsm_end_id_t id, enum re_dialect dialect, const char *s) { assert(set != NULL); assert(set->a != NULL); - set->a[set->count] = id; + set->a[set->count].id = id; + set->a[set->count].dialect = dialect; + set->a[set->count].s = s; set->count++; } @@ -403,7 +411,7 @@ build_literals_fsm(const struct fsm_options *opt, bool show_stats, } static struct fsm * -build_regex_fsm(const struct fsm_options *opt, bool show_stats, +build_pattern_fsm(const struct fsm_options *opt, bool show_stats, const char *charset, enum re_dialect dialect, bool strict, const char *s, enum re_flags flags, fsm_end_id_t id) @@ -481,6 +489,8 @@ build_regex_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } +// TODO: therefore set endid after minimisation + assert(!fsm_empty(fsm)); return fsm; @@ -976,11 +986,8 @@ main(int argc, char *argv[]) * to re_comp(). But not to re_strings(), we pass lit_s there. */ - char **patterns = xmalloc(patterns_count * sizeof *patterns); - - // TODO: patterns_set, fsm_set - struct id_set declined; - struct id_set general; + struct pattern_set declined; + struct pattern_set general; struct literal_set literals[4]; /* unanchored, left, right, both */ /* @@ -1002,11 +1009,10 @@ main(int argc, char *argv[]) * it's the only format that doesn't introduce a syntax with escaping. * So that means we need to track the line number associated with each * regex. And that means the line number is neccessarily the end id. - * So patterns[] is indexed by the end id. + * So each pattern carries its associated end id. * * We have four whole FSMs from re_strings(), each potentially contains - * a set of end ids. And each regex in general.a[i] is a single end id. - * So general.a[i] is the index into patterns[]. + * a set of end ids. And each regex in general.a[i] has a single end id. */ /* @@ -1017,22 +1023,32 @@ main(int argc, char *argv[]) for (fsm_end_id_t id = 0; id < patterns_count; id++) { struct re_err err; - char *lit_s; + char *s, *lit_s; size_t lit_n; + size_t n; enum category r; - size_t pattern_len; - pattern_len = strcspn(q, "\n"); + // TODO: would fgets() here. then we can get rid of patterns_count + n = strcspn(q, "\n"); - patterns[id] = xmalloc(pattern_len + 1); - memcpy(patterns[id], q, pattern_len); - patterns[id][pattern_len] = '\0'; + s = xmalloc(n + 1); + memcpy(s, q, n); + s[n] = '\0'; r = categorize(&opt, dialect, flags, - charset, reject, patterns[id], + charset, reject, s, &err, &lit_s, &lit_n); - q += pattern_len + 1; + q += n + 1; + + /* + * For pattern sets, s storage belongs to the entry in the set + * (and is freed when the set is destroyed). For literals, we're + * done with the original syntax, and use lit_s/lit_n instead. + */ + if (r == CATEGORY_LITERAL) { + free(s); + } if (verbose) { switch (r) { @@ -1045,18 +1061,18 @@ main(int argc, char *argv[]) /* fallthrough */ case CATEGORY_GENERAL: fprintf(stderr, "general #%u: /%s/\n", - id, patterns[id]); + id, s); break; case CATEGORY_ERROR: fprintf(stderr, "declined (%s) #%u: /%s/", - category_reason(r), id, patterns[id]); + category_reason(r), id, s); re_perror(dialect, &err, NULL, NULL); break; default: fprintf(stderr, "declined (%s) #%u: /%s/\n", - category_reason(r), id, patterns[id]); + category_reason(r), id, s); break; } } @@ -1069,7 +1085,8 @@ main(int argc, char *argv[]) case CATEGORY_EMPTY: /* fallthrough */ case CATEGORY_GENERAL: - append_id(general.count < general_limit ? &general : &declined, id); + append_pattern(general.count < general_limit ? &general : &declined, + id, dialect, s); break; default: @@ -1077,7 +1094,7 @@ main(int argc, char *argv[]) exit(EXIT_FAILURE); } - append_id(&declined, id); + append_pattern(&declined, id, dialect, s); } } @@ -1085,7 +1102,6 @@ main(int argc, char *argv[]) * realloc down to size, note this leaves some .a arrays NULL * when its count is 0. */ - patterns = xrealloc(patterns, patterns_count * sizeof *patterns); declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); general.a = xrealloc(general.a, general.count * sizeof *general.a); for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { @@ -1167,14 +1183,14 @@ main(int argc, char *argv[]) for (size_t i = 0; i < general.count; i++) { if (verbose) { fprintf(stderr, "general[%zu]: #%u /%s/\n", - i, general.a[i], patterns[general.a[i]]); + i, general.a[i].id, general.a[i].s); } - struct fsm *fsm = build_regex_fsm(&opt, show_stats, charset, + struct fsm *fsm = build_pattern_fsm(&opt, show_stats, charset, dialect, strict, - patterns[general.a[i]], flags, general.a[i]); + general.a[i].s, flags, general.a[i].id); if (fsm == NULL) { - append_id(&declined, i); + append_pattern(&declined, i, general.a[i].dialect, general.a[i].s); continue; } @@ -1191,13 +1207,14 @@ main(int argc, char *argv[]) } } + for (size_t i = 0; i < general.count; i++) { + free((void *) general.a[i].s); + } free(general.a); fsms = xrealloc(fsms, fsm_count * sizeof *fsms); if (show_stats) { - fprintf(stderr, "declined: %zu patterns\n", - declined.count); fprintf(stderr, "literals (unanchored): %zu patterns, %u states\n", literals[0].count, fsm_countstates(fsms[0])); fprintf(stderr, "literals (^left): %zu patterns, %u states\n", @@ -1208,12 +1225,14 @@ main(int argc, char *argv[]) literals[3].count, fsm_countstates(fsms[3])); fprintf(stderr, "general: %zu patterns (limit %zu)\n", general.count, general_limit); + fprintf(stderr, "declined: %zu patterns\n", + declined.count); } if (verbose) { for (size_t i = 0; i < declined.count; i++) { fprintf(stderr, "declined[%zu]: #%u /%s/\n", - i, declined.a[i], patterns[declined.a[i]]); + i, declined.a[i].id, declined.a[i].s); } } @@ -1232,18 +1251,16 @@ main(int argc, char *argv[]) } for (size_t i = 0; i < declined.count; i++) { - fprintf(f, "%s\n", patterns[declined.a[i]]); + fprintf(f, "%s\n", declined.a[i].s); } fclose(f); } - for (size_t i = 0; i < patterns_count; i++) { - free(patterns[i]); + for (size_t i = 0; i < declined.count; i++) { + free((void *) declined.a[i].s); } - free(declined.a); - free(patterns); if (show_stats) { fprintf(stderr, "fsm_count = %zu FSMs prior to union\n", fsm_count); From 41d82251fdb80a5f1199995824346539ddb27b7d Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 20 Jun 2024 16:11:21 +0100 Subject: [PATCH 167/373] Check end id ambiguity before printing, rather than during. --- src/rx/main.c | 83 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 23 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 08db13117..b6fcc3622 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -496,6 +496,57 @@ build_pattern_fsm(const struct fsm_options *opt, bool show_stats, return fsm; } +static int +check_end_ambiguity(const struct fsm *fsm, fsm_state_t s, void *opaque) +{ + fsm_end_id_t *ids; + size_t count; + char buf[50]; /* 50 looks reasonable for an on-screen limit */ + int n, res; + + assert(fsm != NULL); + assert(opaque == NULL); + + if (!fsm_isend(fsm, s)) { + return 1; + } + + count = fsm_endid_count(fsm, s); + + /* in rx all end states have an end id */ + assert(count > 0); + + if (count == 1) { + return 1; + } + +// TODO: explain this more clearly + fprintf(stderr, "ambigious patterns:"); + + ids = xmalloc(count * sizeof *ids); + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); + +// TODO: print patterns rather than IDs, pass &literals, &general via opaque, find by id + for (fsm_end_id_t i = 0; i < count; i++) { + fprintf(stderr, " #%u", ids[i]); + } + + free(ids); + + n = fsm_example(fsm, s, buf, sizeof buf); + if (-1 == n) { + perror("fsm_example"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "; for example on input '%s%s'\n", buf, + n >= (int) sizeof buf - 1 ? "..." : ""); + + return 0; +} + static int endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque) @@ -520,28 +571,8 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, switch (env->ambig) { case AMBIG_ERROR: - if (count > 1) { - char buf[50]; /* 50 looks reasonable for an on-screen limit */ - int n; - - n = fsm_example(env->fsm, ids[0], buf, sizeof buf); - if (-1 == n) { - perror("fsm_example"); - exit(EXIT_FAILURE); - } - -// TODO: explain this more clearly - fprintf(stderr, "ambigious patterns:"); - - for (fsm_end_id_t i = 0; i < count; i++) { - fprintf(stderr, " %u", ids[i]); - } - - fprintf(f, "; for example on input '%s%s'\n", buf, - n >= (int) sizeof buf - 1 ? "..." : ""); - - exit(EXIT_FAILURE); - } + /* dealt with ahead of the call to print */ + assert(count <= 1); fprintf(f, "return %u;", ids[0]); break; @@ -998,7 +1029,7 @@ main(int argc, char *argv[]) declined.count = 0; general.a = xmalloc(patterns_count * sizeof *general.a); general.count = 0; - + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { literals[i].a = xmalloc(patterns_count * sizeof *literals[i].a); literals[i].count = 0; @@ -1312,6 +1343,12 @@ main(int argc, char *argv[]) fprintf(stderr, "dfa: %u states\n", fsm_countstates(fsm)); } + if (ambig == AMBIG_ERROR) { + if (!fsm_walk_states(fsm, NULL, check_end_ambiguity)) { + exit(EXIT_FAILURE); + } + } + if (!quiet) { print(fsm, opt.prefix, ambig); } From a6deb96aa69e8a60ea4ad48afe15fd8995e5f937 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 20 Jun 2024 19:02:45 +0100 Subject: [PATCH 168/373] No need for the *nwritten parameter. Now this is always equivalent to the count passed in, when the return status is 1. And the return status is always 1 when the count is enough. In all situations we know the count is enough. --- src/rx/main.c | 142 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 57 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index b6fcc3622..7e81a78d3 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -81,6 +81,11 @@ struct endleaf_env { const struct fsm *fsm; }; +struct check_env { + const struct literal_set (*literals)[4]; + const struct pattern_set *general; +}; + typedef void (print_fsm)(struct fsm *, const char *, enum ambig); static void @@ -108,8 +113,27 @@ append_literal(struct literal_set *set, const char *p, size_t n, fsm_end_id_t id set->count++; } +/* TODO: centralise */ +static int +cmp_id(const void *pa, const void *pb) +{ + const fsm_end_id_t *a = pa; + const fsm_end_id_t *b = pb; + + if (*a < *b) { + return -1; + } + + if (*a > *b) { + return 1; + } + + return 0; +} + +/* TODO: centralise */ static fsm_end_id_t -min_id(const fsm_end_id_t *ids, size_t count) +min_endid(const fsm_end_id_t *ids, size_t count) { fsm_end_id_t min; @@ -126,28 +150,6 @@ min_id(const fsm_end_id_t *ids, size_t count) return min; } -static enum re_strings_flags -literal_flags(enum re_literal_category category) -{ - switch (category) { - case RE_LITERAL_UNANCHORED: - return 0; - - case RE_LITERAL_ANCHOR_START: - return RE_STRINGS_ANCHOR_LEFT; - - case RE_LITERAL_ANCHOR_END: - return RE_STRINGS_ANCHOR_RIGHT; - - case RE_LITERAL_ANCHOR_BOTH: - return RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT; - - default: - assert(!"unreached"); - abort(); - } -} - /* this interface doesn't allow us to have \0 in the character set */ static struct fsm * intersect_charset(bool show_stats, const char *charset, struct fsm *fsm) @@ -283,9 +285,6 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags } } - enum re_strings_flags flags = literal_flags(category); - assert(flags >= 0 && flags <= 3); - *lit_s_out = lit_s; *lit_n_out = lit_n; @@ -497,15 +496,18 @@ build_pattern_fsm(const struct fsm_options *opt, bool show_stats, } static int -check_end_ambiguity(const struct fsm *fsm, fsm_state_t s, void *opaque) +check_ambig(const struct fsm *fsm, fsm_state_t s, void *opaque) { + const struct check_env *env = opaque; fsm_end_id_t *ids; size_t count; char buf[50]; /* 50 looks reasonable for an on-screen limit */ int n, res; assert(fsm != NULL); - assert(opaque == NULL); + assert(env != NULL); + assert(env->literals != NULL); + assert(env->general != NULL); if (!fsm_isend(fsm, s)) { return 1; @@ -520,38 +522,61 @@ check_end_ambiguity(const struct fsm *fsm, fsm_state_t s, void *opaque) return 1; } -// TODO: explain this more clearly - fprintf(stderr, "ambigious patterns:"); + n = fsm_example(fsm, s, buf, sizeof buf); + if (-1 == n) { + perror("fsm_example"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "error: ambiguous patterns, for example on input '%s%s':\n", buf, + n >= (int) sizeof buf - 1 ? "..." : ""); ids = xmalloc(count * sizeof *ids); res = fsm_endid_get(fsm, s, count, ids); assert(res == 1); -// TODO: print patterns rather than IDs, pass &literals, &general via opaque, find by id + qsort(ids, count, sizeof *ids, cmp_endid); + for (fsm_end_id_t i = 0; i < count; i++) { - fprintf(stderr, " #%u", ids[i]); - } + for (size_t k = 0; k < sizeof *env->literals / sizeof **env->literals; k++) { + const struct literal_set *literals = &(*env->literals)[k]; + for (size_t j = 0; j < literals->count; j++) { + if (literals->a[j].id == ids[i]) { + // TODO: escape, centralise with libre's re_perror() + fprintf(stderr, "#%u: /%s%.*s%s/\n", + ids[i], + (k & RE_STRINGS_ANCHOR_LEFT) ? "^" : "", + (int) literals->a[j].n, (const char *) literals->a[j].p, + (k & RE_STRINGS_ANCHOR_RIGHT) ? "$" : ""); + goto next; + } + } + } - free(ids); + for (size_t j = 0; j < env->general->count; j++) { + // TODO: delimiters per dialect, centralise with libre's re_perror() + if (env->general->a[i].id == ids[i]) { + fprintf(stderr, "#%u: /%s/\n", + ids[i], + env->general->a[i].s); + goto next; + } + } - n = fsm_example(fsm, s, buf, sizeof buf); - if (-1 == n) { - perror("fsm_example"); - exit(EXIT_FAILURE); +next: ; } - fprintf(stderr, "; for example on input '%s%s'\n", buf, - n >= (int) sizeof buf - 1 ? "..." : ""); + free(ids); return 0; } static int endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) + const void *opaque) { - const struct endleaf_env *env = endleaf_opaque; + const struct endleaf_env *env = opaque; assert(ids != NULL); assert(env != NULL); @@ -567,7 +592,7 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, assert(ids[i] <= INT_MAX); } - /* exactly one end id means no ambiguious patterns */ + /* exactly one end id means no ambiguous patterns */ switch (env->ambig) { case AMBIG_ERROR: @@ -582,7 +607,7 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, * The libfsm api guarentees these ids are unique, * and only appear once each, but are not sorted. */ - fprintf(f, "return %u;", min_id(ids, count)); + fprintf(f, "return %u;", min_endid(ids, count)); break; case AMBIG_MULTIPLE: @@ -1202,13 +1227,6 @@ main(int argc, char *argv[]) } } - for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { - for (size_t j = 0; j < literals[i].count; j++) { - free((void *) literals[i].a[j].p); /* allocated by re_is_literal() */ - } - free(literals[i].a); - } - /* individual regexps */ { for (size_t i = 0; i < general.count; i++) { @@ -1238,11 +1256,6 @@ main(int argc, char *argv[]) } } - for (size_t i = 0; i < general.count; i++) { - free((void *) general.a[i].s); - } - free(general.a); - fsms = xrealloc(fsms, fsm_count * sizeof *fsms); if (show_stats) { @@ -1344,7 +1357,10 @@ main(int argc, char *argv[]) } if (ambig == AMBIG_ERROR) { - if (!fsm_walk_states(fsm, NULL, check_end_ambiguity)) { + if (!fsm_walk_states(fsm, + & (struct check_env) { &literals, &general }, + check_ambig)) + { exit(EXIT_FAILURE); } } @@ -1356,6 +1372,18 @@ main(int argc, char *argv[]) fsm_free(fsm); } + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + for (size_t j = 0; j < literals[i].count; j++) { + free((void *) literals[i].a[j].p); /* allocated by re_is_literal() */ + } + free(literals[i].a); + } + + for (size_t i = 0; i < general.count; i++) { + free((void *) general.a[i].s); + } + free(general.a); + #ifdef __linux__ if (show_stats) { struct rusage ru; From 4e5cf35c1c4e8813c550a68a027090471c79bcd3 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 21 Jun 2024 16:27:02 +0100 Subject: [PATCH 169/373] Switch to reading and allocating line-by-line, rather than mmap(). This allows for eventually iterating over multiple input files. --- src/rx/main.c | 243 ++++++++++++++++++++++++++++---------------------- 1 file changed, 137 insertions(+), 106 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 7e81a78d3..447c0d1e5 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -20,10 +20,12 @@ #include #include +#include #include #include #include #include +#include #include @@ -88,11 +90,81 @@ struct check_env { typedef void (print_fsm)(struct fsm *, const char *, enum ambig); +// TODO: centralise, maybe adt/alloc.h +static FILE * +xopen(const char *s) +{ + FILE *f; + + assert(s != NULL); + + if (0 == strcmp(s, "-")) { + return stdin; + } + + f = fopen(s, "r"); + if (f == NULL) { + perror(s); + exit(EXIT_FAILURE); + } + + return f; +} + +// TODO: centralise +static char * +xgetline(FILE *f) +{ + char *buf; + size_t len; + ssize_t n; + + buf = NULL; + len = 0; + + n = getline(&buf, &len, f); + if (n == -1) { + free(buf); + + if (feof(f)) { + return NULL; + } + + perror("getline"); + exit(EXIT_FAILURE); + } + + /* n == 0 for an empty line */ + if (n > 0) { + assert(buf[n - 1] == '\n'); + buf[n - 1] = '\0'; + } + + return buf; +} + static void -append_pattern(struct pattern_set *set, fsm_end_id_t id, enum re_dialect dialect, const char *s) +append_pattern(struct pattern_set *set, + fsm_end_id_t id, enum re_dialect dialect, const char *s) { + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + assert(set != NULL); - assert(set->a != NULL); + assert(s != NULL); + + if (set->count == 0) { + set->a = xmalloc(low * sizeof *set->a); + } else if (set->count >= low && (set->count & (set->count - 1)) == 0) { + size_t new = set->count * factor; + if (new < set->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + set->a = xrealloc(set->a, new * sizeof *set->a); + } set->a[set->count].id = id; set->a[set->count].dialect = dialect; @@ -101,12 +173,28 @@ append_pattern(struct pattern_set *set, fsm_end_id_t id, enum re_dialect dialect } static void -append_literal(struct literal_set *set, const char *p, size_t n, fsm_end_id_t id) +append_literal(struct literal_set *set, + const char *p, size_t n, fsm_end_id_t id) { + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + assert(set != NULL); - assert(set->a != NULL); assert(p != NULL); + if (set->count == 0) { + set->a = xmalloc(low * sizeof *set->a); + } else if (set->count >= low && (set->count & (set->count - 1)) == 0) { + size_t new = set->count * factor; + if (new < set->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + set->a = xrealloc(set->a, new * sizeof *set->a); + } + set->a[set->count].id = id; set->a[set->count].p = p; set->a[set->count].n = n; @@ -536,7 +624,7 @@ check_ambig(const struct fsm *fsm, fsm_state_t s, void *opaque) res = fsm_endid_get(fsm, s, count, ids); assert(res == 1); - qsort(ids, count, sizeof *ids, cmp_endid); + qsort(ids, count, sizeof *ids, cmp_id); for (fsm_end_id_t i = 0; i < count; i++) { for (size_t k = 0; k < sizeof *env->literals / sizeof **env->literals; k++) { @@ -816,7 +904,6 @@ main(int argc, char *argv[]) size_t general_limit = 0; const char *charset = NULL; - const char *input_file = NULL; const char *declined_file = NULL; enum re_dialect dialect = RE_PCRE; @@ -950,8 +1037,6 @@ main(int argc, char *argv[]) usage(name); exit(EXIT_FAILURE); } - - input_file = argv[0]; } if (charset != NULL && strchr(charset, '\n')) { @@ -966,72 +1051,16 @@ main(int argc, char *argv[]) fprintf(stderr, "reject: [%s]\n", reject); } - struct stat sb; - char *addr; - int fd; - - size_t patterns_count; - /* * Phases: * - * 1. per byte, delimiting - * 2. per pattern, categorization - * 3. per set of literals/general regex to FSMs - * 4. union FSMs, codegen + * 1. per pattern, categorization + * 2. per set of literals/general regex to FSMs + * 3. union FSMs, codegen */ - { - fd = open(input_file, O_RDONLY); - if (fd == -1) { - perror(input_file); - exit(EXIT_FAILURE); - } - - if (fstat(fd, &sb) == -1) { - perror(input_file); - exit(EXIT_FAILURE); - } - - addr = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(EXIT_FAILURE); - } - - /* we assume a trailing newline per pattern in the getc callback */ - if (sb.st_size == 0 || addr[sb.st_size - 1] != '\n') { - fprintf(stderr, "%s: missing newline at end of file\n", input_file); - exit(EXIT_FAILURE); - } - - /* - * Here we iterate per byte, finding the \n delimiter for patterns. - * Henceforth we iterate per pattern id, not per byte. - */ - patterns_count = 0; - for (const char *p = addr; p - addr < sb.st_size; p++) { - if (*p == '\n') { - patterns_count++; - } - } - - /* - * This is so we can fit the endid value in an int for the - * return value of the generated code. - */ - if (patterns_count > INT_MAX) { - fprintf(stderr, "pattern count overflow\n"); - exit(EXIT_FAILURE); - } - - if (show_stats) { - fprintf(stderr, "patterns: %zu\n", patterns_count); - } - } - if (general_limit == 0) { - general_limit = patterns_count; + general_limit = SIZE_MAX; } /* @@ -1046,17 +1075,10 @@ main(int argc, char *argv[]) struct pattern_set general; struct literal_set literals[4]; /* unanchored, left, right, both */ - /* - * We allocate the worse case of patterns_count for these, - * prior to categorization. And then realloc down later. - */ - declined.a = xmalloc(patterns_count * sizeof *declined.a); declined.count = 0; - general.a = xmalloc(patterns_count * sizeof *general.a); - general.count = 0; + general.count = 0; for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { - literals[i].a = xmalloc(patterns_count * sizeof *literals[i].a); literals[i].count = 0; } @@ -1075,28 +1097,32 @@ main(int argc, char *argv[]) * Categorize patterns */ { - const char *q = addr; + FILE *f; + char *s; + +// TODO: iterate argv[] here + + f = xopen(argv[0]); - for (fsm_end_id_t id = 0; id < patterns_count; id++) { + for (fsm_end_id_t id = 0; (s = xgetline(f)); id++) { struct re_err err; - char *s, *lit_s; + char *lit_s; size_t lit_n; - size_t n; enum category r; - // TODO: would fgets() here. then we can get rid of patterns_count - n = strcspn(q, "\n"); - - s = xmalloc(n + 1); - memcpy(s, q, n); - s[n] = '\0'; + /* + * This is so we can fit the endid value in an int for the + * return value of the generated code. + */ + if (id > INT_MAX) { + fprintf(stderr, "pattern count overflow\n"); + exit(EXIT_FAILURE); + } r = categorize(&opt, dialect, flags, charset, reject, s, &err, &lit_s, &lit_n); - q += n + 1; - /* * For pattern sets, s storage belongs to the entry in the set * (and is freed when the set is destroyed). For literals, we're @@ -1154,33 +1180,32 @@ main(int argc, char *argv[]) } } + fclose(f); + /* * realloc down to size, note this leaves some .a arrays NULL * when its count is 0. */ - declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); - general.a = xrealloc(general.a, general.count * sizeof *general.a); + if (declined.count > 0) { + declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); + } + if (general.count > 0) { + general.a = xrealloc(general.a, general.count * sizeof *general.a); + } for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { - literals[i].a = xrealloc(literals[i].a, literals[i].count * sizeof *literals[i].a); + if (literals[i].count > 0) { + literals[i].a = xrealloc(literals[i].a, literals[i].count * sizeof *literals[i].a); + } } } - if (-1 == munmap(addr, sb.st_size)) { - perror("munmap"); - exit(EXIT_FAILURE); - } - - if (-1 == close(fd)) { - perror("close"); - exit(EXIT_FAILURE); - } - /* * This is conservative allocation, the declined list may still grow yet. * We track actual usage in fsm_count. */ - struct fsm **fsms = xmalloc(((sizeof literals / sizeof *literals) - + patterns_count - declined.count) * sizeof *fsms); + struct fsm **fsms = xmalloc( + (general.count + sizeof literals / sizeof *literals) + * sizeof *fsms); size_t fsm_count = 0; /* @@ -1304,7 +1329,9 @@ main(int argc, char *argv[]) for (size_t i = 0; i < declined.count; i++) { free((void *) declined.a[i].s); } - free(declined.a); + if (declined.count > 0) { + free(declined.a); + } if (show_stats) { fprintf(stderr, "fsm_count = %zu FSMs prior to union\n", fsm_count); @@ -1376,13 +1403,17 @@ main(int argc, char *argv[]) for (size_t j = 0; j < literals[i].count; j++) { free((void *) literals[i].a[j].p); /* allocated by re_is_literal() */ } - free(literals[i].a); + if (literals[i].count > 0) { + free(literals[i].a); + } } for (size_t i = 0; i < general.count; i++) { free((void *) general.a[i].s); } - free(general.a); + if (general.count > 0) { + free(general.a); + } #ifdef __linux__ if (show_stats) { From 341dfc86d0830734d1cac9e1e90b25a23a2d48f8 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 21 Jun 2024 17:23:05 +0100 Subject: [PATCH 170/373] C99 doesn't allow a different type qualifier here, only C2x. --- src/rx/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rx/main.c b/src/rx/main.c index 447c0d1e5..c225cdff0 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -84,7 +84,7 @@ struct endleaf_env { }; struct check_env { - const struct literal_set (*literals)[4]; + struct literal_set (*literals)[4]; const struct pattern_set *general; }; From 9779714007cae455c91d999945e4934fd01643b4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 22 Jun 2024 20:44:20 +0100 Subject: [PATCH 171/373] Multi-file input. When only one file is given, the endid is per pattern (that is, the line number) in the input file. When multiple files are given, the endid is the argv[] index the file. That is, all patterns within each file share the same endid. --- src/rx/main.c | 79 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index c225cdff0..2c2c03734 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -57,13 +57,13 @@ enum category { }; struct pattern { - fsm_end_id_t id; + fsm_end_id_t id; /* either per file or per pattern */ enum re_dialect dialect; const char *s; }; struct literal { - fsm_end_id_t id; + fsm_end_id_t id; /* either per file or per literal */ const void *p; size_t n; }; @@ -887,7 +887,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] input-file\n", name); + printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); printf(" %s -h\n", name); } @@ -928,10 +928,12 @@ main(int argc, char *argv[]) .io = FSM_IO_PAIR }; +// TODO: prepend comment with rx invocation to output // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: manpage // TODO: cli flag to set prefix -// TODO: argv[] multiple files, one per id +// TODO: option to set dialect per file from file extension +// TODO: consider de-duplicating arrays. not sure there's any reason { const char *name = argv[0]; @@ -1033,7 +1035,7 @@ main(int argc, char *argv[]) argc -= optind; argv += optind; - if (argc != 1) { + if (argc == 0) { usage(name); exit(EXIT_FAILURE); } @@ -1096,15 +1098,18 @@ main(int argc, char *argv[]) /* * Categorize patterns */ - { + for (int arg = 0; arg < argc; arg++) { + fsm_end_id_t id = 0; FILE *f; char *s; -// TODO: iterate argv[] here + f = xopen(argv[arg]); - f = xopen(argv[0]); + if (argc > 1) { + id = arg; + } - for (fsm_end_id_t id = 0; (s = xgetline(f)); id++) { + while (s = xgetline(f), s != NULL) { struct re_err err; char *lit_s; size_t lit_n; @@ -1135,39 +1140,43 @@ main(int argc, char *argv[]) if (verbose) { switch (r) { case CATEGORY_LITERAL: - fprintf(stderr, "literal #%u '%.*s'\n", - id, (int) lit_n, lit_s); + fprintf(stderr, "literal %s:#%u '%.*s'\n", + argv[arg], id, (int) lit_n, lit_s); break; case CATEGORY_EMPTY: /* fallthrough */ case CATEGORY_GENERAL: - fprintf(stderr, "general #%u: /%s/\n", - id, s); + fprintf(stderr, "general %s:#%u: /%s/\n", + argv[arg], id, s); break; case CATEGORY_ERROR: - fprintf(stderr, "declined (%s) #%u: /%s/", - category_reason(r), id, s); + fprintf(stderr, "declined (%s) %s:#%u: /%s/", + category_reason(r), + argv[arg], id, s); re_perror(dialect, &err, NULL, NULL); break; default: - fprintf(stderr, "declined (%s) #%u: /%s/\n", - category_reason(r), id, s); + fprintf(stderr, "declined (%s) %s:#%u: /%s/\n", + category_reason(r), + argv[arg], id, s); break; } } switch (r) { case CATEGORY_LITERAL: - append_literal(&literals[flags], lit_s, lit_n, id); + append_literal(&literals[flags], lit_s, lit_n, + id); break; case CATEGORY_EMPTY: /* fallthrough */ case CATEGORY_GENERAL: - append_pattern(general.count < general_limit ? &general : &declined, + append_pattern( + general.count < general_limit ? &general : &declined, id, dialect, s); break; @@ -1178,24 +1187,28 @@ main(int argc, char *argv[]) append_pattern(&declined, id, dialect, s); } + + if (argc == 1) { + id++; + } } fclose(f); + } - /* - * realloc down to size, note this leaves some .a arrays NULL - * when its count is 0. - */ - if (declined.count > 0) { - declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); - } - if (general.count > 0) { - general.a = xrealloc(general.a, general.count * sizeof *general.a); - } - for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { - if (literals[i].count > 0) { - literals[i].a = xrealloc(literals[i].a, literals[i].count * sizeof *literals[i].a); - } + /* + * realloc down to size, note this leaves some .a arrays NULL + * when its count is 0. + */ + if (declined.count > 0) { + declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); + } + if (general.count > 0) { + general.a = xrealloc(general.a, general.count * sizeof *general.a); + } + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + if (literals[i].count > 0) { + literals[i].a = xrealloc(literals[i].a, literals[i].count * sizeof *literals[i].a); } } From b436b8101278f20ca126358978fd68cc3ed5bd2b Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 24 Jun 2024 13:15:32 +0100 Subject: [PATCH 172/373] Skip re_is_literal()'s parsing an AST construction for literals. This should find the same result anyway, there's just no need to go to the trouble of constructing an AST here, when we already know the pattern is a literal. --- src/rx/main.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 2c2c03734..7946956e3 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -298,7 +298,7 @@ static enum category categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags flags, const char *charset, const char *reject, const char *s, struct re_err *err, - char **lit_s_out, size_t *lit_n_out) + const char **lit_s_out, size_t *lit_n_out) { enum re_literal_category category; char *lit_s; @@ -309,6 +309,18 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags assert(lit_s_out != NULL); assert(lit_n_out != NULL); + /* + * As an optimisation, we skip libre's parsing for RE_LITERAL and + * categorize empirically based on the dialect. We can't do this + * inside re_is_literal() because that uses the getc callback, + * and this shortcut depends on the storage for the input string s. + */ + if (dialect == RE_LITERAL) { + *lit_s_out = s; + *lit_n_out = strlen(s); + return CATEGORY_LITERAL; + } + /* * -1: Error * 1: Literal, *s may or may not be NULL (i.e. for an empty string), *n >= 0 @@ -934,6 +946,8 @@ main(int argc, char *argv[]) // TODO: cli flag to set prefix // TODO: option to set dialect per file from file extension // TODO: consider de-duplicating arrays. not sure there's any reason +// TODO: decide if i want dialect==RE_LITERAL unanchored or not, maybe a cli flag either way +// TODO: maybe a re(1)-style -F... argument to |= in some flags { const char *name = argv[0]; @@ -1111,7 +1125,7 @@ main(int argc, char *argv[]) while (s = xgetline(f), s != NULL) { struct re_err err; - char *lit_s; + const char *lit_s; size_t lit_n; enum category r; @@ -1133,7 +1147,7 @@ main(int argc, char *argv[]) * (and is freed when the set is destroyed). For literals, we're * done with the original syntax, and use lit_s/lit_n instead. */ - if (r == CATEGORY_LITERAL) { + if (dialect != RE_LITERAL && r == CATEGORY_LITERAL) { free(s); } From d546585a7eaee4b4e68808c5221f941a9108626f Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 24 Jun 2024 13:16:30 +0100 Subject: [PATCH 173/373] Override dialect by file extension. --- src/rx/main.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/rx/main.c b/src/rx/main.c index 7946956e3..901086134 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -944,7 +944,6 @@ main(int argc, char *argv[]) // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: manpage // TODO: cli flag to set prefix -// TODO: option to set dialect per file from file extension // TODO: consider de-duplicating arrays. not sure there's any reason // TODO: decide if i want dialect==RE_LITERAL unanchored or not, maybe a cli flag either way // TODO: maybe a re(1)-style -F... argument to |= in some flags @@ -1117,6 +1116,14 @@ main(int argc, char *argv[]) FILE *f; char *s; + const char *ext = strrchr(argv[arg], '.'); + if (ext != NULL) { + ext++; + fprintf(stderr, "overriding dialect by extension for %s: %s\n", + argv[arg], ext); + dialect = dialect_name(ext); + } + f = xopen(argv[arg]); if (argc > 1) { From 6d2f1f3da51bbe66cd702f11215be02c016d77c1 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 24 Jun 2024 15:56:21 +0100 Subject: [PATCH 174/373] -p for prefix. --- src/rx/main.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 901086134..1aedd390b 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -899,7 +899,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); + printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-p prefix] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); printf(" %s -h\n", name); } @@ -943,7 +943,6 @@ main(int argc, char *argv[]) // TODO: prepend comment with rx invocation to output // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) // TODO: manpage -// TODO: cli flag to set prefix // TODO: consider de-duplicating arrays. not sure there's any reason // TODO: decide if i want dialect==RE_LITERAL unanchored or not, maybe a cli flag either way // TODO: maybe a re(1)-style -F... argument to |= in some flags @@ -952,7 +951,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:ikl::n:r:R:Qquv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:ikl::n:p:r:R:Qquv"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -1006,6 +1005,10 @@ main(int argc, char *argv[]) general_limit = atoi(optarg); break; + case 'p': + opt.prefix = optarg; + break; + case 'R': reject = optarg; From 45e3cfc733740d6ddbf552cee0b1260e971e8849 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 24 Jun 2024 16:12:31 +0100 Subject: [PATCH 175/373] -F for re_flags. --- src/rx/main.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 1aedd390b..7833ed5ef 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -888,6 +888,33 @@ print_name(const char *name) exit(EXIT_FAILURE); } +/* TODO: centralize */ +static void +parse_flags(const char *arg, enum re_flags *flags) +{ + for (; *arg; arg++) { + switch (*arg) { + case 'i': + *flags = *flags | RE_ICASE; + break; + + case 's': + *flags = *flags | RE_SINGLE; + break; + + case 'x': + *flags = *flags | RE_EXTENDED; + break; + + /* others? */ + + default: + printf("unknown flag '%c'\n", *arg); + exit(EXIT_FAILURE); + } + } +} + static void usage(const char *name) { @@ -924,7 +951,7 @@ main(int argc, char *argv[]) * TODO: we need to provide a way to disable RE_END_NL * TODO: RE_SINGLE. also be careful, flags disrupt re_is_literal() */ - const enum re_flags flags = 0; // TODO: notably not RE_END_NL + enum re_flags flags = 0; // TODO: notably not RE_END_NL /* regex syntax characters we decline at compile time * because NFA->DFA is too costly */ @@ -945,13 +972,12 @@ main(int argc, char *argv[]) // TODO: manpage // TODO: consider de-duplicating arrays. not sure there's any reason // TODO: decide if i want dialect==RE_LITERAL unanchored or not, maybe a cli flag either way -// TODO: maybe a re(1)-style -F... argument to |= in some flags { const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:ikl::n:p:r:R:Qquv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:ikF:l::n:p:r:R:Qquv"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -1001,6 +1027,10 @@ main(int argc, char *argv[]) print = print_name(optarg); break; + case 'F': + parse_flags(optarg, &flags); + break; + case 'n': general_limit = atoi(optarg); break; @@ -1067,6 +1097,7 @@ main(int argc, char *argv[]) fprintf(stderr, "charset: [%s]\n", charset == NULL ? "(none)" : charset); fprintf(stderr, "reject: [%s]\n", reject); + fprintf(stderr, "flags: %#x\n", (unsigned) flags); } /* From c7d242f50e42bc23d433565a0aad50ed08faf3a1 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 24 Jun 2024 16:18:22 +0100 Subject: [PATCH 176/373] No need to handle endids during fsm_minimise here. --- src/rx/main.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 7833ed5ef..173aa6531 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -556,11 +556,6 @@ build_pattern_fsm(const struct fsm_options *opt, bool show_stats, assert(!fsm_empty(fsm)); - if (!fsm_setendid(fsm, id)) { - perror("fsm_setendid"); - exit(EXIT_FAILURE); - } - // XXX: collapses down to nothing because for ^.*abc.*$ the middle is a subset of the .* // XXX: not sure at all // XXX: this is the wrong operation! @@ -588,10 +583,13 @@ build_pattern_fsm(const struct fsm_options *opt, bool show_stats, exit(EXIT_FAILURE); } -// TODO: therefore set endid after minimisation - assert(!fsm_empty(fsm)); + if (!fsm_setendid(fsm, id)) { + perror("fsm_setendid"); + exit(EXIT_FAILURE); + } + return fsm; } From 53164aea894421e3914d6c7fe65f9f876eb305e6 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 24 Jun 2024 16:45:09 +0100 Subject: [PATCH 177/373] Don't call intersect_charset() with an empty charset, it's clearer to handle in the caller. --- src/rx/main.c | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 173aa6531..22c3a29a1 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -244,10 +244,7 @@ intersect_charset(bool show_stats, const char *charset, struct fsm *fsm) { struct fsm *q; - if (charset == NULL) { - return fsm; - } - + assert(charset != NULL); assert(fsm_all(fsm, fsm_isdfa)); size_t pre = fsm_countstates(fsm); @@ -478,20 +475,22 @@ build_literals_fsm(const struct fsm_options *opt, bool show_stats, assert(!fsm_empty(fsm)); } - fsm = intersect_charset(show_stats, charset, fsm); - if (fsm == NULL) { - perror("intersect_charset"); - exit(EXIT_FAILURE); + if (charset != NULL) { + fsm = intersect_charset(show_stats, charset, fsm); + if (fsm == NULL) { + perror("intersect_charset"); + exit(EXIT_FAILURE); + } + + if (fsm_countstates(fsm) == 0) { + fprintf(stderr, "literals produced empty intersection\n"); + exit(EXIT_FAILURE); + } } /* We don't minimise here because this fsm has multiple endids, * and the resulting FSM would be very similar to the current DFA */ - if (fsm_countstates(fsm) == 0) { - fprintf(stderr, "literals produced empty intersection\n"); - exit(EXIT_FAILURE); - } - #ifndef NDEBUG /* * We could test to see that the fsm isn't any different. @@ -556,22 +555,19 @@ build_pattern_fsm(const struct fsm_options *opt, bool show_stats, assert(!fsm_empty(fsm)); -// XXX: collapses down to nothing because for ^.*abc.*$ the middle is a subset of the .* -// XXX: not sure at all + if (charset != NULL) { // XXX: this is the wrong operation! -// instead of .* we should go edge-by-edge and remove anything not in the charset -// but that would give the same result -// i want to intersect_charset but without folding the fsm down to nothing // XXX: i *think* intersect_charset must be doing the wrong thing. somehow - (void) charset; - (void) show_stats; + (void) charset; + (void) show_stats; /* - fsm = intersect_charset(show_stats, charset, fsm); - if (fsm == NULL) { - perror("intersect_charset"); - exit(EXIT_FAILURE); - } + fsm = intersect_charset(show_stats, charset, fsm); + if (fsm == NULL) { + perror("intersect_charset"); + exit(EXIT_FAILURE); + } */ + } /* * This FSM has only one endid. fsm_minimise() does now maintain From 423230353f278e886aa03f48f0cdd6c2bdf0e5b2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 25 Jun 2024 10:43:58 +0100 Subject: [PATCH 178/373] Comment generated code --- src/libfsm/print/awk.c | 1 + src/libfsm/print/go.c | 1 + src/libfsm/print/irdot.c | 1 + src/libfsm/print/rust.c | 2 ++ src/libfsm/print/sh.c | 1 + src/libfsm/print/vmasm.c | 3 +++ src/libfsm/print/vmdot.c | 3 +-- 7 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index 462a3ab7f..e91a75537 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -283,6 +283,7 @@ fsm_print_awk_complete(FILE *f, const struct ir *ir, } else { fprintf(f, "\n"); + fprintf(f, "# generated\n"); fprintf(f, "function %smain(", prefix); switch (opt->io) { diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 7b3616f9d..f235c2a5d 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -279,6 +279,7 @@ fsm_print_go_complete(FILE *f, const struct ir *ir, fprintf(f, "package %sfsm\n", package_prefix); fprintf(f, "\n"); + fprintf(f, "// generated\n"); fprintf(f, "func %sMatch", prefix); switch (opt->io) { diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 8e91029ca..76c895092 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -335,6 +335,7 @@ fsm_print_ir(FILE *f, const struct fsm *fsm) return -1; } + fprintf(f, "# generated\n"); fprintf(f, "digraph G {\n"); fprintf(f, "\tnode [ shape = box, style = rounded ];\n"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index af2b0f302..7d2b16d52 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -384,6 +384,8 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, return -1; } + fprintf(f, "// generated\n"); + if (opt->fragment) { fsm_print_rustfrag(f, &a, ir, opt, cp, opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 04e84663e..ba2c096d7 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -202,6 +202,7 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "IFS=\n"); fprintf(f, "\n"); + fprintf(f, "# generated\n"); fprintf(f, "while true; do\n"); fprintf(f, "\tcase $l in\n"); fprintf(f, "\tstart)\n"); diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 046610c44..82bafe96c 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -83,12 +83,14 @@ print_asm_amd64(FILE *f, const char *prefix, /* print preamble */ switch (dialect) { case AMD64_ATT: + fprintf(f, "%s generated\n", comment); fprintf(f, ".globl %s%s%s\n", sigil, prefix, "match"); fprintf(f, ".text\n"); fprintf(f, "%s%s%s:\n", sigil, prefix, "match"); break; case AMD64_NASM: + fprintf(f, "%s generated\n", comment); fprintf(f, "section .text\n"); fprintf(f, "global %s%s%s\n", sigil, prefix, "match"); fprintf(f, "%s%s%s:\n", sigil, prefix, "match"); @@ -99,6 +101,7 @@ print_asm_amd64(FILE *f, const char *prefix, fprintf(f, "#include \"textflag.h\"\n"); fprintf(f, "\n"); + fprintf(f, "%s generated\n", comment); switch (opt->io) { case FSM_IO_STR: fprintf(f, "// func %s%s(data string) int\n", prefix, "Match"); diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index bdcb2e3de..990ea157f 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -344,8 +344,7 @@ fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options return -1; } } else { - fprintf(f, "\n"); - + fprintf(f, "# generated\n"); fprintf(f, "digraph G {\n"); fprintf(f, "\tnode [ shape = plaintext ];\n"); From ff51a75d245ba1ea7aabaf9228ce8d23f0efd98a Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 25 Jun 2024 10:54:37 +0100 Subject: [PATCH 179/373] Codegen for various languages. I'm not particularly thrilled about the handling for different AMBIG_ modes here. I'm thinking eventually it might make sense to move this stuff into libfsm proper and share it with the other cli tools. --- src/rx/main.c | 118 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 22c3a29a1..da0b3f45a 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -40,10 +40,14 @@ #include #include +/* TODO: find a better way to handle this, maybe centralise with re(1) */ enum ambig { - AMBIG_ERROR, - AMBIG_EARLIEST, - AMBIG_MULTIPLE + AMBIG_ERROR = 1 << 0, + AMBIG_EARLIEST = 1 << 1, + AMBIG_MULTIPLE = 1 << 2, + + AMBIG_SINGLE = AMBIG_ERROR | AMBIG_EARLIEST, + AMBIG_ANY = ~0 }; enum category { @@ -88,8 +92,6 @@ struct check_env { const struct pattern_set *general; }; -typedef void (print_fsm)(struct fsm *, const char *, enum ambig); - // TODO: centralise, maybe adt/alloc.h static FILE * xopen(const char *s) @@ -737,14 +739,22 @@ endleaf_c(FILE *f, const fsm_end_id_t *ids, size_t count, return 0; } -static void -print_fsm_vmc(struct fsm *fsm, const char *prefix, enum ambig ambig) +static int +print_vmc_ambig(FILE *f, const struct fsm *fsm, enum ambig ambig) { const struct fsm_options *old; struct fsm_options tmp; - if (prefix == NULL) { - prefix = "fsm"; + old = fsm_getoptions(fsm); + assert(old != NULL); + + tmp = *old; + tmp.fragment = true, + tmp.endleaf_opaque = & (struct endleaf_env) { ambig, fsm }; + tmp.endleaf = endleaf_c; + + if (tmp.prefix == NULL) { + tmp.prefix = "fsm"; } fprintf(stdout, "/* generated */\n"); @@ -755,10 +765,9 @@ print_fsm_vmc(struct fsm *fsm, const char *prefix, enum ambig ambig) fprintf(stdout, "\n"); // TODO: not bool? -// TODO: return type depends on enum ambig -// TODO: but also override return -1 +// TODO: also override return -1 fprintf(stdout, "int\n"); - fprintf(stdout, "%s_main(const char *s, size_t n", prefix); + fprintf(stdout, "%s_main(const char *s, size_t n", tmp.prefix); if (ambig == AMBIG_MULTIPLE) { fprintf(stdout, ",\n"); fprintf(stdout, "\tconst unsigned **ids, size_t *count"); @@ -767,19 +776,31 @@ print_fsm_vmc(struct fsm *fsm, const char *prefix, enum ambig ambig) fprintf(stdout, "{\n"); fprintf(stdout, "\tconst char *b = s, *e = s + n;\n"); - old = fsm_getoptions(fsm); - assert(old != NULL); + fsm_setoptions((struct fsm *) fsm, &tmp); /* XXX: const */ + fsm_print_vmc(f, fsm); + fsm_setoptions((struct fsm *) fsm, old); /* XXX: const */ - tmp = *old; - tmp.fragment = true, - tmp.endleaf_opaque = & (struct endleaf_env) { ambig, fsm }; - tmp.endleaf = endleaf_c, + fprintf(stdout, "}\n"); - fsm_setoptions(fsm, &tmp); - fsm_print_vmc(stdout, fsm); - fsm_setoptions(fsm, old); + return 0; /* XXX */ +} - fprintf(stdout, "}\n"); +static int +print_vmc_error(FILE *f, const struct fsm *fsm) +{ + return print_vmc_ambig(f, fsm, AMBIG_ERROR); +} + +static int +print_vmc_earliest(FILE *f, const struct fsm *fsm) +{ + return print_vmc_ambig(f, fsm, AMBIG_EARLIEST); +} + +static int +print_vmc_multiple(FILE *f, const struct fsm *fsm) +{ + return print_vmc_ambig(f, fsm, AMBIG_MULTIPLE); } static enum re_dialect @@ -851,24 +872,54 @@ io(const char *name) exit(EXIT_FAILURE); } -static print_fsm * -print_name(const char *name) +static fsm_print * +print_name(const char *name, enum ambig ambig) { size_t i; + bool failed_mask; struct { const char *name; - print_fsm *print; + fsm_print *print; + enum ambig mask; } a[] = { - { "vmc", print_fsm_vmc } + { "c", print_vmc_error, AMBIG_ERROR }, + { "c", print_vmc_earliest, AMBIG_EARLIEST }, + { "c", print_vmc_multiple, AMBIG_MULTIPLE }, + { "fsm", fsm_print_fsm, AMBIG_ANY }, + { "rust", fsm_print_rust, AMBIG_ERROR }, + { "go", fsm_print_go, AMBIG_ERROR }, + { "ir", fsm_print_ir, AMBIG_ANY }, + { "irjson", fsm_print_irjson, AMBIG_ANY }, + { "vmdot", fsm_print_vmdot, AMBIG_ANY }, + { "vmops_c", fsm_print_vmops_c, AMBIG_ERROR }, + { "vmops_h", fsm_print_vmops_c, AMBIG_ERROR }, + { "vmops_main", fsm_print_vmops_main, AMBIG_ERROR }, + { "amd64", fsm_print_vmasm_amd64_nasm, AMBIG_ERROR }, + { "amd64_att", fsm_print_vmasm_amd64_att, AMBIG_ERROR }, + { "amd64_nasm", fsm_print_vmasm_amd64_nasm, AMBIG_ERROR }, + { "amd64_go", fsm_print_vmasm_amd64_go, AMBIG_ERROR } }; assert(name != NULL); + failed_mask = false; + for (i = 0; i < sizeof a / sizeof *a; i++) { - if (0 == strcmp(a[i].name, name)) { + if (0 != strcmp(a[i].name, name)) { + continue; + } + + if ((a[i].mask & ambig)) { return a[i].print; } + + failed_mask = true; + } + + if (failed_mask) { + fprintf(stderr, "unsupported ambig mode for output language\n"); + exit(EXIT_FAILURE); } fprintf(stderr, "unrecognised output language; valid languages are: "); @@ -932,7 +983,8 @@ main(int argc, char *argv[]) bool verbose = false; bool show_stats = false; enum ambig ambig = AMBIG_ERROR; - print_fsm *print = print_fsm_vmc; + const char *lang = "c"; + fsm_print *print = NULL; size_t general_limit = 0; @@ -971,7 +1023,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:ikF:l::n:p:r:R:Qquv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:ikF:l:n:p:r:R:Qquv"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -1018,7 +1070,7 @@ main(int argc, char *argv[]) break; case 'l': - print = print_name(optarg); + lang = optarg; break; case 'F': @@ -1081,6 +1133,8 @@ main(int argc, char *argv[]) } } + print = print_name(lang, ambig); + if (charset != NULL && strchr(charset, '\n')) { fprintf(stderr, "charset cannot contain \\n " "(because regexes are delimited by newlines)\n"); @@ -1455,7 +1509,9 @@ main(int argc, char *argv[]) } if (!quiet) { - print(fsm, opt.prefix, ambig); + if (-1 == print(stdout, fsm)) { + exit(EXIT_FAILURE); + } } fsm_free(fsm); From 4aa804a3f3aee919ac16681ecdb5867fccc01c16 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 25 Jun 2024 12:13:21 +0100 Subject: [PATCH 180/373] -x for unanchored literals. --- src/rx/main.c | 58 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index da0b3f45a..9d43b9bcd 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -240,6 +240,28 @@ min_endid(const fsm_end_id_t *ids, size_t count) return min; } +static enum re_strings_flags +literal_flags(enum re_literal_category category) +{ + switch (category) { + case RE_LITERAL_UNANCHORED: + return 0; + + case RE_LITERAL_ANCHOR_START: + return RE_STRINGS_ANCHOR_LEFT; + + case RE_LITERAL_ANCHOR_END: + return RE_STRINGS_ANCHOR_RIGHT; + + case RE_LITERAL_ANCHOR_BOTH: + return RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT; + + default: + assert(!"unreached"); + abort(); + } +} + /* this interface doesn't allow us to have \0 in the character set */ static struct fsm * intersect_charset(bool show_stats, const char *charset, struct fsm *fsm) @@ -297,7 +319,7 @@ static enum category categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags flags, const char *charset, const char *reject, const char *s, struct re_err *err, - const char **lit_s_out, size_t *lit_n_out) + const char **lit_s_out, size_t *lit_n_out, enum re_strings_flags *strings_flags) { enum re_literal_category category; char *lit_s; @@ -317,6 +339,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags if (dialect == RE_LITERAL) { *lit_s_out = s; *lit_n_out = strlen(s); + *strings_flags = RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT; return CATEGORY_LITERAL; } @@ -386,6 +409,7 @@ categorize(const struct fsm_options *opt, enum re_dialect dialect, enum re_flags *lit_s_out = lit_s; *lit_n_out = lit_n; + *strings_flags = literal_flags(category); return CATEGORY_LITERAL; } @@ -971,7 +995,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQquv] [-C charset] [-k io] [-l ] [-p prefix] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); + printf("usage: %s: [-ciQquvx] [-C charset] [-k io] [-l ] [-p prefix] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); printf(" %s -h\n", name); } @@ -982,6 +1006,7 @@ main(int argc, char *argv[]) bool strict = false; bool verbose = false; bool show_stats = false; + bool unanchored_literals = false; enum ambig ambig = AMBIG_ERROR; const char *lang = "c"; fsm_print *print = NULL; @@ -991,7 +1016,7 @@ main(int argc, char *argv[]) const char *charset = NULL; const char *declined_file = NULL; - enum re_dialect dialect = RE_PCRE; + enum re_dialect default_dialect = RE_PCRE; /* * TODO: we need to provide a way to disable RE_END_NL @@ -1023,7 +1048,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:ikF:l:n:p:r:R:Qquv"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:ikF:l:n:p:r:R:Qquvx"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -1094,7 +1119,7 @@ main(int argc, char *argv[]) break; case 'r': - dialect = dialect_name(optarg); + default_dialect = dialect_name(optarg); break; case 'Q': @@ -1117,6 +1142,10 @@ main(int argc, char *argv[]) verbose = true; break; + case 'x': + unanchored_literals = true; + break; + case '?': default: usage(name); @@ -1197,6 +1226,7 @@ main(int argc, char *argv[]) fsm_end_id_t id = 0; FILE *f; char *s; + enum re_dialect dialect = default_dialect; const char *ext = strrchr(argv[arg], '.'); if (ext != NULL) { @@ -1217,6 +1247,7 @@ main(int argc, char *argv[]) const char *lit_s; size_t lit_n; enum category r; + enum re_strings_flags strings_flags; /* * This is so we can fit the endid value in an int for the @@ -1229,7 +1260,7 @@ main(int argc, char *argv[]) r = categorize(&opt, dialect, flags, charset, reject, s, - &err, &lit_s, &lit_n); + &err, &lit_s, &lit_n, &strings_flags); /* * For pattern sets, s storage belongs to the entry in the set @@ -1240,11 +1271,20 @@ main(int argc, char *argv[]) free(s); } + if (dialect == RE_LITERAL && r == CATEGORY_LITERAL && unanchored_literals) { + strings_flags = 0; + } + if (verbose) { switch (r) { case CATEGORY_LITERAL: fprintf(stderr, "literal %s:#%u '%.*s'\n", argv[arg], id, (int) lit_n, lit_s); + fprintf(stderr, "literal %s:#%u /%s%.*s%s/\n", + argv[arg], id, + (strings_flags & RE_STRINGS_ANCHOR_LEFT) ? "^" : "", + (int) lit_n, lit_s, + (strings_flags & RE_STRINGS_ANCHOR_RIGHT) ? "$" : ""); break; case CATEGORY_EMPTY: @@ -1271,7 +1311,9 @@ main(int argc, char *argv[]) switch (r) { case CATEGORY_LITERAL: - append_literal(&literals[flags], lit_s, lit_n, + assert(strings_flags >= 0 && strings_flags <= 3); + append_literal(&literals[strings_flags], + lit_s, lit_n, id); break; @@ -1377,7 +1419,7 @@ main(int argc, char *argv[]) } struct fsm *fsm = build_pattern_fsm(&opt, show_stats, charset, - dialect, strict, + general.a[i].dialect, strict, general.a[i].s, flags, general.a[i].id); if (fsm == NULL) { append_pattern(&declined, i, general.a[i].dialect, general.a[i].s); From e7fd0f418642d4e14b293966bf49a92a01b47be6 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 28 Jun 2024 14:56:53 +0100 Subject: [PATCH 181/373] Include num_re_errors in per-file failures. --- src/retest/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/main.c b/src/retest/main.c index e01c93e7c..9d7d8d583 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -1181,7 +1181,7 @@ process_test_file(const char *filename, exit(EXIT_FAILURE); } - return num_errors; + return num_errors + num_re_errors; } static enum fsm_io From e66e6cf62ca6ac9e54fbee256e5ba69f662d4213 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 28 Jun 2024 15:34:33 +0100 Subject: [PATCH 182/373] Skip an IO API/lang combination that we don't support. --- tests/retest/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/retest/Makefile b/tests/retest/Makefile index ec6cc66a9..5c3512ccd 100644 --- a/tests/retest/Makefile +++ b/tests/retest/Makefile @@ -11,6 +11,9 @@ RETEST=${BUILD}/bin/retest .for lang in vm asm c vmc .for io in pair str +# XXX: we don't have FSM_IO_STR for asm yet +.if ${lang} != asm || ${io} != str + .for n in ${TEST.tests/retest:T:R:C/^tests_//} ${TEST_OUTDIR.tests/retest}/res-${lang}-${io}${n}: ${TEST_SRCDIR.tests/retest}/tests_${n}.tst @@ -22,6 +25,8 @@ test:: ${TEST_OUTDIR.tests/retest}/res-${lang}-${io}${n} .endfor +.endif + .endfor .endfor From b0d1de6242271c0332ceefa40682438cd07f3592 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 28 Jun 2024 15:36:26 +0100 Subject: [PATCH 183/373] Quieten GNU ld about something I don't understand. --- src/retest/runner.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/retest/runner.c b/src/retest/runner.c index c5c2cde82..7d3ddad4a 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -242,8 +242,12 @@ compile(enum implementation impl, return 0; } - if (0 != systemf("%s %s -shared %s -o %s", + if (0 != systemf("%s %s -shared %s %s -o %s", cc ? cc : "gcc", cflags ? cflags : "", + + // for "missing .note.GNU-stack section implies executable stack" + "-Wl,-z,noexecstack", + tmp_o, tmp_so)) { return 0; From d756f9e66589707f33e28cec1c54f604ffedf7ac Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 1 Jul 2024 12:42:08 +0100 Subject: [PATCH 184/373] Sort endids written out by fsm_endid_get(). This is for caller convenience, I keep calling qsort for various reasons whenever I call this function, for example to use memcmp on ids. But the final motivation here was that I wanted to memcmp a const-qualified pointer to an id set stored elsewhere, and I can't do that without allocating for a local copy. And that seems silly when we can just qsort here. --- include/fsm/fsm.h | 3 +-- src/libfsm/endids.c | 4 ++- src/libfsm/minimise.c | 12 --------- src/libfsm/minimise_test_oracle.c | 3 +-- src/libfsm/print/api.c | 15 ----------- src/libfsm/print/fsm.c | 15 ----------- tests/capture/capture4.c | 2 +- tests/endids/endids0.c | 7 +----- tests/endids/endids0_many_endids.c | 13 ++-------- tests/endids/endids1_determinise.c | 7 +----- .../endids/endids1_determinise_and_minimise.c | 7 +----- tests/endids/endids2_union.c | 12 ++------- tests/endids/endids2_union_many_endids.c | 16 ++---------- tests/endids/endids4.c | 12 +-------- tests/endids/endids5.c | 11 +------- tests/endids/endids6.c | 25 +++---------------- tests/endids/endids7.c | 16 ++---------- tests/endids/endids7_with_duplicates.c | 16 ++---------- tests/endids/endids8.c | 10 +------- tests/endids/endids9.c | 9 +------ tests/endids/endids_utils.h | 3 --- tests/endids/utils.c | 11 -------- tests/gen/gtest.c | 2 +- tests/re_strings/testutil.c | 8 +++--- 24 files changed, 33 insertions(+), 206 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 269ee5c87..de5de136e 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -222,8 +222,7 @@ fsm_endid_set(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); * id_buf is expected to have enough cells (according to id_buf_count) * to store all the end IDs. You can find this with fsm_endid_count(). * - * The end IDs in the buffer may appear in any order, - * but will not have duplicates. + * The end IDs in the buffer are sorted and do not have duplicates. * * A state with no end IDs set is considered equivalent to a state * that has the empty set, this API does not distinguish these cases. diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 66860e88a..c86277865 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -734,7 +734,9 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, ids[id_i] = b->ids->ids[id_i]; } - /* todo: could sort them here, if it matters. */ + /* sorting for caller convenience */ + qsort(ids, count, sizeof *ids, cmp_endids); + return 1; } else { /* collision */ #if LOG_ENDIDS > 4 diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 8100e22da..d520b39fa 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -939,14 +939,6 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) return res; } -static int -cmp_end_ids(const void *pa, const void *pb) -{ - const fsm_end_id_t a = *(fsm_end_id_t *)pa; - const fsm_end_id_t b = *(fsm_end_id_t *)pb; - return a < b ? -1 : a > b ? 1 : 0; -} - static int collect_end_ids(const struct fsm *fsm, fsm_state_t s, struct end_metadata_end *e) @@ -965,10 +957,6 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, int res = fsm_endid_get(fsm, s, e->count, e->ids); assert(res == 1); - /* sort, to make comparison easier later */ - qsort(e->ids, e->count, - sizeof(e->ids[0]), cmp_end_ids); - #if LOG_ECS fprintf(stderr, "%d:", s); for (size_t i = 0; i < written; i++) { diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 6cd1848c4..d2cfa404d 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -186,8 +186,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) continue; } - int eres = fsm_endid_get(fsm, i, - count_a, ids_a); + int eres = fsm_endid_get(fsm, i, count_a, ids_a); assert(eres == 1); bool found = false; diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 7a0a9ffa5..f25a1d97a 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -27,19 +27,6 @@ #include #include -/* TODO: centralise */ -static int -comp_end_id(const void *a, const void *b) -{ - assert(a != NULL); - assert(b != NULL); - - if (* (fsm_end_id_t *) a < * (fsm_end_id_t *) b) { return -1; } - if (* (fsm_end_id_t *) a > * (fsm_end_id_t *) b) { return +1; } - - return 0; -} - static int rangeclass(unsigned char x, unsigned char y) { @@ -161,8 +148,6 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) res = fsm_endid_get(fsm, end, count, ids); assert(res == 1); - qsort(ids, count, sizeof *ids, comp_end_id); - fprintf(f, "\tif (fsm_isend(fsm, s[%u])) {\n", end); for (size_t id = 0; id < count; id++) { fprintf(f, "\t\tif (!fsm_endid_set(fsm, s[%u], %zu)) { return 0; }\n", end, id); diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 46bc428ff..a3cb471cf 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -24,19 +24,6 @@ #include #include -/* TODO: centralise */ -static int -comp_end_id(const void *a, const void *b) -{ - assert(a != NULL); - assert(b != NULL); - - if (* (fsm_end_id_t *) a < * (fsm_end_id_t *) b) { return -1; } - if (* (fsm_end_id_t *) a > * (fsm_end_id_t *) b) { return +1; } - - return 0; -} - /* TODO: centralise */ static int findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) @@ -325,8 +312,6 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) } if (count > 0) { - qsort(ids, count, sizeof *ids, comp_end_id); - fprintf(f, " = ["); for (size_t id = 0; id < count; id++) { diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 313086c97..5ba8c941c 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -223,7 +223,7 @@ check(const struct fsm *fsm, const char *string, int gres; fsm_end_id_t ids[2]; - gres = fsm_endid_get(fsm, end, 2, ids); + gres = fsm_endid_get(fsm, end, fsm_endid_count(fsm, end), ids); if (gres != 1) { assert(!"fsm_getendids failed"); } diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index c9117c0b0..1f5e6f9b6 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -48,12 +48,7 @@ int main(void) assert( fsm_endid_count(fsm, state_ind) == 1); - ret = fsm_endid_get( - fsm, - state_ind, - 1, - &id); - + ret = fsm_endid_get(fsm, state_ind, 1, &id); assert(ret == 1); assert(id == 1); diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index 0af45112b..86a66063c 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -100,20 +100,14 @@ int main(void) assert(ret == 1); - /* sort endids and compare */ - qsort(&endids[0], - sizeof endids / sizeof endids[0], sizeof endids[0], - cmp_endids); - for (i=0; i < fsm_endid_count(fsm, state_ind); i++) { + for (i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { assert(endids[i] == sorted_all_ids[i]); } -#if 0 /* endids should be sorted */ - for (i=0; i < nwritten; i++) { + for (i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { assert(endids[i] == sorted_all_ids[i]); } -#endif /* 0 */ nend++; } @@ -160,9 +154,6 @@ int main(void) assert( ids[0] == matches[i].endid ); #endif /* 0 */ - /* sort endids and compare */ - qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < count; j++) { assert(ids[j] == sorted_all_ids[j]); } diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 3860aefca..472807eed 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -44,12 +44,7 @@ int main(void) assert( fsm_endid_count(fsm, state_ind) == 1); - ret = fsm_endid_get( - fsm, - state_ind, - 1, - &endid); - + ret = fsm_endid_get(fsm, state_ind, 1, &endid); assert(ret == 1); assert( endid == 1 ); } diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index bb0711905..a2feafb18 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -49,12 +49,7 @@ int main(void) assert(fsm_endid_count(fsm, state_ind) == 1); - ret = fsm_endid_get( - fsm, - state_ind, - 1, - &endid); - + ret = fsm_endid_get(fsm, state_ind, 1, &endid); assert(ret == 1); assert( endid == 1 ); } diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index c990e7c95..0d6182437 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -60,15 +60,9 @@ int main(void) count = fsm_endid_count(comb, state_ind); // fprintf(stderr, "state %u, count = %zu\n", state_ind, count); + assert(count > 0 && count <= 2); - assert( count > 0 && count <= 2); - - ret = fsm_endid_get( - comb, - state_ind, - 2, - &endids[0]); - + ret = fsm_endid_get( comb, state_ind, count, &endids[0]); assert(ret == 1); if (count == 1) { @@ -158,8 +152,6 @@ int main(void) assert( count == matches[i].count ); - qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < count; j++) { assert( ids[j] == matches[i].endid[j] ); } diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index deafcff1e..a573f1921 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -189,15 +189,9 @@ int main(void) memset(&ids[0], 0, sizeof ids); count = fsm_endid_count(fsm, state_ind); - assert(count > 0 && count <= sizeof ids/sizeof ids[0]); - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids/sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); memset(&tested_pattern[0], 0, sizeof tested_pattern); @@ -253,15 +247,9 @@ int main(void) memset(&ids[0], 0, sizeof ids); count = fsm_endid_count(fsm, state_ind); - assert(count <= NUM_ENDIDS_TOTAL); - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids/sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); } } diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index 80afbcc39..a4530c93b 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -71,20 +71,12 @@ int main(void) int ret; count = fsm_endid_count(comb, state_ind); - printf("state %u count = %zu\n", state_ind, count); - assert(count == 2); - ret = fsm_endid_get( - comb, - state_ind, - 2, - &endids[0]); - + ret = fsm_endid_get(comb, state_ind, count, &endids[0]); assert(ret == 1); - qsort(&endids[0], count, sizeof endids[0], cmp_endids); assert(endids[0] == 1 && endids[1] == 2); } } @@ -149,8 +141,6 @@ int main(void) assert( count == matches[i].count ); - qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < count; j++) { assert( ids[j] == matches[i].endid[j] ); } diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index d6a713843..50030ea0d 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -72,17 +72,10 @@ int main(void) int ret; count = fsm_endid_count(comb, state_ind); - printf("state %u count = %zu\n", state_ind, count); - assert(count == 1); - ret = fsm_endid_get( - comb, - state_ind, - 2, - &endids[0]); - + ret = fsm_endid_get(comb, state_ind, count, &endids[0]); assert(ret == 1); assert(endids[0] == 1); @@ -153,8 +146,6 @@ int main(void) assert( count == matches[i].count ); - qsort(&ids[0], count, sizeof ids[0], cmp_endids); - for (j=0; j < count; j++) { assert( ids[j] == matches[i].endid[j] ); } diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index 34b261118..e237c2881 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -96,15 +96,9 @@ int main(void) int ret; count = fsm_endid_count(fsm, state_ind); + assert(count > 0 && count <= 2); - assert( count > 0 && count <= 2); - - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids / sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); info[ninfo].state = state_ind; @@ -114,7 +108,6 @@ int main(void) assert(ids[0] == 1 || ids[0] == 2); info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&ids[0], count, sizeof ids[0], cmp_endids); assert(ids[0] == 1 && ids[1] == 2); info[ninfo].ids[0] = ids[0]; info[ninfo].ids[1] = ids[1]; @@ -139,22 +132,12 @@ int main(void) assert( state_ind == info[info_ind].state ); count = fsm_endid_count(fsm, state_ind); - assert(count > 0 && count <= 2); + assert(count == info[info_ind].count); - assert( count == info[info_ind].count ); - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids / sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); - if (count > 1) { - qsort(&ids[0], count, sizeof ids[0], cmp_endids); - } - for (ind=0; ind < count; ind++) { fsm_end_id_t expected = info[info_ind].ids[ind]; switch (info[info_ind].ids[ind]) { diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index 6508308eb..3021be830 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -108,15 +108,9 @@ int main(void) int ret; count = fsm_endid_count(fsm, state_ind); - assert( count > 0 && count <= 2); - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids / sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); info[ninfo].state = state_ind; @@ -126,7 +120,6 @@ int main(void) assert(ids[0] == 1 || ids[0] == 2); info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&ids[0], count, sizeof ids[0], cmp_endids); assert(ids[0] == 1 && ids[1] == 2); info[ninfo].ids[0] = ids[0]; info[ninfo].ids[1] = ids[1]; @@ -152,14 +145,9 @@ int main(void) assert( state_ind == info[info_ind].state ); count = fsm_endid_count(fsm, state_ind); - assert(count == 1); - ret = fsm_endid_get( - fsm, - state_ind, - 1, - &id); + ret = fsm_endid_get(fsm, state_ind, 1, &id); assert(ret == 1); if (info[info_ind].count == 2) { diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index 6b01d5f15..939c4b306 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -112,15 +112,9 @@ int main(void) int ret; count = fsm_endid_count(fsm, state_ind); - assert( count > 0 && count <= 2); - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids / sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); info[ninfo].state = state_ind; @@ -130,7 +124,6 @@ int main(void) assert(ids[0] == 1 || ids[0] == 2); info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&ids[0], count, sizeof ids[0], cmp_endids); assert(ids[0] == 1 && ids[1] == 2); info[ninfo].ids[0] = ids[0]; info[ninfo].ids[1] = ids[1]; @@ -156,14 +149,9 @@ int main(void) assert( state_ind == info[info_ind].state ); count = fsm_endid_count(fsm, state_ind); - assert(count == 1); - ret = fsm_endid_get( - fsm, - state_ind, - 1, - &endid); + ret = fsm_endid_get( fsm, state_ind, 1, &endid); assert(ret == 1); if (info[info_ind].count == 2) { diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index 86a2b28c8..12b36fae0 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -77,20 +77,12 @@ int main(void) int ret; count = fsm_endid_count(comb, state_ind); - printf("state %u count = %zu\n", state_ind, count); - assert(count == 3); - ret = fsm_endid_get( - comb, - state_ind, - sizeof ids / sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(comb, state_ind, count, &ids[0]); assert(ret == 1); - qsort(&ids[0], count, sizeof ids[0], cmp_endids); assert(ids[0] == 1 && ids[1] == 2 && ids[2] == 4); } } diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index 33a872155..519399973 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -76,15 +76,9 @@ int main(void) int ret; count = fsm_endid_count(fsm, state_ind); - assert( count > 0 && count <= 2); - ret = fsm_endid_get( - fsm, - state_ind, - sizeof ids / sizeof ids[0], - &ids[0]); - + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); assert(ret == 1); info[ninfo].state = state_ind; @@ -94,7 +88,6 @@ int main(void) assert(ids[0] == 11 || ids[0] == 12); info[ninfo].ids[0] = ids[0]; } else if (count == 2) { - qsort(&ids[0], count, sizeof ids[0], cmp_endids); assert(ids[0] == 11 && ids[1] == 12); info[ninfo].ids[0] = ids[0]; info[ninfo].ids[1] = ids[1]; diff --git a/tests/endids/endids_utils.h b/tests/endids/endids_utils.h index f1aaefb5d..e2aad628a 100644 --- a/tests/endids/endids_utils.h +++ b/tests/endids/endids_utils.h @@ -7,8 +7,5 @@ int match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **endids_ptr, size_t *num_endids_ptr); -int -cmp_endids(const void *pa, const void *pb); - #endif /* ENDIDS_UTILS_H */ diff --git a/tests/endids/utils.c b/tests/endids/utils.c index 882df92c4..28d65a1dd 100644 --- a/tests/endids/utils.c +++ b/tests/endids/utils.c @@ -47,14 +47,3 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end return ret; } -int -cmp_endids(const void *pa, const void *pb) { - const fsm_end_id_t *a = pa; - const fsm_end_id_t *b = pb; - - if (*a < *b) { return -1; } - if (*a > *b) { return 1; } - return 0; -} - - diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index 124c8be1b..ae4e94eed 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -60,7 +60,7 @@ gtest_matches_cb(const struct fsm *fsm, #define ID_BUF_COUNT 1 fsm_end_id_t ids[ID_BUF_COUNT]; int gres = fsm_endid_get(fsm, - end_state, ID_BUF_COUNT, ids); + end_state, fsm_endid_count(fsm, end_state), ids); if (gres != 1) { fprintf(stderr, diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 50cc21be0..66afa1a38 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -47,11 +47,13 @@ run_test(const char **strings) const int res = fsm_exec(fsm, fsm_sgetc, string, &end, NULL); assert(res > 0); /* match */ - int eres = fsm_endid_get(fsm, end, - MAX_INPUTS, ids); + size_t count = fsm_endid_count(fsm, end); + assert(count <= MAX_INPUTS); + + int eres = fsm_endid_get(fsm, end, count, ids); assert(eres == 1); bool found = false; - for (size_t i = 0; i < fsm_endid_count(fsm, end); i++) { + for (size_t i = 0; i < count; i++) { if (ids[i] == id) { found = true; break; From 093b703d1b20dec63b430d7034dd42fa64c65dc4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 27 Jun 2024 10:19:27 +0100 Subject: [PATCH 185/373] First cut at introducing llvm ir codegen. --- .github/workflows/ci.yml | 2 +- fuzz/target.c | 1 + include/fsm/print.h | 2 + include/print/esc.h | 4 + src/fsm/main.c | 1 + src/libfsm/libfsm.syms | 1 + src/libfsm/print/Makefile | 1 + src/libfsm/print/llvm.c | 527 ++++++++++++++++++++++++++++++++++++++ src/print/Makefile | 1 + src/print/llvm.c | 44 ++++ src/re/main.c | 44 ++++ src/retest/main.c | 3 + src/retest/reperf.c | 3 + src/retest/runner.c | 31 ++- src/retest/runner.h | 6 + 15 files changed, 669 insertions(+), 2 deletions(-) create mode 100644 src/libfsm/print/llvm.c create mode 100644 src/print/llvm.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27fdf78f7..ce24dbcf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -481,7 +481,7 @@ jobs: cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op - lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm ] + lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm, llvm ] exclude: - os: macos cc: gcc # it's clang anyway diff --git a/fuzz/target.c b/fuzz/target.c index 87086b929..1a9690b48 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -329,6 +329,7 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const r |= fsm_print_sh(f, fsm); r |= fsm_print_go(f, fsm); r |= fsm_print_rust(f, fsm); + r |= fsm_print_llvm(f, fsm); assert(r == 0 || errno != 0); fsm_free(fsm); diff --git a/include/fsm/print.h b/include/fsm/print.h index 790e620bc..e85b6102c 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -26,6 +26,7 @@ struct fsm; * fsm_print_vmc - ISO C90 code, VM style * fsm_print_vmdot - Graphviz Dot format, showing VM opcodes * fsm_print_rust - Rust code + * fsm_print_llvm - Live Laugh Virtual Machine * fsm_print_sh - Shell script (bash dialect) * fsm_print_go - Go code * fsm_print_goasm/vmasm_* - Assembly in various dialects @@ -58,6 +59,7 @@ fsm_print fsm_print_vmasm_amd64_go; /* output amd64 assembler in Go format */ fsm_print fsm_print_sh; fsm_print fsm_print_go; fsm_print fsm_print_rust; +fsm_print fsm_print_llvm; fsm_print fsm_print_vmops_c; fsm_print fsm_print_vmops_h; fsm_print fsm_print_vmops_main; diff --git a/include/print/esc.h b/include/print/esc.h index 6cf52d395..7d7dd63fd 100644 --- a/include/print/esc.h +++ b/include/print/esc.h @@ -22,6 +22,7 @@ escputc json_escputc; escputc pcre_escputc; escputc rust_escputc_char; escputc rust_escputc_str; +escputc llvm_escputc_char; int awk_escputcharlit(FILE *f, const struct fsm_options *opt, char c); @@ -29,6 +30,9 @@ awk_escputcharlit(FILE *f, const struct fsm_options *opt, char c); int c_escputcharlit(FILE *f, const struct fsm_options *opt, char c); +void +llvm_escputcharlit(FILE *f, const struct fsm_options *opt, char c); + void rust_escputcharlit(FILE *f, const struct fsm_options *opt, char c); diff --git a/src/fsm/main.c b/src/fsm/main.c index a255ae8ee..b6f011504 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -167,6 +167,7 @@ print_name(const char *name) { "vmc", fsm_print_vmc }, { "vmdot", fsm_print_vmdot }, { "rust", fsm_print_rust }, + { "llvm", fsm_print_llvm }, { "sh", fsm_print_sh }, { "go", fsm_print_go }, diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 2801f3703..b90cc01e7 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -50,6 +50,7 @@ fsm_print_vmasm_amd64_att fsm_print_vmasm_amd64_nasm fsm_print_vmasm_amd64_go fsm_print_rust +fsm_print_llvm fsm_print_sh fsm_print_go fsm_print_vmops_c diff --git a/src/libfsm/print/Makefile b/src/libfsm/print/Makefile index c2911318b..f128a62d9 100644 --- a/src/libfsm/print/Makefile +++ b/src/libfsm/print/Makefile @@ -11,6 +11,7 @@ SRC += src/libfsm/print/irdot.c SRC += src/libfsm/print/irjson.c SRC += src/libfsm/print/json.c SRC += src/libfsm/print/rust.c +SRC += src/libfsm/print/llvm.c SRC += src/libfsm/print/sh.c SRC += src/libfsm/print/go.c SRC += src/libfsm/print/vmc.c diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c new file mode 100644 index 000000000..21dbaaa85 --- /dev/null +++ b/src/libfsm/print/llvm.c @@ -0,0 +1,527 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include "libfsm/internal.h" + +#include "libfsm/vm/vm.h" + +#include "ir.h" + +#define OPAQUE_POINTERS 1 + +#ifdef OPAQUE_POINTERS // llvm >= 15 +static const char *ptr_i8 = "ptr"; +static const char *ptr_i32 = "ptr"; +static const char *ptr_void = "ptr"; +#else +static const char *ptr_i8 = "i8*"; +static const char *ptr_i32 = "i32*"; +static const char *ptr_void = "i8*"; +#endif + +/* + * If we had a stack, the current set of live values would be a frame. + * We're a DFA, so we don't have a stack. But I still think of them as a frame. + */ +struct frame { + unsigned r; + unsigned b; + unsigned c; +}; + +static unsigned +use(const unsigned *n) +{ + assert(*n > 0); + return *n - 1; +} + +static unsigned +decl(unsigned *n) +{ + return (*n)++; +} + +static int +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +{ + assert(f != NULL); + assert(leaf_opaque == NULL); + + (void) ids; + (void) count; + (void) leaf_opaque; + + /* XXX: this should be FSM_UNKNOWN or something non-EOF, + * maybe user defined */ + fprintf(f, "return TOK_UNKNOWN;"); + + return 0; +} + +static const char * +cmp_operator(int cmp) +{ + switch (cmp) { + case VM_CMP_LT: return "ult"; + case VM_CMP_LE: return "ule"; + case VM_CMP_EQ: return "eq"; + case VM_CMP_GE: return "uge"; + case VM_CMP_GT: return "ugt"; + case VM_CMP_NE: return "ne"; + + case VM_CMP_ALWAYS: + default: + assert("unreached"); + return NULL; + } +} + +static void +print_decl(FILE *f, const char *name, unsigned n) +{ + fprintf(f, "\t%%%s%u = ", name, n); +} + +static void +print_ret(FILE *f, long l) +{ + fprintf(f, "\tret i32 %ld\n", l); +} + +// TODO: variadic +static void +print_label(FILE *f, const struct dfavm_op_ir *op, bool decl) +{ + if (!decl) { + fprintf(f, "%%"); + } + + fprintf(f, "l%" PRIu32, op->index); + + if (decl) { + fprintf(f, ":\n"); + } +} + +static void +print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, + struct frame *frame) +{ + assert(frame != NULL); + assert(op->cmp != VM_CMP_ALWAYS); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp %s i8 %%c%u, ", + cmp_operator(op->cmp), use(&frame->c)); + llvm_escputcharlit(f, opt, op->cmp_arg); + fprintf(f, " ; "); + c_escputcharlit(f, opt, op->cmp_arg); // C escaping for a comment + fprintf(f, "\n"); +} + +static int +print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, + const struct ir *ir, enum dfavm_op_end end_bits) +{ + if (end_bits == VM_END_FAIL) { + print_ret(f, -1); + } else { + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { + return -1; + } + } else { + print_ret(f, op->ir_state - ir->states); + } + } + + return 0; +} + +static void +print_jump(FILE *f, const struct dfavm_op_ir *dest) +{ + fprintf(f, "\tbr label "); + print_label(f, dest, false); + fprintf(f, "\n"); +} + +static void +print_fetch(FILE *f, const struct fsm_options *opt, + struct frame *frame, const struct dfavm_op_ir *next) +{ + unsigned n = decl(&frame->c); + unsigned b = decl(&frame->b); + + assert(frame != NULL); + assert(next != NULL); + + /* + * Emitting phi here would mean we'd need to keep track of the + * basic blocks that jump to the current block. That's entirely + * possible with our IR, but seems like a lot of work when we + * can just alloca and let llvm do it for us. + */ + + // TODO: consider emitting fetch(%s, %n) + + switch (opt->io) { + case FSM_IO_GETC: { + print_decl(f, "i", n); + fprintf(f, "tail call i32 %%fsm_getc(%s noundef %%getc_opaque) #1\n", + ptr_i8); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp eq i32 %%i%u, -1 ; EOF\n", n); + + fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%f%u\n", + use(&frame->r), b, b); + fprintf(f, "f%u:\n", b); + + print_decl(f, "c", n); + fprintf(f, "trunc i32 %%i%u to i8\n", n); + + print_jump(f, next); + break; + } + + case FSM_IO_STR: { + /* + * If we increment %s instead of maintaining %n, we'd need to + * do it post-EOS check. Otherwise we'd skip the first character + * (or start with the pointer out of bounds). + */ + print_decl(f, "n", n); + fprintf(f, "load i32, %s %%n\n", ptr_i32); + + print_decl(f, "n.new", n); + fprintf(f, "add i32 1, %%n%u\n", n); + + fprintf(f, "\tstore i32 %%n.new%u, %s %%n\n", n, ptr_i32); + + print_decl(f, "p", n); + fprintf(f, "getelementptr inbounds i8, %s %%s, i32 %%n%u\n", + ptr_i8, n); + + print_decl(f, "c", n); + fprintf(f, "load i8, %s %%p%u\n", + ptr_i8, n); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp eq i8 %%c%u, 0 ; EOT\n", + n); + + // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: + fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%l%u\n", + use(&frame->r), b, next->index); + break; + } + + case FSM_IO_PAIR: { + print_decl(f, "n", n); + fprintf(f, "load i32, %s %%n\n", ptr_i32); + + print_decl(f, "n.new", n); + fprintf(f, "add i32 1, %%n%u\n", n); + + fprintf(f, "\tstore i32 %%n.new%u, %s %%n\n", n, ptr_i32); + + print_decl(f, "p", n); + fprintf(f, "getelementptr inbounds i8, %s %%b, i32 %%n%u\n", + ptr_i8, n); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp eq %s %%p%u, %%e ; EOF\n", + ptr_i8, n); + + fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%f%u\n", + use(&frame->r), b, b); + fprintf(f, "f%u:\n", b); + + print_decl(f, "c", n); + fprintf(f, "load i8, %s %%p%u\n", + ptr_i8, n); + + print_jump(f, next); + break; + } + + default: + fprintf(stderr, "unsupported IO API\n"); + exit(EXIT_FAILURE); + } + + // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: + fprintf(f, "t%u:\n", use(&frame->b)); +} + +/* TODO: eventually to be non-static */ +static int +fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, + const struct fsm_options *opt, const struct ir *ir, + const char *cp, + int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), + const void *leaf_opaque) +{ + struct dfavm_op_ir *op; + + assert(f != NULL); + assert(a != NULL); + assert(opt != NULL); + assert(cp != NULL); + + /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. + * We'll need that in order to be able to use the leaf callback here. */ + (void) leaf; + (void) leaf_opaque; + + /* TODO: we'll need to heed cp for e.g. lx's codegen */ + (void) cp; + + { + uint32_t l; + + l = 0; + + for (op = a->linked; op != NULL; op = op->next) { + op->index = l++; + } + } + + print_jump(f, a->linked); + + struct frame frame = { 0, 0, 0 }; + for (op = a->linked; op != NULL; op = op->next) { + print_label(f, op, true); + + if (op->ir_state != NULL && op->ir_state->example != NULL) { + /* C's escaping seems to be a subset of llvm's, and these are + * for comments anyway. So I'm borrowing this for C here */ + fprintf(f, "\t; e.g. \""); + escputs(f, opt, c_escputc_str, op->ir_state->example); + fprintf(f, "\""); + + fprintf(f, "\n"); + } + + switch (op->instr) { + case VM_OP_STOP: + if (op->cmp != VM_CMP_ALWAYS) { + unsigned b = decl(&frame.b); + unsigned next = op->next->index; + + print_cond(f, op, opt, &frame); + +// TODO: fold into print_cond() + fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%l%u\n", + use(&frame.r), b, next); + fprintf(f, "t%u:\n", b); + } + + if (-1 == print_end(f, op, opt, ir, op->u.stop.end_bits)) { + return -1; + } + break; + + case VM_OP_FETCH: { + print_fetch(f, opt, &frame, op->next); + + if (-1 == print_end(f, op, opt, ir, op->u.fetch.end_bits)) { + return -1; + } + break; + } + + case VM_OP_BRANCH: { + const struct dfavm_op_ir *dest = op->u.br.dest_arg; + + if (op->cmp == VM_CMP_ALWAYS) { + print_jump(f, dest); + } else { + unsigned next = op->next->index; + + print_cond(f, op, opt, &frame); + + fprintf(f, "\tbr i1 %%r%u, label ", use(&frame.r)); + print_label(f, dest, false); + fprintf(f, ", label %%l%u\n", next); + } + break; + } + + default: + assert(!"unreached"); + break; + } + } + +// TODO: collate ret values together at the end, keeps them out of the i-cache maybe +// or better yet, have one ret: label and emit a phi + + return 0; +} + +static int +fsm_print_llvm_complete(FILE *f, const struct ir *ir, + const struct fsm_options *opt, const char *prefix, const char *cp) +{ + static const struct dfavm_assembler_ir zero; + struct dfavm_assembler_ir a; + + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; + + assert(f != NULL); + assert(ir != NULL); + assert(opt != NULL); + + a = zero; + + if (!dfavm_compile_ir(&a, ir, vm_opts)) { + return -1; + } + + if (opt->fragment) { + fsm_print_llvmfrag(f, &a, opt, ir, cp, + opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + goto error; + } + + fprintf(f, "; generated\n"); + fprintf(f, "define dso_local i32 @%smain", prefix); + + switch (opt->io) { + case FSM_IO_GETC: +#ifdef OPAQUE_POINTERS + fprintf(f, "(i32 (%s)* nocapture noundef readonly %%fsm_getc, %s noundef %%getc_opaque)", + ptr_i8, ptr_void); +#else + fprintf(f, "(ptr nocapture noundef readonly %%fsm_getc, ptr noundef %%getc_opaque)"); +#endif + break; + + case FSM_IO_STR: + fprintf(f, "(%s nocapture noundef readonly %%s)", + ptr_i8); + break; + + case FSM_IO_PAIR: + fprintf(f, "(%s noundef readonly %%b, %s noundef readnone %%e)", + ptr_i8, ptr_i8); + break; + + default: + fprintf(f, "(%s nocapture noundef readonly %%b, %s nocapture noundef readonly %%e)", + ptr_i8, ptr_i8); + break; + } + + fprintf(f, " local_unnamed_addr"); + fprintf(f, " hot nosync nounwind norecurse willreturn"); + fprintf(f, "#0 {\n"); + + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + fprintf(f, "\t%%n = alloca i32\n"); + fprintf(f, "\tstore i32 -1, %s %%n\n", ptr_i32); + break; + + case FSM_IO_PAIR: + fprintf(f, "\t%%n = alloca i32\n"); + fprintf(f, "\tstore i32 -1, %s %%n\n", ptr_i32); + break; + + default: + fprintf(stderr, "unsupported IO API\n"); + exit(EXIT_FAILURE); + } + + fsm_print_llvmfrag(f, &a, opt, ir, cp, + opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + + fprintf(f, "}\n"); + fprintf(f, "\n"); + + dfavm_opasm_finalize_op(&a); + + if (ferror(f)) { + return -1; + } + + return 0; + +error: + + dfavm_opasm_finalize_op(&a); + + return -1; +} + +int +fsm_print_llvm(FILE *f, const struct fsm *fsm) +{ + struct ir *ir; + const char *prefix; + const char *cp; + int r; + + assert(f != NULL); + assert(fsm != NULL); + assert(fsm->opt != NULL); + + ir = make_ir(fsm); + if (ir == NULL) { + return -1; + } + + if (fsm->opt->prefix != NULL) { + prefix = fsm->opt->prefix; + } else { + prefix = "fsm_"; + } + + if (fsm->opt->cp != NULL) { + cp = fsm->opt->cp; + } else { + cp = "c"; /* XXX */ + } + + r = fsm_print_llvm_complete(f, ir, fsm->opt, prefix, cp); + + free_ir(fsm, ir); + + return r; +} + diff --git a/src/print/Makefile b/src/print/Makefile index 898028f5c..77b4446a0 100644 --- a/src/print/Makefile +++ b/src/print/Makefile @@ -6,6 +6,7 @@ SRC += src/print/dot.c SRC += src/print/abnf.c SRC += src/print/fsm.c SRC += src/print/json.c +SRC += src/print/llvm.c SRC += src/print/pcre.c SRC += src/print/rust.c SRC += src/print/tok.c diff --git a/src/print/llvm.c b/src/print/llvm.c new file mode 100644 index 000000000..ab0c69720 --- /dev/null +++ b/src/print/llvm.c @@ -0,0 +1,44 @@ +/* + * Copyright 2008-2017 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include + +int +llvm_escputc_char(FILE *f, const struct fsm_options *opt, char c) +{ + assert(f != NULL); + assert(opt != NULL); + + if (opt->always_hex) { + return fprintf(f, "u%#02x", (unsigned char) c); + } + + return fprintf(f, "%u", (unsigned char) c); +} + +void +llvm_escputcharlit(FILE *f, const struct fsm_options *opt, char c) +{ + assert(f != NULL); + assert(opt != NULL); + + if (opt->always_hex || (unsigned char) c > SCHAR_MAX) { + fprintf(f, "u%#02x", (unsigned char) c); + return; + } + + llvm_escputc_char(f, opt, c); +} + diff --git a/src/re/main.c b/src/re/main.c index ea2c68ae8..a7c8bc803 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -124,6 +124,7 @@ print_name(const char *name, { "vmc", fsm_print_vmc, NULL }, { "vmdot", fsm_print_vmdot, NULL }, { "rust", fsm_print_rust, NULL }, + { "llvm", fsm_print_llvm, NULL }, { "sh", fsm_print_sh, NULL }, { "go", fsm_print_go, NULL }, @@ -493,6 +494,47 @@ endleaf_rust(FILE *f, return 0; } +static int +endleaf_llvm(FILE *f, + const fsm_end_id_t *ids, size_t count, + const void *endleaf_opaque) +{ + const struct match *m; + unsigned n; + size_t i; + + assert(endleaf_opaque == NULL); + + (void) f; + (void) endleaf_opaque; + + n = 0; + + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + n |= 1U << m->id; + } + } + + fprintf(f, "\tret i32 u%#x", (unsigned) n); + + fprintf(f, " ; "); + + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + + if (m->next != NULL || i < count - 1) { + fprintf(f, ", "); + } + } + } + + fprintf(f, "\n"); + + return 0; +} + static int endleaf_dot(FILE *f, const fsm_end_id_t *ids, size_t count, @@ -1186,6 +1228,8 @@ main(int argc, char *argv[]) opt.endleaf = endleaf_c; } else if (print_fsm == fsm_print_rust) { opt.endleaf = endleaf_rust; + } else if (print_fsm == fsm_print_llvm) { + opt.endleaf = endleaf_llvm; } else if (print_fsm == fsm_print_dot || print_fsm == fsm_print_vmdot) { opt.endleaf = patterns ? endleaf_dot : NULL; } else if (print_fsm == fsm_print_json) { diff --git a/src/retest/main.c b/src/retest/main.c index 9d7d8d583..a5309e6c3 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -179,6 +179,7 @@ usage(void) fprintf(stderr, " vmc compile as per fsm_print_vmc()\n"); fprintf(stderr, " vmops compile as per fsm_print_vmops_{c,h,main}()\n"); fprintf(stderr, " rust compile as per fsm_print_rust()\n"); + fprintf(stderr, " llvm compile as per fsm_print_llvm()\n"); fprintf(stderr, "\n"); fprintf(stderr, " -x \n"); @@ -1280,6 +1281,8 @@ main(int argc, char *argv[]) impl = IMPL_GO; } else if (strcmp(optarg, "goasm") == 0) { impl = IMPL_GOASM; + } else if (strcmp(optarg, "llvm") == 0) { + impl = IMPL_LLVM; } else if (strcmp(optarg, "rust") == 0) { impl = IMPL_RUST; } else { diff --git a/src/retest/reperf.c b/src/retest/reperf.c index b87f16b3f..d722a93a4 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -1062,6 +1062,7 @@ usage(void) fprintf(stderr, " c compile as per fsm_print_c()\n"); fprintf(stderr, " vmc compile as per fsm_print_vmc()\n"); fprintf(stderr, " rust compile as per fsm_print_rust()\n"); + fprintf(stderr, " llvm compile as per fsm_print_llvm()\n"); fprintf(stderr, "\n"); fprintf(stderr, " -x \n"); @@ -1155,6 +1156,8 @@ main(int argc, char *argv[]) impl = IMPL_INTERPRET; } else if (strcmp(optarg, "c") == 0) { impl = IMPL_C; + } else if (strcmp(optarg, "llvm") == 0) { + impl = IMPL_LLVM; } else if (strcmp(optarg, "rust") == 0) { impl = IMPL_RUST; } else if (strcmp(optarg, "asm") == 0) { diff --git a/src/retest/runner.c b/src/retest/runner.c index 7d3ddad4a..517d2667d 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -86,6 +86,7 @@ print(const struct fsm *fsm, enum implementation impl, switch (impl) { case IMPL_C: e = fsm_print_c(f, fsm); break; case IMPL_RUST: e = fsm_print_rust(f, fsm); break; + case IMPL_LLVM: e = fsm_print_llvm(f, fsm); break; case IMPL_VMC: e = fsm_print_vmc(f, fsm); break; case IMPL_GOASM: e = fsm_print_vmasm_amd64_go(f, fsm); break; case IMPL_VMASM: e = fsm_print_vmasm_amd64_att(f, fsm); break; @@ -145,7 +146,7 @@ compile(enum implementation impl, case IMPL_C: case IMPL_VMC: case IMPL_VMOPS: - if (0 != systemf("%s %s -xc -shared -fPIC %s -o %s", + if (0 != systemf("%s %s -shared -fPIC %s -o %s", cc ? cc : "gcc", cflags ? cflags : "-std=c89 -pedantic -Wall -Werror -O3", tmp_src, tmp_so)) { @@ -164,6 +165,16 @@ compile(enum implementation impl, break; + case IMPL_LLVM: + if (0 != systemf("clang %s -shared -fPIC %s -o %s", + cflags ? cflags : "-pedantic -Wall -Werror -Wno-override-module -O3", + tmp_src, tmp_so)) + { + return 0; + } + + break; + case IMPL_GO: case IMPL_GOASM: { char tmp_o1[] = "/tmp/fsmcompile_o1-XXXXXX.o"; @@ -287,6 +298,7 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation char tmp_src_go[] = "/tmp/fsmcompile_src-XXXXXX.go"; char tmp_src_c[] = "/tmp/fsmcompile_src-XXXXXX.c"; char tmp_src_rs[] = "/tmp/fsmcompile_src-XXXXXX.rs"; + char tmp_src_ll[] = "/tmp/fsmcompile_src-XXXXXX.ll"; char tmp_src_s[] = "/tmp/fsmcompile_src-XXXXXX.s"; char *tmp_src; @@ -295,6 +307,7 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation case IMPL_C: case IMPL_VMC: tmp_src = tmp_src_c; break; case IMPL_RUST: tmp_src = tmp_src_rs; break; + case IMPL_LLVM: tmp_src = tmp_src_ll; break; case IMPL_GOASM: case IMPL_VMASM: tmp_src = tmp_src_s; break; case IMPL_GO: tmp_src = tmp_src_go; break; @@ -359,6 +372,11 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation r->u.impl_rust.func = (int64_t (*)(const unsigned char *, size_t)) (uintptr_t) dlsym(h, "retest_trampoline"); break; + case IMPL_LLVM: + r->u.impl_llvm.h = h; + r->u.impl_llvm.func = (int (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm_main"); + break; + case IMPL_GO: case IMPL_GOASM: r->u.impl_go.h = h; @@ -394,6 +412,7 @@ fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation switch (impl) { case IMPL_C: + case IMPL_LLVM: case IMPL_RUST: case IMPL_VMASM: case IMPL_VMC: @@ -436,6 +455,12 @@ fsm_runner_finalize(struct fsm_runner *r) } break; + case IMPL_LLVM: + if (r->u.impl_llvm.h != NULL) { + dlclose(r->u.impl_llvm.h); + } + break; + case IMPL_GO: case IMPL_GOASM: if (r->u.impl_go.h != NULL) { @@ -477,6 +502,10 @@ fsm_runner_run(const struct fsm_runner *r, const char *s, size_t n) assert(r->u.impl_rust.func != NULL); return r->u.impl_rust.func((const unsigned char *)s, n) >= 0; + case IMPL_LLVM: + assert(r->u.impl_llvm.func != NULL); + return r->u.impl_llvm.func(s, s+n) >= 0; + case IMPL_GO: case IMPL_GOASM: assert(r->u.impl_go.func != NULL); diff --git a/src/retest/runner.h b/src/retest/runner.h index 1df4d546f..abe88435f 100644 --- a/src/retest/runner.h +++ b/src/retest/runner.h @@ -27,6 +27,7 @@ enum error_type { enum implementation { IMPL_C, IMPL_RUST, + IMPL_LLVM, IMPL_GO, IMPL_GOASM, IMPL_VMC, @@ -50,6 +51,11 @@ struct fsm_runner { int64_t (*func)(const unsigned char *, size_t); } impl_rust; + struct { + void *h; + int (*func)(const char *, const char *); + } impl_llvm; + struct { void *h; int64_t (*func)(const unsigned char *, int64_t); From 2371f42df1daf9495d8d2d4c39bfb024cd09438c Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 28 Jun 2024 14:42:00 +0100 Subject: [PATCH 186/373] -opaque-pointers for clang 14 in github CI. --- src/retest/runner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/runner.c b/src/retest/runner.c index 517d2667d..1cdf0cf81 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -166,7 +166,7 @@ compile(enum implementation impl, break; case IMPL_LLVM: - if (0 != systemf("clang %s -shared -fPIC %s -o %s", + if (0 != systemf("clang %s -shared -fPIC -opaque-pointers %s -o %s", cflags ? cflags : "-pedantic -Wall -Werror -Wno-override-module -O3", tmp_src, tmp_so)) { From 09e1299dc1b57583ee623b4d59c68e48488b2c83 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 29 Jun 2024 17:29:26 +0100 Subject: [PATCH 187/373] Maintain n index from 0, not -1. This produces a little more code (there's an additional f: branch), but also it works. So who's judging? --- src/libfsm/print/llvm.c | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 21dbaaa85..bcfa121b1 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -172,6 +172,16 @@ print_jump(FILE *f, const struct dfavm_op_ir *dest) fprintf(f, "\n"); } +static void +print_incr(FILE *f, const char *name, unsigned n) +{ + // TODO: emit incr()? + print_decl(f, "n.new", n); + fprintf(f, "add i32 1, %%%s%u\n", name, n); + + fprintf(f, "\tstore i32 %%%s.new%u, %s %%%s\n", name, n, ptr_i32, name); +} + static void print_fetch(FILE *f, const struct fsm_options *opt, struct frame *frame, const struct dfavm_op_ir *next) @@ -212,19 +222,10 @@ print_fetch(FILE *f, const struct fsm_options *opt, } case FSM_IO_STR: { - /* - * If we increment %s instead of maintaining %n, we'd need to - * do it post-EOS check. Otherwise we'd skip the first character - * (or start with the pointer out of bounds). - */ + // TODO: we could increment %s instead of maintaining %n here print_decl(f, "n", n); fprintf(f, "load i32, %s %%n\n", ptr_i32); - print_decl(f, "n.new", n); - fprintf(f, "add i32 1, %%n%u\n", n); - - fprintf(f, "\tstore i32 %%n.new%u, %s %%n\n", n, ptr_i32); - print_decl(f, "p", n); fprintf(f, "getelementptr inbounds i8, %s %%s, i32 %%n%u\n", ptr_i8, n); @@ -238,8 +239,13 @@ print_fetch(FILE *f, const struct fsm_options *opt, n); // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: - fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%l%u\n", - use(&frame->r), b, next->index); + fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%f%u\n", + use(&frame->r), b, b); + fprintf(f, "f%u:\n", b); + + print_incr(f, "n", n); + + print_jump(f, next); break; } @@ -247,11 +253,6 @@ print_fetch(FILE *f, const struct fsm_options *opt, print_decl(f, "n", n); fprintf(f, "load i32, %s %%n\n", ptr_i32); - print_decl(f, "n.new", n); - fprintf(f, "add i32 1, %%n%u\n", n); - - fprintf(f, "\tstore i32 %%n.new%u, %s %%n\n", n, ptr_i32); - print_decl(f, "p", n); fprintf(f, "getelementptr inbounds i8, %s %%b, i32 %%n%u\n", ptr_i8, n); @@ -268,6 +269,8 @@ print_fetch(FILE *f, const struct fsm_options *opt, fprintf(f, "load i8, %s %%p%u\n", ptr_i8, n); + print_incr(f, "n", n); + print_jump(f, next); break; } @@ -455,12 +458,12 @@ fsm_print_llvm_complete(FILE *f, const struct ir *ir, case FSM_IO_STR: fprintf(f, "\t%%n = alloca i32\n"); - fprintf(f, "\tstore i32 -1, %s %%n\n", ptr_i32); + fprintf(f, "\tstore i32 0, %s %%n\n", ptr_i32); break; case FSM_IO_PAIR: fprintf(f, "\t%%n = alloca i32\n"); - fprintf(f, "\tstore i32 -1, %s %%n\n", ptr_i32); + fprintf(f, "\tstore i32 0, %s %%n\n", ptr_i32); break; default: From d49de862b3f7fcd2f56b751b9a96840930b28da8 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 29 Jun 2024 17:54:47 +0100 Subject: [PATCH 188/373] We need -mllvm for opaque pointers as well as -opaque-pointers. --- src/retest/runner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/runner.c b/src/retest/runner.c index 1cdf0cf81..b45b6b7e0 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -166,7 +166,7 @@ compile(enum implementation impl, break; case IMPL_LLVM: - if (0 != systemf("clang %s -shared -fPIC -opaque-pointers %s -o %s", + if (0 != systemf("clang %s -shared -fPIC -mllvm -opaque-pointers %s -o %s", cflags ? cflags : "-pedantic -Wall -Werror -Wno-override-module -O3", tmp_src, tmp_so)) { From f6e7ef60708c3b00d54ab34e688782745fe9babb Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 29 Jun 2024 17:55:06 +0100 Subject: [PATCH 189/373] Run these retest tests for llvm too. --- tests/retest/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/retest/Makefile b/tests/retest/Makefile index 5c3512ccd..94fa19b69 100644 --- a/tests/retest/Makefile +++ b/tests/retest/Makefile @@ -8,7 +8,7 @@ DIR += ${TEST_OUTDIR.tests/retest} RETEST=${BUILD}/bin/retest -.for lang in vm asm c vmc +.for lang in vm asm c vmc llvm .for io in pair str # XXX: we don't have FSM_IO_STR for asm yet From 2a6445d738a7567ac0f365629da4eee391a6e08c Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 29 Jun 2024 23:57:52 +0100 Subject: [PATCH 190/373] Consistency for print_label() --- src/libfsm/print/llvm.c | 58 +++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index bcfa121b1..1a95de5e1 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -111,15 +112,18 @@ print_ret(FILE *f, long l) fprintf(f, "\tret i32 %ld\n", l); } -// TODO: variadic static void -print_label(FILE *f, const struct dfavm_op_ir *op, bool decl) +print_label(FILE *f, bool decl, const char *fmt, ...) { + va_list ap; + if (!decl) { - fprintf(f, "%%"); + fprintf(f, "label %%"); } - fprintf(f, "l%" PRIu32, op->index); + va_start(ap, fmt); + vfprintf(f, fmt, ap); + va_end(ap); if (decl) { fprintf(f, ":\n"); @@ -167,8 +171,8 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, static void print_jump(FILE *f, const struct dfavm_op_ir *dest) { - fprintf(f, "\tbr label "); - print_label(f, dest, false); + fprintf(f, "\tbr "); + print_label(f, false, "l%" PRIu32, dest->index); fprintf(f, "\n"); } @@ -210,8 +214,11 @@ print_fetch(FILE *f, const struct fsm_options *opt, print_decl(f, "r", decl(&frame->r)); fprintf(f, "icmp eq i32 %%i%u, -1 ; EOF\n", n); - fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%f%u\n", - use(&frame->r), b, b); + fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); + print_label(f, false, "t%u", b); + fprintf(f, ", "); + print_label(f, false, "f%u", b); + fprintf(f, "\n"); fprintf(f, "f%u:\n", b); print_decl(f, "c", n); @@ -239,8 +246,11 @@ print_fetch(FILE *f, const struct fsm_options *opt, n); // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: - fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%f%u\n", - use(&frame->r), b, b); + fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); + print_label(f, false, "t%u", b); + fprintf(f, ", "); + print_label(f, false, "f%u", b); + fprintf(f, "\n"); fprintf(f, "f%u:\n", b); print_incr(f, "n", n); @@ -261,8 +271,11 @@ print_fetch(FILE *f, const struct fsm_options *opt, fprintf(f, "icmp eq %s %%p%u, %%e ; EOF\n", ptr_i8, n); - fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%f%u\n", - use(&frame->r), b, b); + fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); + print_label(f, false, "t%u", b); + fprintf(f, ", "); + print_label(f, false, "f%u", b); + fprintf(f, "\n"); fprintf(f, "f%u:\n", b); print_decl(f, "c", n); @@ -321,7 +334,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, struct frame frame = { 0, 0, 0 }; for (op = a->linked; op != NULL; op = op->next) { - print_label(f, op, true); + print_label(f, true, "l%" PRIu32, op->index); if (op->ir_state != NULL && op->ir_state->example != NULL) { /* C's escaping seems to be a subset of llvm's, and these are @@ -337,13 +350,14 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, case VM_OP_STOP: if (op->cmp != VM_CMP_ALWAYS) { unsigned b = decl(&frame.b); - unsigned next = op->next->index; print_cond(f, op, opt, &frame); -// TODO: fold into print_cond() - fprintf(f, "\tbr i1 %%r%u, label %%t%u, label %%l%u\n", - use(&frame.r), b, next); + fprintf(f, "\tbr i1 %%r%u, ", use(&frame.r)); + print_label(f, false, "t%u", b); + fprintf(f, ", "); + print_label(f, false, "l%" PRIu32, op->next->index); + fprintf(f, "\n"); fprintf(f, "t%u:\n", b); } @@ -367,13 +381,13 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->cmp == VM_CMP_ALWAYS) { print_jump(f, dest); } else { - unsigned next = op->next->index; - print_cond(f, op, opt, &frame); - fprintf(f, "\tbr i1 %%r%u, label ", use(&frame.r)); - print_label(f, dest, false); - fprintf(f, ", label %%l%u\n", next); + fprintf(f, "\tbr i1 %%r%u, ", use(&frame.r)); + print_label(f, false, "l%" PRIu32, dest->index); + fprintf(f, ", "); + print_label(f, false, "l%" PRIu32, op->next->index); + fprintf(f, "\n"); } break; } From 13f93ddf723105db355e42f2d7361231a72e6e6b Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 30 Jun 2024 13:20:15 +0100 Subject: [PATCH 191/373] Whitespace. --- src/libfsm/print/llvm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 1a95de5e1..7d4a9f98c 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -464,7 +464,7 @@ fsm_print_llvm_complete(FILE *f, const struct ir *ir, fprintf(f, " local_unnamed_addr"); fprintf(f, " hot nosync nounwind norecurse willreturn"); - fprintf(f, "#0 {\n"); + fprintf(f, " #0 {\n"); switch (opt->io) { case FSM_IO_GETC: From 73e40744237c5746cafd943658b6787779839abc Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 30 Jun 2024 00:19:07 +0100 Subject: [PATCH 192/373] Factor out print_branch() --- src/libfsm/print/llvm.c | 59 +++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 7d4a9f98c..0788a1a78 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -186,6 +186,25 @@ print_incr(FILE *f, const char *name, unsigned n) fprintf(f, "\tstore i32 %%%s.new%u, %s %%%s\n", name, n, ptr_i32, name); } +static void +print_branch(FILE *f, const struct frame *frame, + const struct dfavm_op_ir *op_true, const struct dfavm_op_ir *op_false) +{ + fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); + if (op_true != NULL) { + print_label(f, false, "l%" PRIu32, op_true->index); + } else { + print_label(f, false, "t%u", use(&frame->b)); + } + fprintf(f, ", "); + if (op_false != NULL) { + print_label(f, false, "l%" PRIu32, op_false->index); + } else { + print_label(f, false, "f%u", use(&frame->b)); + } + fprintf(f, "\n"); +} + static void print_fetch(FILE *f, const struct fsm_options *opt, struct frame *frame, const struct dfavm_op_ir *next) @@ -214,12 +233,8 @@ print_fetch(FILE *f, const struct fsm_options *opt, print_decl(f, "r", decl(&frame->r)); fprintf(f, "icmp eq i32 %%i%u, -1 ; EOF\n", n); - fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); - print_label(f, false, "t%u", b); - fprintf(f, ", "); - print_label(f, false, "f%u", b); - fprintf(f, "\n"); - fprintf(f, "f%u:\n", b); + print_branch(f, frame, NULL, NULL); + print_label(f, true, "f%u", b); print_decl(f, "c", n); fprintf(f, "trunc i32 %%i%u to i8\n", n); @@ -246,12 +261,8 @@ print_fetch(FILE *f, const struct fsm_options *opt, n); // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: - fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); - print_label(f, false, "t%u", b); - fprintf(f, ", "); - print_label(f, false, "f%u", b); - fprintf(f, "\n"); - fprintf(f, "f%u:\n", b); + print_branch(f, frame, NULL, NULL); + print_label(f, true, "f%u", b); print_incr(f, "n", n); @@ -271,12 +282,8 @@ print_fetch(FILE *f, const struct fsm_options *opt, fprintf(f, "icmp eq %s %%p%u, %%e ; EOF\n", ptr_i8, n); - fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); - print_label(f, false, "t%u", b); - fprintf(f, ", "); - print_label(f, false, "f%u", b); - fprintf(f, "\n"); - fprintf(f, "f%u:\n", b); + print_branch(f, frame, NULL, NULL); + print_label(f, true, "f%u", b); print_decl(f, "c", n); fprintf(f, "load i8, %s %%p%u\n", @@ -352,13 +359,8 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, unsigned b = decl(&frame.b); print_cond(f, op, opt, &frame); - - fprintf(f, "\tbr i1 %%r%u, ", use(&frame.r)); - print_label(f, false, "t%u", b); - fprintf(f, ", "); - print_label(f, false, "l%" PRIu32, op->next->index); - fprintf(f, "\n"); - fprintf(f, "t%u:\n", b); + print_branch(f, &frame, NULL, op->next); + print_label(f, true, "t%u", b); } if (-1 == print_end(f, op, opt, ir, op->u.stop.end_bits)) { @@ -382,12 +384,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, print_jump(f, dest); } else { print_cond(f, op, opt, &frame); - - fprintf(f, "\tbr i1 %%r%u, ", use(&frame.r)); - print_label(f, false, "l%" PRIu32, dest->index); - fprintf(f, ", "); - print_label(f, false, "l%" PRIu32, op->next->index); - fprintf(f, "\n"); + print_branch(f, &frame, dest, op->next); } break; } From 2c31f63cc5f29f4b6fc53c36ec7d977ecb37d917 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 30 Jun 2024 13:19:57 +0100 Subject: [PATCH 193/373] Centralise `fail:` nodes. --- src/libfsm/print/llvm.c | 105 +++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 29 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 0788a1a78..9c9e7a84b 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -42,6 +42,8 @@ static const char *ptr_i32 = "i32*"; static const char *ptr_void = "i8*"; #endif +static const struct dfavm_op_ir fail; // used as a unqiue address only + /* * If we had a stack, the current set of live values would be a frame. * We're a DFA, so we don't have a stack. But I still think of them as a frame. @@ -113,23 +115,53 @@ print_ret(FILE *f, long l) } static void -print_label(FILE *f, bool decl, const char *fmt, ...) +vprint_label(FILE *f, bool decl, const char *fmt, va_list ap) { - va_list ap; + assert(fmt != NULL); if (!decl) { fprintf(f, "label %%"); } - va_start(ap, fmt); vfprintf(f, fmt, ap); - va_end(ap); if (decl) { fprintf(f, ":\n"); } } +static void +print_label(FILE *f, bool decl, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vprint_label(f, decl, fmt, ap); + va_end(ap); +} + +static void +print_target(FILE *f, const struct dfavm_op_ir *op, + const char *fmt, ...) +{ + va_list ap; + + if (op == &fail) { + print_label(f, false, "fail"); + } else if (op != NULL && op->instr == VM_OP_STOP && op->cmp == VM_CMP_ALWAYS && op->u.stop.end_bits == VM_END_FAIL) { + print_label(f, false, "fail"); + } else if (op != NULL) { + print_label(f, false, "l%" PRIu32, op->index); + } else if (fmt == NULL) { + assert("!unreached"); + abort(); + } else { + va_start(ap, fmt); + vprint_label(f, false, fmt, ap); + va_end(ap); + } +} + static void print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, struct frame *frame) @@ -172,7 +204,7 @@ static void print_jump(FILE *f, const struct dfavm_op_ir *dest) { fprintf(f, "\tbr "); - print_label(f, false, "l%" PRIu32, dest->index); + print_target(f, dest, NULL); fprintf(f, "\n"); } @@ -191,23 +223,16 @@ print_branch(FILE *f, const struct frame *frame, const struct dfavm_op_ir *op_true, const struct dfavm_op_ir *op_false) { fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); - if (op_true != NULL) { - print_label(f, false, "l%" PRIu32, op_true->index); - } else { - print_label(f, false, "t%u", use(&frame->b)); - } + print_target(f, op_true, "t%u", use(&frame->b)); fprintf(f, ", "); - if (op_false != NULL) { - print_label(f, false, "l%" PRIu32, op_false->index); - } else { - print_label(f, false, "f%u", use(&frame->b)); - } + print_target(f, op_false, "f%u", use(&frame->b)); fprintf(f, "\n"); } static void print_fetch(FILE *f, const struct fsm_options *opt, - struct frame *frame, const struct dfavm_op_ir *next) + struct frame *frame, const struct dfavm_op_ir *next, + enum dfavm_op_end end_bits) { unsigned n = decl(&frame->c); unsigned b = decl(&frame->b); @@ -233,7 +258,9 @@ print_fetch(FILE *f, const struct fsm_options *opt, print_decl(f, "r", decl(&frame->r)); fprintf(f, "icmp eq i32 %%i%u, -1 ; EOF\n", n); - print_branch(f, frame, NULL, NULL); + print_branch(f, frame, + end_bits == VM_END_FAIL ? &fail : NULL, + NULL); print_label(f, true, "f%u", b); print_decl(f, "c", n); @@ -260,8 +287,9 @@ print_fetch(FILE *f, const struct fsm_options *opt, fprintf(f, "icmp eq i8 %%c%u, 0 ; EOT\n", n); - // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: - print_branch(f, frame, NULL, NULL); + print_branch(f, frame, + end_bits == VM_END_FAIL ? &fail : NULL, + NULL); print_label(f, true, "f%u", b); print_incr(f, "n", n); @@ -282,7 +310,9 @@ print_fetch(FILE *f, const struct fsm_options *opt, fprintf(f, "icmp eq %s %%p%u, %%e ; EOF\n", ptr_i8, n); - print_branch(f, frame, NULL, NULL); + print_branch(f, frame, + end_bits == VM_END_FAIL ? &fail : NULL, + NULL); print_label(f, true, "f%u", b); print_decl(f, "c", n); @@ -300,8 +330,10 @@ print_fetch(FILE *f, const struct fsm_options *opt, exit(EXIT_FAILURE); } - // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: - fprintf(f, "t%u:\n", use(&frame->b)); + if (end_bits != VM_END_FAIL) { + // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: + fprintf(f, "t%u:\n", use(&frame->b)); + } } /* TODO: eventually to be non-static */ @@ -339,9 +371,14 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, print_jump(f, a->linked); + fprintf(f, "fail:\n"); + print_ret(f, -1); + struct frame frame = { 0, 0, 0 }; for (op = a->linked; op != NULL; op = op->next) { - print_label(f, true, "l%" PRIu32, op->index); + if (op->instr != VM_OP_STOP || op->cmp != VM_CMP_ALWAYS || op->u.stop.end_bits != VM_END_FAIL) { + print_label(f, true, "l%" PRIu32, op->index); + } if (op->ir_state != NULL && op->ir_state->example != NULL) { /* C's escaping seems to be a subset of llvm's, and these are @@ -359,19 +396,27 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, unsigned b = decl(&frame.b); print_cond(f, op, opt, &frame); - print_branch(f, &frame, NULL, op->next); - print_label(f, true, "t%u", b); + print_branch(f, &frame, + op->u.stop.end_bits == VM_END_FAIL ? &fail : NULL, + op->next); + if (op->u.stop.end_bits != VM_END_FAIL) { + print_label(f, true, "t%u", b); + } } - if (-1 == print_end(f, op, opt, ir, op->u.stop.end_bits)) { + if (op->u.stop.end_bits == VM_END_FAIL) { + /* handled above */ + } else if (-1 == print_end(f, op, opt, ir, op->u.stop.end_bits)) { return -1; } break; case VM_OP_FETCH: { - print_fetch(f, opt, &frame, op->next); + print_fetch(f, opt, &frame, op->next, op->u.fetch.end_bits); - if (-1 == print_end(f, op, opt, ir, op->u.fetch.end_bits)) { + if (op->u.fetch.end_bits == VM_END_FAIL) { + /* handled in print_fetch() */ + } else if (-1 == print_end(f, op, opt, ir, op->u.fetch.end_bits)) { return -1; } break; @@ -384,7 +429,9 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, print_jump(f, dest); } else { print_cond(f, op, opt, &frame); - print_branch(f, &frame, dest, op->next); + print_branch(f, &frame, + dest, + op->next); } break; } From 8697e9851364a084bf8ba30a8cf4d540d7c02b76 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 1 Jul 2024 00:51:20 +0100 Subject: [PATCH 194/373] Construct retlist and output deduped rets. This isn't used yet, but it's a step towards generating phi. This looks like: ``` fail: ret i32 -1 ret0: ret i32 u0x1 ; "a" ret1: ret i32 u0x2 ; "b" ret2: ret i32 u0x4 ; "c" ret3: ret i32 u0x3 ; "a", "b" ret5: ret i32 u0x6 ; "b", "c" ret6: ret i32 u0x7 ; "a", "b", "c" ``` WIP --- src/libfsm/print/llvm.c | 186 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 180 insertions(+), 6 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 9c9e7a84b..a9acd9060 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -336,6 +337,150 @@ print_fetch(FILE *f, const struct fsm_options *opt, } } +struct ret { + size_t count; + const fsm_end_id_t *ids; + const struct ir_state *ir_state; // TODO: remove when we return endids only +}; + +struct ret_list { + size_t count; + struct ret *a; +}; + +static bool +append_ret(struct ret_list *list, + const fsm_end_id_t *ids, size_t count, + const struct ir_state *ir_state) +{ + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + + assert(list != NULL); + + if (list->count == 0) { + list->a = malloc(low * sizeof *list->a); + if (list->a == NULL) { + return false; + } + } else if (list->count >= low && (list->count & (list->count - 1)) == 0) { + void *tmp; + size_t new = list->count * factor; + if (new < list->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + tmp = realloc(list->a, new * sizeof *list->a); + if (tmp == NULL) { + return false; + } + + list->a = tmp; + } + + list->a[list->count].ids = ids; + list->a[list->count].count = count; + + /* .ir_state is not part of the "key" for struct ret, + * we ignore it for comparisons and effectively pick + * an arbitrary ir_state for output */ + list->a[list->count].ir_state = ir_state; + + list->count++; + + return true; +} + +static int +cmp_ret(const void *pa, const void *pb) +{ + const struct ret *a = pa; + const struct ret *b = pb; + + if (a->count < b->count) { + return -1; + } + + if (a->count > b->count) { + return 1; + } + + /* .ir_state intentionally ignored */ + + return memcmp(a->ids, b->ids, a->count); +} + +/* TODO: centralise */ +static int +cmp_id(const void *pa, const void *pb) +{ + const fsm_end_id_t *a = pa; + const fsm_end_id_t *b = pb; + + if (*a < *b) { + return -1; + } + + if (*a > *b) { + return 1; + } + + return 0; +} + +static bool +build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) +{ + const struct dfavm_op_ir *op; + + assert(list != NULL); + + for (op = a; op != NULL; op = op->next) { + switch (op->instr) { + case VM_OP_STOP: + if (op->u.stop.end_bits == VM_END_FAIL) { + /* always present */ + continue; + } + + break; + + case VM_OP_FETCH: + if (op->u.fetch.end_bits == VM_END_FAIL) { + /* always present */ + continue; + } + + break; + + case VM_OP_BRANCH: + continue; + + default: + assert(!"unreached"); + abort(); + } + + // sorting ids here so we can memcmp() (struct ret).a + qsort(op->ir_state->endids.ids, op->ir_state->endids.count, + sizeof *op->ir_state->endids.ids, cmp_id); + + // TODO; pick up endids here + if (!append_ret(list, + op->ir_state->endids.ids, op->ir_state->endids.count, + op->ir_state)) + { + return false; + } + } + + qsort(list->a, list->count, sizeof *list->a, cmp_ret); + + return true; +} + /* TODO: eventually to be non-static */ static int fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, @@ -369,10 +514,42 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, } } - print_jump(f, a->linked); + { + struct ret_list retlist; + size_t prev; - fprintf(f, "fail:\n"); - print_ret(f, -1); + retlist.count = 0; + build_retlist(&retlist, a->linked); + + print_jump(f, a->linked); + + print_label(f, true, "fail"); + print_ret(f, -1); + + for (size_t i = 0; i < retlist.count; i++) { + if (i > 0 && cmp_ret(&retlist.a[i], &retlist.a[prev]) == 0) { + continue; + } + + print_label(f, true, "ret%u", i); + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, + retlist.a[i].ids, retlist.a[i].count, + opt->endleaf_opaque)) + { + return -1; + } + } else { + print_ret(f, retlist.a[i].ir_state - ir->states); + } + + prev = i; + } + + if (retlist.count > 0) { + free(retlist.a); + } + } struct frame frame = { 0, 0, 0 }; for (op = a->linked; op != NULL; op = op->next) { @@ -442,9 +619,6 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, } } -// TODO: collate ret values together at the end, keeps them out of the i-cache maybe -// or better yet, have one ret: label and emit a phi - return 0; } From b11d5427d0a42395daa17dc72ac40c782244dcf2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 1 Jul 2024 22:29:40 +0100 Subject: [PATCH 195/373] Branch to centralised retlist labels. --- src/libfsm/print/llvm.c | 155 ++++++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 68 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index a9acd9060..2d896aa1c 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -179,28 +179,6 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, fprintf(f, "\n"); } -static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - const struct ir *ir, enum dfavm_op_end end_bits) -{ - if (end_bits == VM_END_FAIL) { - print_ret(f, -1); - } else { - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - print_ret(f, op->ir_state - ir->states); - } - } - - return 0; -} - static void print_jump(FILE *f, const struct dfavm_op_ir *dest) { @@ -394,42 +372,49 @@ append_ret(struct ret_list *list, } static int -cmp_ret(const void *pa, const void *pb) +cmp_ret_by_endid(const void *pa, const void *pb) { const struct ret *a = pa; const struct ret *b = pb; - if (a->count < b->count) { - return -1; - } - - if (a->count > b->count) { - return 1; - } + if (a->count < b->count) { return -1; } + if (a->count > b->count) { return +1; } /* .ir_state intentionally ignored */ - return memcmp(a->ids, b->ids, a->count); + return memcmp(a->ids, b->ids, a->count * sizeof *a->ids); } -/* TODO: centralise */ static int -cmp_id(const void *pa, const void *pb) +cmp_ret_by_state(const void *pa, const void *pb) { - const fsm_end_id_t *a = pa; - const fsm_end_id_t *b = pb; + const struct ret *a = pa; + const struct ret *b = pb; - if (*a < *b) { - return -1; - } + if (a->ir_state < b->ir_state) { return -1; } + if (a->ir_state > b->ir_state) { return +1; } - if (*a > *b) { - return 1; - } + /* .ids intentionally ignored */ return 0; } +static struct ret * +find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, + int (*cmp)(const void *pa, const void *pb)) +{ + struct ret key; + + assert(op != NULL); + assert(cmp != NULL); + + key.count = op->ir_state->endids.count; + key.ids = op->ir_state->endids.ids; + key.ir_state = op->ir_state; + + return bsearch(&key, list->a, list->count, sizeof *list->a, cmp); +} + static bool build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) { @@ -441,7 +426,7 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) switch (op->instr) { case VM_OP_STOP: if (op->u.stop.end_bits == VM_END_FAIL) { - /* always present */ + /* %fail is special, don't add to retlist */ continue; } @@ -449,7 +434,7 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) case VM_OP_FETCH: if (op->u.fetch.end_bits == VM_END_FAIL) { - /* always present */ + /* %fail is special, don't add to retlist */ continue; } @@ -463,11 +448,6 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) abort(); } - // sorting ids here so we can memcmp() (struct ret).a - qsort(op->ir_state->endids.ids, op->ir_state->endids.count, - sizeof *op->ir_state->endids.ids, cmp_id); - - // TODO; pick up endids here if (!append_ret(list, op->ir_state->endids.ids, op->ir_state->endids.count, op->ir_state)) @@ -476,8 +456,6 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) } } - qsort(list->a, list->count, sizeof *list->a, cmp_ret); - return true; } @@ -489,6 +467,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { + struct ret_list retlist; struct dfavm_op_ir *op; assert(f != NULL); @@ -515,24 +494,46 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, } { - struct ret_list retlist; - size_t prev; - retlist.count = 0; build_retlist(&retlist, a->linked); + /* sort for both dedup and bsearch */ + qsort(retlist.a, retlist.count, sizeof *retlist.a, + opt->endleaf != NULL ? cmp_ret_by_endid : cmp_ret_by_state); + + /* + * If we're going by endleaf, we deal with endids. + * If we don't have endleaf, we deal with state numbers. + */ + if (opt->endleaf != NULL && retlist.count > 0) { + size_t j = 0; + + /* deduplicate based on endids only. + * j is the start of a run; i increments until we find + * the start of the next run */ + for (size_t i = 1; i < retlist.count; i++) { + assert(i > j); + if (cmp_ret_by_endid(&retlist.a[j], &retlist.a[i]) == 0) { + continue; + } + + j++; + retlist.a[j] = retlist.a[i]; + } + + retlist.count = j + 1; + + assert(retlist.count > 0); + } + print_jump(f, a->linked); print_label(f, true, "fail"); print_ret(f, -1); for (size_t i = 0; i < retlist.count; i++) { - if (i > 0 && cmp_ret(&retlist.a[i], &retlist.a[prev]) == 0) { - continue; - } - - print_label(f, true, "ret%u", i); if (opt->endleaf != NULL) { + print_label(f, true, "ret%zu", i); if (-1 == opt->endleaf(f, retlist.a[i].ids, retlist.a[i].count, opt->endleaf_opaque)) @@ -540,14 +541,9 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, return -1; } } else { + print_label(f, true, "ret%zu", i); print_ret(f, retlist.a[i].ir_state - ir->states); } - - prev = i; - } - - if (retlist.count > 0) { - free(retlist.a); } } @@ -572,6 +568,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->cmp != VM_CMP_ALWAYS) { unsigned b = decl(&frame.b); +// TODO: our ret%u: label goes in place of t%u here, which means we're replacing the NULL print_cond(f, op, opt, &frame); print_branch(f, &frame, op->u.stop.end_bits == VM_END_FAIL ? &fail : NULL, @@ -583,8 +580,17 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->u.stop.end_bits == VM_END_FAIL) { /* handled above */ - } else if (-1 == print_end(f, op, opt, ir, op->u.stop.end_bits)) { - return -1; + } else { + assert(retlist.count > 0); + const struct ret *ret = find_ret(&retlist, op, + opt->endleaf != NULL ? cmp_ret_by_endid : cmp_ret_by_state); + assert(ret != NULL); + assert(ret >= retlist.a && ret <= (retlist.a + retlist.count)); + assert(ret->count == op->ir_state->endids.count); + assert(0 == memcmp(ret->ids, op->ir_state->endids.ids, ret->count)); + fprintf(f, "\tbr "); + print_label(f, false, "ret%u", ret - retlist.a); + fprintf(f, "\n"); } break; @@ -593,8 +599,17 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->u.fetch.end_bits == VM_END_FAIL) { /* handled in print_fetch() */ - } else if (-1 == print_end(f, op, opt, ir, op->u.fetch.end_bits)) { - return -1; + } else { + assert(retlist.count > 0); + const struct ret *ret = find_ret(&retlist, op, + opt->endleaf != NULL ? cmp_ret_by_endid : cmp_ret_by_state); + assert(ret != NULL); + assert(ret >= retlist.a && ret <= (retlist.a + retlist.count)); + assert(ret->count == op->ir_state->endids.count); + assert(0 == memcmp(ret->ids, op->ir_state->endids.ids, ret->count)); + fprintf(f, "\tbr "); + print_label(f, false, "ret%u", ret - retlist.a); + fprintf(f, "\n"); } break; } @@ -619,6 +634,10 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, } } + if (retlist.count > 0) { + free(retlist.a); + } + return 0; } From 0ae60dba8c8b18aa8af15ec64f8efe0648121744 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 2 Jul 2024 00:26:51 +0100 Subject: [PATCH 196/373] Jump via ret*: labels to a single stop: This keeps the phi node constant size wrt the number of basic blocks. I'm very proud of this idea. --- src/libfsm/print/llvm.c | 69 +++++++++++++++++++++++++++++++++-------- src/re/main.c | 7 +++-- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 2d896aa1c..a2728dafd 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -109,12 +109,6 @@ print_decl(FILE *f, const char *name, unsigned n) fprintf(f, "\t%%%s%u = ", name, n); } -static void -print_ret(FILE *f, long l) -{ - fprintf(f, "\tret i32 %ld\n", l); -} - static void vprint_label(FILE *f, bool decl, const char *fmt, va_list ap) { @@ -528,23 +522,72 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, print_jump(f, a->linked); - print_label(f, true, "fail"); - print_ret(f, -1); - + /* + * We're jumping to ret*: labels, and having each jump + * to a single stop: with a phi instruction. + * + * This looks like: + * + * stop: + * %ret = phi i32 + * [u0x1, %ret0], ; "abc" + * [u0x2, %ret1], ; "xyz" + * [u0x3, %ret2], ; "abc", "xyz" + * [-1, %fail] + * ret i32 %ret + * fail: + * br label %stop + * ret0: + * br label %stop + * ret1: + * br label %stop + * ret2: + * br label %stop + * + * And we jump to stop: via the ret*: labels rather than + * to a phi node directly. This helps for two reasons: + * + * - We don't need to track every location in the DFA + * that can stop. In other words, many basic blocks + * jump to the same ret*: label. + * + * - The number of basic blocks grows quadratically + * (as DFA grow quadratically), but the set of endids + * remains constant. So the phi list is small. + * + * llvm doesn't find this optimisation for us. + */ + print_label(f, true, "stop"); + fprintf(f, "\t%%ret = phi i32\n"); for (size_t i = 0; i < retlist.count; i++) { + fprintf(f, "\t "); + if (opt->endleaf != NULL) { - print_label(f, true, "ret%zu", i); if (-1 == opt->endleaf(f, retlist.a[i].ids, retlist.a[i].count, - opt->endleaf_opaque)) + &i)) // XXX: passing &i rather than opt->endleaf_opaque is a hack { return -1; } } else { - print_label(f, true, "ret%zu", i); - print_ret(f, retlist.a[i].ir_state - ir->states); + fprintf(f, "[%td, %%ret%zu],\n", + retlist.a[i].ir_state - ir->states, i); } } + fprintf(f, "\t%s[-1, %%fail]\n", " "); + fprintf(f, "\tret i32 %%ret\n"); + + print_label(f, true, "fail"); + fprintf(f, "\tbr "); + print_label(f, false, "stop"); + fprintf(f, "\n"); + + for (size_t i = 0; i < retlist.count; i++) { + print_label(f, true, "ret%zu", i); + fprintf(f, "\tbr "); + print_label(f, false, "stop"); + fprintf(f, "\n"); + } } struct frame frame = { 0, 0, 0 }; diff --git a/src/re/main.c b/src/re/main.c index a7c8bc803..c29108ab9 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -503,10 +503,11 @@ endleaf_llvm(FILE *f, unsigned n; size_t i; - assert(endleaf_opaque == NULL); + /* hack for llvm codegen only */ + const size_t *ret = endleaf_opaque; + assert(ret != NULL); (void) f; - (void) endleaf_opaque; n = 0; @@ -516,7 +517,7 @@ endleaf_llvm(FILE *f, } } - fprintf(f, "\tret i32 u%#x", (unsigned) n); + fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, *ret); fprintf(f, " ; "); From 49118f53930f4c7af396f716ca16ccdaa445a41c Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 4 Jul 2024 08:31:34 +0100 Subject: [PATCH 197/373] An arguably less worse workaround to keep .endleaf_opaque available. --- src/libfsm/print/llvm.c | 2 +- src/re/main.c | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index a2728dafd..90828ce95 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -565,7 +565,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, retlist.a[i].ids, retlist.a[i].count, - &i)) // XXX: passing &i rather than opt->endleaf_opaque is a hack + opt->endleaf_opaque)) { return -1; } diff --git a/src/re/main.c b/src/re/main.c index c29108ab9..e164279f7 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -503,11 +503,20 @@ endleaf_llvm(FILE *f, unsigned n; size_t i; - /* hack for llvm codegen only */ - const size_t *ret = endleaf_opaque; - assert(ret != NULL); + /* + * XXX: Hack for llvm codegen only, we don't have a way to pass this + * value from fsm_print_llvm(). We're working around that by making + * an assumption about the ordering for ret label numbers, that + * they're output sequentially. + * + * And also it's not very forward-thinking to keep this in static storage. + */ + static size_t ret = 0; + + assert(endleaf_opaque == NULL); (void) f; + (void) endleaf_opaque; n = 0; @@ -517,7 +526,7 @@ endleaf_llvm(FILE *f, } } - fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, *ret); + fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, ret++); fprintf(f, " ; "); From 0b84d755bdaaac0271014385af605ef65df41ef4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 6 Jul 2024 01:30:01 +0100 Subject: [PATCH 198/373] Centralise fsm_print() and friends. Now you call fsm_print(f, fsm, FSM_PRINT_*) rather than fsm_print_*(). This might look a bit cumbersome to the caller, but I'm doing it so we have a single entry point to put shared stuff. More on that later. --- examples/bm/libfsm.c | 2 +- examples/glob/main.c | 4 +- examples/iprange/main.c | 126 ++++++++---------- examples/utf8dfa/main.c | 12 +- examples/words/main.c | 14 +- fuzz/target.c | 50 ++++---- include/fsm/options.h | 1 + include/fsm/print.h | 75 +++++------ src/fsm/main.c | 70 +++++----- src/libfsm/Makefile | 1 + src/libfsm/determinise.c | 2 +- src/libfsm/epsilons.c | 4 +- src/libfsm/libfsm.syms | 16 +-- src/libfsm/minimise.c | 4 +- src/libfsm/print.c | 58 +++++++++ src/libfsm/print.h | 37 ++++++ src/libfsm/print/vmasm.c | 14 +- src/libre/Makefile | 1 + src/libre/ast_analysis.c | 1 - src/libre/ast_compile.c | 4 - src/libre/libre.syms | 6 +- src/libre/print.c | 43 +++++++ src/libre/print.h | 28 ++-- src/libre/re.c | 1 - src/lx/Makefile | 1 + src/lx/main.c | 48 ++++--- src/lx/print.c | 44 +++++++ src/lx/print.h | 24 +++- src/lx/print/dot.c | 2 +- src/re/main.c | 155 ++++++++++++++--------- src/retest/main.c | 5 +- src/retest/reperf.c | 5 +- src/retest/runner.c | 20 +-- tests/aho_corasick/actest.c | 10 +- tests/capture/captest.c | 2 +- tests/capture/capture3.c | 14 +- tests/capture/capture4.c | 10 +- tests/capture/capture5.c | 6 +- tests/capture/capture_concat1.c | 4 +- tests/capture/capture_concat2.c | 4 +- tests/capture/capture_union1.c | 4 +- tests/gen/gen3.c | 2 +- tests/minimise/minimise_test_case_list.c | 4 +- theft/fuzz_capture_string_set.c | 8 +- theft/fuzz_nfa.c | 2 +- 45 files changed, 564 insertions(+), 384 deletions(-) create mode 100644 src/libfsm/print.c create mode 100644 src/libfsm/print.h create mode 100644 src/libre/print.c create mode 100644 src/lx/print.c diff --git a/examples/bm/libfsm.c b/examples/bm/libfsm.c index 27bedbb8e..80b013d63 100644 --- a/examples/bm/libfsm.c +++ b/examples/bm/libfsm.c @@ -80,7 +80,7 @@ main(int argc, char *argv[]) printf("#include \n"); printf("\n"); - fsm_print_c(stdout, fsm); + fsm_print(stdout, fsm, FSM_PRINT_C); printf("int\n"); printf("main(void)\n"); diff --git a/examples/glob/main.c b/examples/glob/main.c index 38970e559..a5af0f317 100644 --- a/examples/glob/main.c +++ b/examples/glob/main.c @@ -55,7 +55,7 @@ match(const struct fsm *fsm, const char *s) assert(fsm_all(fsm, fsm_isdfa)); assert(s != NULL); - if (1 != fsm_exec(fsm, fsm_sgetc, &s, &state)) { + if (1 != fsm_exec(fsm, fsm_sgetc, &s, &state, NULL)) { return 0; } @@ -196,7 +196,7 @@ main(int argc, char *argv[]) } if (!quiet) { - fsm_print_fsm(stdout, fsm); + fsm_print(stdout, fsm, FSM_PRINT_FSM); } matched = match(fsm, argv[1]); diff --git a/examples/iprange/main.c b/examples/iprange/main.c index d88593eff..6ce49993b 100644 --- a/examples/iprange/main.c +++ b/examples/iprange/main.c @@ -49,7 +49,8 @@ #define VALUE_SEP ',' static struct fsm *fsm; -static struct fsm_state *fsmstart; +static fsm_state_t fsm_start; +static fsm_state_t fsm_none = (fsm_state_t) -1; static unsigned char zeroes[16]; static unsigned char ones[16]; @@ -61,10 +62,10 @@ struct record { unsigned id; struct fsm *fsm; - struct fsm_state *start; - struct fsm_state *end; + fsm_state_t start; + fsm_state_t end; struct { - struct fsm_state *s; + fsm_state_t s; unsigned char c; } regs[16]; }; @@ -120,21 +121,22 @@ get_id(char *rec, size_t reclen) exit(-1); } - r->start = fsm_addstate(r->fsm); - if (r->start == NULL) { + if (!fsm_addstate(r->fsm, &r->start)) { perror("fsm_addstate"); exit(-1); } fsm_setstart(r->fsm, r->start); - r->end = fsm_addstate(r->fsm); - if (r->end == NULL) { + if (!fsm_addstate(r->fsm, &r->end)) { perror("fsm_addstate"); exit(-1); } fsm_setend(r->fsm, r->end, 1); - memset(r->regs, 0, sizeof r->regs); + for (size_t i = 0; i < sizeof r->regs; i++) { + r->regs[i].c = '\0'; + r->regs[i].s = fsm_none; + } RB_INSERT(recmap, &recmap, r); @@ -170,7 +172,7 @@ usage(void) exit(-1); } -struct fsm_state * +static fsm_state_t get_from(struct record *r, unsigned oct) { if (oct == 0) { @@ -180,15 +182,16 @@ get_from(struct record *r, unsigned oct) return r->regs[oct - 1].s; } -struct fsm_state * +static fsm_state_t get_to(struct record *r, unsigned oct, unsigned noct, const unsigned char *octs) { - if (r->regs[oct].s != NULL && r->regs[oct].c == octs[oct]) { + fsm_state_t to; + + if (r->regs[oct].s != fsm_none && r->regs[oct].c == octs[oct]) { return r->regs[oct].s; } - struct fsm_state *to = fsm_addstate(r->fsm); - if (to == NULL) { + if (!fsm_addstate(r->fsm, &to)) { perror("fsm_addstate"); exit(-1); } @@ -197,7 +200,7 @@ get_to(struct record *r, unsigned oct, unsigned noct, const unsigned char *octs) r->regs[oct].c = octs[oct]; for (unsigned i = oct + 1; i < noct; i++) { - r->regs[i].s = NULL; + r->regs[i].s = fsm_none; } return to; @@ -207,8 +210,7 @@ static void gen_edge_range(struct record *r, unsigned oct, const unsigned char *octs, unsigned range_start, unsigned range_end, unsigned noct, unsigned final) { - struct fsm_state *from, *to; - struct fsm_edge *e; + fsm_state_t from, to; from = get_from(r, oct); if (oct > 0) { @@ -225,8 +227,7 @@ gen_edge_range(struct record *r, unsigned oct, const unsigned char *octs, } for (unsigned i = range_start; i <= range_end; i++) { - e = fsm_addedge_literal(r->fsm, from, to, (unsigned char) i); - if (e == NULL) { + if (!fsm_addedge_literal(r->fsm, from, to, (unsigned char) i)) { perror("fsm_addedge_literal"); exit(-1); } @@ -420,47 +421,19 @@ important(unsigned n) } } -static void -carryopaque(const struct fsm *src_fsm, const fsm_state_t *src_set, size_t n, - struct fsm *dst_fsm, struct fsm_state *dst_state) +static int +leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { - void *o = NULL; - size_t i; - - assert(src_fsm != NULL); - assert(src_set != NULL); - assert(n > 0); - assert(dst_fsm != NULL); - assert(fsm_isend(dst_fsm, dst_state)); - assert(fsm_getopaque(dst_fsm, dst_state) == NULL); - - for (i = 0; i < n; i++) { - /* - * The opaque data is attached to end states only, so we skip - * non-end states here. - */ - if (!fsm_isend(src_fsm, src_set[i])) { - continue; - } - - assert(fsm_getopaque(src_fsm, src_set[i]) != NULL); + const struct record *r; - if (o == NULL) { - o = fsm_getopaque(src_fsm, src_set[i]); - fsm_setopaque(dst_fsm, dst_state, o); - continue; - } + (void) leaf_opaque; - assert(o == fsm_getopaque(src_fsm, src_set[i])); + if (count != 1) { + fprintf(f, "endid conflict\n"); + exit(EXIT_FAILURE); } -} -static int -leaf(FILE *f, const void *state_opaque, const void *leaf_opaque) -{ - const struct record *r = state_opaque; - - (void) leaf_opaque; + r = (const void *) (intptr_t) ids[0]; /* XXX */ if (r == NULL) { fprintf(f, "return -1;"); @@ -473,19 +446,27 @@ leaf(FILE *f, const void *state_opaque, const void *leaf_opaque) } static int -endleaf_dot(FILE *f, const void *state_opaque, const void *endleaf_opaque) +endleaf_dot(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque) { const struct record *r; assert(f != NULL); - assert(state_opaque != NULL); assert(endleaf_opaque == NULL); (void) endleaf_opaque; - r = state_opaque; + fprintf(f, "label = <"); + + for (size_t i = 0; i < count; i++) { + r = (const void *) (intptr_t) ids[i]; /* XXX */ + + fprintf(f, "%s", r->rec); /* XXX: escape */ + if (i + 1 < count) { + fprintf(f, ", "); + } + } - fprintf(f, "label = <%s>", r->rec); /* XXX: escape */ + fprintf(f, ">"); return 0; } @@ -556,8 +537,7 @@ main(int argc, char **argv) return -1; } - fsmstart = fsm_addstate(fsm); - if (fsmstart == NULL) { + if (!fsm_addstate(fsm, &fsm_start)) { perror("fsm_addstate"); return -1; } @@ -661,17 +641,17 @@ main(int argc, char **argv) exit(-1); } - fsm_setendopaque(r->fsm, r); + fsm_setendid(r->fsm, (intptr_t) r); /* XXX */ (void) fsm_getstart(r->fsm, &start); - fsm = fsm_merge(fsm, r->fsm); + fsm = fsm_merge(fsm, r->fsm, NULL); if (fsm == NULL) { perror("fsm_merge"); exit(-1); } - if (!fsm_addedge_epsilon(fsm, fsmstart, start)) { + if (!fsm_addedge_epsilon(fsm, fsm_start, start)) { perror("fsm_addedge_epsilon"); exit(-1); } @@ -686,7 +666,7 @@ main(int argc, char **argv) } } - fsm_setstart(fsm, fsmstart); + fsm_setstart(fsm, fsm_start); if (progress) { tend = time(NULL); @@ -696,15 +676,9 @@ main(int argc, char **argv) tstart = time(NULL); } - { - opt.carryopaque = carryopaque; - - if (!fsm_determinise(fsm) == 0) { - perror("fsm_determinise"); - exit(-1); - } - - opt.carryopaque = NULL; + if (!fsm_determinise(fsm) == 0) { + perror("fsm_determinise"); + exit(-1); } if (progress) { @@ -717,11 +691,11 @@ main(int argc, char **argv) opt.cp = "c"; opt.leaf = leaf; opt.leaf_opaque = NULL; - fsm_print_c(stdout, fsm); + fsm_print(stdout, fsm, FSM_PRINT_C); } else if (odot) { opt.endleaf = endleaf_dot; opt.endleaf_opaque = NULL; - fsm_print_dot(stdout, fsm); + fsm_print(stdout, fsm, FSM_PRINT_DOT); } } diff --git a/examples/utf8dfa/main.c b/examples/utf8dfa/main.c index fa26bbddd..d987575b3 100644 --- a/examples/utf8dfa/main.c +++ b/examples/utf8dfa/main.c @@ -120,7 +120,7 @@ main(int argc, char *argv[]) { struct fsm *fsm; fsm_state_t start; - fsm_print *print; + enum fsm_print_lang lang; opt.anonymous_states = 1; opt.consolidate_edges = 1; @@ -128,7 +128,7 @@ main(int argc, char *argv[]) opt.comments = 0; opt.case_ranges = 0; - print = fsm_print_api; + lang = FSM_PRINT_API; { int c; @@ -139,11 +139,11 @@ main(int argc, char *argv[]) case 'l': if (strcmp(optarg, "dot") == 0) { - print = fsm_print_dot; + lang = FSM_PRINT_DOT; } else if (strcmp(optarg, "api") == 0) { - print = fsm_print_api; + lang = FSM_PRINT_API; } else if (strcmp(optarg, "c") == 0) { - print = fsm_print_c; + lang = FSM_PRINT_C; } else { fprintf(stderr, "Invalid language '%s'", optarg); exit(1); @@ -220,7 +220,7 @@ main(int argc, char *argv[]) exit(1); } - print(stdout, fsm); + fsm_print(stdout, fsm, lang); fsm_free(fsm); diff --git a/examples/words/main.c b/examples/words/main.c index 5aebcf8cc..1fe19ddf1 100644 --- a/examples/words/main.c +++ b/examples/words/main.c @@ -28,7 +28,7 @@ int main(int argc, char *argv[]) { fsm_state_t start; char s[BUFSIZ]; int (*dmf)(struct fsm *); - fsm_print *print; + enum fsm_print_lang lang; unsigned long ms, mt; int timing; int native = 0; @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) { opt.consolidate_edges = 1; dmf = NULL; - print = NULL; + lang = FSM_PRINT_NONE; timing = 0; { @@ -55,8 +55,8 @@ int main(int argc, char *argv[]) { case 'n': native = 1; break; case 'N': unanchored = 1; break; - case 'c': print = fsm_print_dot; break; - case 'f': print = fsm_print_fsm; break; + case 'c': lang = FSM_PRINT_DOT; break; + case 'f': lang = FSM_PRINT_FSM; break; case 't': timing = 1; @@ -116,7 +116,7 @@ int main(int argc, char *argv[]) { } if (ahocorasick) { - if (!re_strings_add_str(g, s)) { + if (!re_strings_add_str(g, s, NULL)) { perror("re_strings_add_str"); exit(EXIT_FAILURE); } @@ -211,9 +211,7 @@ int main(int argc, char *argv[]) { + ((long) post.tv_nsec - (long) pre.tv_nsec) / 1000000; } - if (print != NULL) { - print(stdout, fsm); - } + fsm_print(stdout, fsm, lang); if (timing) { printf("construction, reduction, total: %lu, %lu, %lu\n", ms, mt, ms + mt); diff --git a/fuzz/target.c b/fuzz/target.c index 1a9690b48..6bb6f2772 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -137,7 +137,7 @@ codegen(const struct fsm *fsm) { FILE *dev_null = fopen("/dev/null", "w"); assert(dev_null != NULL); - fsm_print_c(dev_null, fsm); + fsm_print(dev_null, fsm, FSM_PRINT_C); fclose(dev_null); return 1; } @@ -245,14 +245,14 @@ shuffle_minimise(const char *pattern) __func__, s_i, pattern, expected_state_count, cp_state_count); fprintf(stderr, "== original input:\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fprintf(stderr, "== expected:\n"); - fsm_print_fsm(stderr, oracle_min); + fsm_print(stderr, oracle_min, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print_fsm(stderr, cp); + fsm_print(stderr, cp, FSM_PRINT_FSM); fsm_free(cp); fsm_free(oracle_min); @@ -312,24 +312,30 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const /* see if this triggers any asserts */ int r = 0; - r |= fsm_print_api(f, fsm); - r |= fsm_print_awk(f, fsm); - r |= fsm_print_c(f, fsm); - r |= fsm_print_dot(f, fsm); - r |= fsm_print_fsm(f, fsm); - r |= fsm_print_ir(f, fsm); - r |= fsm_print_irjson(f, fsm); - r |= fsm_print_json(f, fsm); - r |= fsm_print_vmc(f, fsm); - r |= fsm_print_vmdot(f, fsm); - r |= fsm_print_vmasm(f, fsm); - r |= fsm_print_vmasm_amd64_att(f, fsm); - r |= fsm_print_vmasm_amd64_nasm(f, fsm); - r |= fsm_print_vmasm_amd64_go(f, fsm); - r |= fsm_print_sh(f, fsm); - r |= fsm_print_go(f, fsm); - r |= fsm_print_rust(f, fsm); - r |= fsm_print_llvm(f, fsm); + + r |= fsm_print(f, fsm, FSM_PRINT_AMD64_ATT); + r |= fsm_print(f, fsm, FSM_PRINT_AMD64_GO); + r |= fsm_print(f, fsm, FSM_PRINT_AMD64_NASM); + + r |= fsm_print(f, fsm, FSM_PRINT_API); + r |= fsm_print(f, fsm, FSM_PRINT_AWK); + r |= fsm_print(f, fsm, FSM_PRINT_C); + r |= fsm_print(f, fsm, FSM_PRINT_DOT); + r |= fsm_print(f, fsm, FSM_PRINT_FSM); + r |= fsm_print(f, fsm, FSM_PRINT_GO); + r |= fsm_print(f, fsm, FSM_PRINT_IR); + r |= fsm_print(f, fsm, FSM_PRINT_IRJSON); + r |= fsm_print(f, fsm, FSM_PRINT_JSON); + r |= fsm_print(f, fsm, FSM_PRINT_LLVM); + r |= fsm_print(f, fsm, FSM_PRINT_RUST); + r |= fsm_print(f, fsm, FSM_PRINT_SH); + r |= fsm_print(f, fsm, FSM_PRINT_VMC); + r |= fsm_print(f, fsm, FSM_PRINT_VMDOT); + + r |= fsm_print(f, fsm, FSM_PRINT_VMOPS_C); + r |= fsm_print(f, fsm, FSM_PRINT_VMOPS_H); + r |= fsm_print(f, fsm, FSM_PRINT_VMOPS_MAIN); + assert(r == 0 || errno != 0); fsm_free(fsm); diff --git a/include/fsm/options.h b/include/fsm/options.h index e66c16b71..0f861baef 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -10,6 +10,7 @@ #include struct fsm; +struct fsm_alloc; struct fsm_state; enum fsm_io { diff --git a/include/fsm/print.h b/include/fsm/print.h index e85b6102c..61a2f7be8 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -11,58 +11,43 @@ struct fsm; +enum fsm_print_lang { + FSM_PRINT_NONE, /* No output */ + + FSM_PRINT_AMD64_ATT, /* Assembly in various dialects */ + FSM_PRINT_AMD64_GO, + FSM_PRINT_AMD64_NASM, + + FSM_PRINT_API, /* C code which calls the fsm(3) API */ + FSM_PRINT_AWK, /* awk code (gawk dialect) */ + FSM_PRINT_C, /* ISO C90 code */ + FSM_PRINT_DOT, /* Graphviz Dot format, intended for rendering graphically */ + FSM_PRINT_FSM, /* fsm(5) .fsm format, suitable for parsing by fsm(1) */ + FSM_PRINT_GO, /* Go code */ + FSM_PRINT_IR, /* Codegen IR as Dot */ + FSM_PRINT_IRJSON, /* Codegen IR as JSON */ + FSM_PRINT_JSON, /* JavaScript Object Notation */ + FSM_PRINT_LLVM, /* Live Laugh Virtual Machine */ + FSM_PRINT_RUST, /* Rust code */ + FSM_PRINT_SH, /* Shell script (bash dialect) */ + FSM_PRINT_VMC, /* ISO C90 code, VM style */ + FSM_PRINT_VMDOT, /* Graphviz Dot format, showing VM opcodes */ + + FSM_PRINT_VMOPS_C, /* VM opcodes as a datastructure */ + FSM_PRINT_VMOPS_H, + FSM_PRINT_VMOPS_MAIN +}; + /* * Print an FSM to the given file stream. The output is written in the format - * specified. The available formats are: - * - * fsm_print_api - C code which calls the fsm(3) API - * fsm_print_awk - awk code (gawk dialect) - * fsm_print_c - ISO C90 code - * fsm_print_dot - Graphviz Dot format, intended for rendering graphically - * fsm_print_fsm - fsm(5) .fsm format, suitable for parsing by fsm(1) - * fsm_print_ir - Codegen IR as Dot - * fsm_print_irjson - Codegen IR as JSON - * fsm_print_json - JavaScript Object Notation - * fsm_print_vmc - ISO C90 code, VM style - * fsm_print_vmdot - Graphviz Dot format, showing VM opcodes - * fsm_print_rust - Rust code - * fsm_print_llvm - Live Laugh Virtual Machine - * fsm_print_sh - Shell script (bash dialect) - * fsm_print_go - Go code - * fsm_print_goasm/vmasm_* - Assembly in various dialects - * fsm_print_vmops_* - VM opcodes as a datastructure + * specified. * * The output options may be NULL, indicating to use defaults. * - * TODO: explain constraints - * * Returns 0, or -1 on error and errno will be set. An errno of ENOTSUP means - * the requested IO API is not implemented for this output format. + * the requested IO API is not implemented for this output language. */ - -typedef int (fsm_print)(FILE *f, const struct fsm *fsm); - -fsm_print fsm_print_api; -fsm_print fsm_print_awk; -fsm_print fsm_print_c; -fsm_print fsm_print_dot; -fsm_print fsm_print_fsm; -fsm_print fsm_print_ir; -fsm_print fsm_print_irjson; -fsm_print fsm_print_json; -fsm_print fsm_print_vmc; -fsm_print fsm_print_vmdot; -fsm_print fsm_print_vmasm; -fsm_print fsm_print_vmasm_amd64_att; /* output amd64 assembler in AT&T format */ -fsm_print fsm_print_vmasm_amd64_nasm; /* output amd64 assembler in NASM format */ -fsm_print fsm_print_vmasm_amd64_go; /* output amd64 assembler in Go format */ -fsm_print fsm_print_sh; -fsm_print fsm_print_go; -fsm_print fsm_print_rust; -fsm_print fsm_print_llvm; -fsm_print fsm_print_vmops_c; -fsm_print fsm_print_vmops_h; -fsm_print fsm_print_vmops_main; +int fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang); #endif diff --git a/src/fsm/main.c b/src/fsm/main.c index b6f011504..80540ee16 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -148,40 +148,40 @@ io(const char *name) exit(EXIT_FAILURE); } -static fsm_print * -print_name(const char *name) +static enum fsm_print_lang +lang_name(const char *name) { size_t i; struct { const char *name; - fsm_print *f; + enum fsm_print_lang lang; } a[] = { - { "api", fsm_print_api }, - { "awk", fsm_print_awk }, - { "c", fsm_print_c }, - { "dot", fsm_print_dot }, - { "fsm", fsm_print_fsm }, - { "ir", fsm_print_ir }, - { "json", fsm_print_json }, - { "vmc", fsm_print_vmc }, - { "vmdot", fsm_print_vmdot }, - { "rust", fsm_print_rust }, - { "llvm", fsm_print_llvm }, - { "sh", fsm_print_sh }, - { "go", fsm_print_go }, - - { "amd64", fsm_print_vmasm }, - { "amd64_att", fsm_print_vmasm_amd64_att }, - { "amd64_nasm", fsm_print_vmasm_amd64_nasm }, - { "amd64_go", fsm_print_vmasm_amd64_go } + { "api", FSM_PRINT_API }, + { "awk", FSM_PRINT_AWK }, + { "c", FSM_PRINT_C }, + { "dot", FSM_PRINT_DOT }, + { "fsm", FSM_PRINT_FSM }, + { "ir", FSM_PRINT_IR }, + { "json", FSM_PRINT_JSON }, + { "vmc", FSM_PRINT_VMC }, + { "vmdot", FSM_PRINT_VMDOT }, + { "rust", FSM_PRINT_RUST }, + { "llvm", FSM_PRINT_LLVM }, + { "sh", FSM_PRINT_SH }, + { "go", FSM_PRINT_GO }, + + { "amd64", FSM_PRINT_AMD64_NASM }, + { "amd64_att", FSM_PRINT_AMD64_ATT }, + { "amd64_nasm", FSM_PRINT_AMD64_NASM }, + { "amd64_go", FSM_PRINT_AMD64_GO } }; assert(name != NULL); for (i = 0; i < sizeof a / sizeof *a; i++) { if (0 == strcmp(a[i].name, name)) { - return a[i].f; + return a[i].lang; } } @@ -366,11 +366,11 @@ int main(int argc, char *argv[]) { unsigned iterations, i; - double elapsed; - fsm_print *print; + enum fsm_print_lang lang; enum op op; const char *charset; struct fsm *fsm; + double elapsed; int xfiles; int r; size_t generate_bounds = 0; @@ -385,7 +385,7 @@ main(int argc, char *argv[]) opt.io = FSM_IO_GETC; xfiles = 0; - print = NULL; + lang = FSM_PRINT_NONE; query = NULL; walk = NULL; op = OP_IDENTITY; @@ -415,8 +415,8 @@ main(int argc, char *argv[]) break; case 'x': xfiles = 1; break; - case 'l': print = print_name(optarg); break; - case 'p': print = fsm_print_fsm; break; + case 'l': lang = lang_name(optarg); break; + case 'p': lang = FSM_PRINT_FSM; break; case 'q': query = query_name(optarg, &walk); break; case 'd': op = op_name("determinise"); break; @@ -465,7 +465,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if ((op == OP_EQUAL) + !!print > 1) { + if ((op == OP_EQUAL) + (lang != FSM_PRINT_NONE) > 1) { fprintf(stderr, "-t equal and -p are mutually exclusive\n"); return EXIT_FAILURE; } @@ -708,15 +708,13 @@ main(int argc, char *argv[]) } } - if (print != NULL) { - if (-1 == print(stdout, fsm)) { - if (errno == ENOTSUP) { - fprintf(stderr, "unsupported IO API\n"); - } else { - perror("print_fsm"); - } - exit(EXIT_FAILURE); + if (-1 == fsm_print(stdout, fsm, lang)) { + if (errno == ENOTSUP) { + fprintf(stderr, "unsupported IO API\n"); + } else { + perror("fsm_print"); } + exit(EXIT_FAILURE); } if (generate_bounds > 0) { diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 2b7aba005..c1085763e 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -16,6 +16,7 @@ SRC += src/libfsm/exec.c SRC += src/libfsm/fsm.c SRC += src/libfsm/gen.c SRC += src/libfsm/mode.c +SRC += src/libfsm/print.c SRC += src/libfsm/start.c SRC += src/libfsm/state.c SRC += src/libfsm/trim.c diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index f0ae42f2a..d89259908 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -46,7 +46,7 @@ fsm_determinise(struct fsm *nfa) #if LOG_DETERMINISE_CAPTURES || LOG_INPUT fprintf(stderr, "# post_remove_epsilons, pre_determinise\n"); - fsm_print_fsm(stderr, nfa); + fsm_print(stderr, nfa, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index f8bb1d9d3..a5bb4da48 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -152,7 +152,7 @@ fsm_remove_epsilons(struct fsm *nfa) #if LOG_RESULT fprintf(stderr, "=== %s: about to update capture actions\n", __func__); - fsm_print_fsm(stderr, nfa); + fsm_print(stderr, nfa, FSM_PRINT_FSM); #endif if (!remap_capture_actions(nfa, eclosures)) { @@ -160,7 +160,7 @@ fsm_remove_epsilons(struct fsm *nfa) } #if LOG_RESULT - fsm_print_fsm(stderr, nfa); + fsm_print(stderr, nfa, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index b90cc01e7..28dfb9cc9 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -35,24 +35,24 @@ fsm_generate_cb_printf_escaped fsm_parse # +fsm_print +fsm_print_amd64_att +fsm_print_amd64_go +fsm_print_amd64_nasm fsm_print_api fsm_print_awk fsm_print_c fsm_print_dot fsm_print_fsm +fsm_print_go fsm_print_ir fsm_print_irjson fsm_print_json -fsm_print_vmc -fsm_print_vmdot -fsm_print_vmasm -fsm_print_vmasm_amd64_att -fsm_print_vmasm_amd64_nasm -fsm_print_vmasm_amd64_go -fsm_print_rust fsm_print_llvm +fsm_print_rust fsm_print_sh -fsm_print_go +fsm_print_vmc +fsm_print_vmdot fsm_print_vmops_c fsm_print_vmops_h fsm_print_vmops_main diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index d520b39fa..4ba32d297 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -144,10 +144,10 @@ fsm_minimise(struct fsm *fsm) fprintf(stderr, "%s: expected minimal DFA with %zu states, got %zu\n", __func__, exp_count, got_count); fprintf(stderr, "== expected:\n"); - fsm_print_fsm(stderr, oracle); + fsm_print(stderr, oracle, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print_fsm(stderr, dst); + fsm_print(stderr, dst, FSM_PRINT_FSM); assert(!"non-minimal result"); } diff --git a/src/libfsm/print.c b/src/libfsm/print.c new file mode 100644 index 000000000..2c99ac572 --- /dev/null +++ b/src/libfsm/print.c @@ -0,0 +1,58 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include +#include + +#include "print.h" + +int +fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) +{ + int (*print)(FILE *f, const struct fsm *fsm); + + assert(fsm != NULL); + + if (lang == FSM_PRINT_NONE) { + return 0; + } + + switch (lang) { + case FSM_PRINT_AMD64_ATT: print = fsm_print_amd64_att; break; + case FSM_PRINT_AMD64_GO: print = fsm_print_amd64_go; break; + case FSM_PRINT_AMD64_NASM: print = fsm_print_amd64_nasm; break; + + case FSM_PRINT_API: print = fsm_print_api; break; + case FSM_PRINT_AWK: print = fsm_print_awk; break; + case FSM_PRINT_C: print = fsm_print_c; break; + case FSM_PRINT_DOT: print = fsm_print_dot; break; + case FSM_PRINT_FSM: print = fsm_print_fsm; break; + case FSM_PRINT_GO: print = fsm_print_go; break; + case FSM_PRINT_IR: print = fsm_print_ir; break; + case FSM_PRINT_IRJSON: print = fsm_print_irjson; break; + case FSM_PRINT_JSON: print = fsm_print_json; break; + case FSM_PRINT_LLVM: print = fsm_print_llvm; break; + case FSM_PRINT_RUST: print = fsm_print_rust; break; + case FSM_PRINT_SH: print = fsm_print_sh; break; + case FSM_PRINT_VMC: print = fsm_print_vmc; break; + case FSM_PRINT_VMDOT: print = fsm_print_vmdot; break; + + case FSM_PRINT_VMOPS_C: print = fsm_print_vmops_c; break; + case FSM_PRINT_VMOPS_H: print = fsm_print_vmops_h; break; + case FSM_PRINT_VMOPS_MAIN: print = fsm_print_vmops_main; break; + + default: + errno = ENOTSUP; + return -1; + } + + return print(f, fsm); +} + diff --git a/src/libfsm/print.h b/src/libfsm/print.h new file mode 100644 index 000000000..a0c63e513 --- /dev/null +++ b/src/libfsm/print.h @@ -0,0 +1,37 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#ifndef FSM_INTERNAL_PRINT_H +#define FSM_INTERNAL_PRINT_H + +struct fsm; + +typedef int fsm_print_f(FILE *f, const struct fsm *fsm); + +fsm_print_f fsm_print_amd64_att; +fsm_print_f fsm_print_amd64_go; +fsm_print_f fsm_print_amd64_nasm; + +fsm_print_f fsm_print_api; +fsm_print_f fsm_print_awk; +fsm_print_f fsm_print_c; +fsm_print_f fsm_print_dot; +fsm_print_f fsm_print_fsm; +fsm_print_f fsm_print_go; +fsm_print_f fsm_print_ir; +fsm_print_f fsm_print_irjson; +fsm_print_f fsm_print_json; +fsm_print_f fsm_print_llvm; +fsm_print_f fsm_print_rust; +fsm_print_f fsm_print_sh; +fsm_print_f fsm_print_vmc; + +fsm_print_f fsm_print_vmdot; +fsm_print_f fsm_print_vmops_c; +fsm_print_f fsm_print_vmops_h; +fsm_print_f fsm_print_vmops_main; + +#endif diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 046610c44..5cfa43fc3 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -437,27 +437,19 @@ print_vmasm_encoding(FILE *f, const struct fsm *fsm, enum asm_dialect dialect) } int -fsm_print_vmasm(FILE *f, const struct fsm *fsm) -{ - static const enum asm_dialect default_dialect = AMD64_NASM; - - return print_vmasm_encoding(f, fsm, default_dialect); -} - -int -fsm_print_vmasm_amd64_att(FILE *f, const struct fsm *fsm) +fsm_print_amd64_att(FILE *f, const struct fsm *fsm) { return print_vmasm_encoding(f, fsm, AMD64_ATT); } int -fsm_print_vmasm_amd64_nasm(FILE *f, const struct fsm *fsm) +fsm_print_amd64_nasm(FILE *f, const struct fsm *fsm) { return print_vmasm_encoding(f, fsm, AMD64_NASM); } int -fsm_print_vmasm_amd64_go(FILE *f, const struct fsm *fsm) +fsm_print_amd64_go(FILE *f, const struct fsm *fsm) { return print_vmasm_encoding(f, fsm, AMD64_GO); } diff --git a/src/libre/Makefile b/src/libre/Makefile index a88a92418..67617ab92 100644 --- a/src/libre/Makefile +++ b/src/libre/Makefile @@ -9,6 +9,7 @@ SRC += src/libre/ast_compile.c SRC += src/libre/ast_new_from_fsm.c SRC += src/libre/ast_rewrite.c SRC += src/libre/ac.c +SRC += src/libre/print.c SRC += src/libre/re_strings.c # generated diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 03d6d79b7..8fd3b6395 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -13,7 +13,6 @@ #include #include "class.h" -#include "print.h" #include "ast.h" #include "ast_analysis.h" diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index b64edd290..852b267fc 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -28,10 +28,6 @@ #define LOG_LINKAGE 0 -#if LOG_LINKAGE -#include "print.h" -#endif - enum link_side { LINK_START, LINK_END diff --git a/src/libre/libre.syms b/src/libre/libre.syms index bc4e89c2a..a4f1a223b 100644 --- a/src/libre/libre.syms +++ b/src/libre/libre.syms @@ -4,12 +4,14 @@ re_flags re_strerror re_perror -# XXX: for re(1) -ast_new_from_fsm +ast_print ast_print_dot ast_print_abnf ast_print_pcre ast_print_tree + +# XXX: for re(1) +ast_new_from_fsm ast_free re_parse re_strings diff --git a/src/libre/print.c b/src/libre/print.c new file mode 100644 index 000000000..151545da5 --- /dev/null +++ b/src/libre/print.c @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#include "print.h" + +int +ast_print(FILE *f, const struct ast *ast, const struct fsm_options *opt, + enum re_flags re_flags, enum ast_print_lang lang) +{ + ast_print_f *print; + + assert(ast != NULL); + + if (lang == AST_PRINT_NONE) { + return 0; + } + + switch (lang) { + case AST_PRINT_ABNF: print = ast_print_abnf; break; + case AST_PRINT_DOT: print = ast_print_dot; break; + case AST_PRINT_PCRE: print = ast_print_pcre; break; + case AST_PRINT_TREE: print = ast_print_tree; break; + + default: + errno = ENOTSUP; + return -1; + } + + // TODO: return int + print(f, opt, re_flags, ast); + + return 0; +} + diff --git a/src/libre/print.h b/src/libre/print.h index b7bc2dba8..2e70a4d51 100644 --- a/src/libre/print.h +++ b/src/libre/print.h @@ -1,5 +1,5 @@ /* - * Copyright 2018 Scott Vokes + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -7,17 +7,27 @@ #ifndef RE_INTERNAL_PRINT_H #define RE_INTERNAL_PRINT_H -#include - struct ast; struct fsm_options; -typedef void (ast_print)(FILE *f, const struct fsm_options *opt, - enum re_flags re_flags, const struct ast *ast); +enum ast_print_lang { + AST_PRINT_NONE, + AST_PRINT_DOT, + AST_PRINT_ABNF, + AST_PRINT_PCRE, + AST_PRINT_TREE +}; + +// TODO: return int +typedef void ast_print_f(FILE *f, const struct fsm_options *opt, + enum re_flags re_flags, const struct ast *ast); + +ast_print_f ast_print_abnf; +ast_print_f ast_print_dot; +ast_print_f ast_print_pcre; +ast_print_f ast_print_tree; -ast_print ast_print_dot; -ast_print ast_print_abnf; -ast_print ast_print_pcre; -ast_print ast_print_tree; +int ast_print(FILE *f, const struct ast *ast, const struct fsm_options *opt, + enum re_flags re_flags, enum ast_print_lang lang); #endif diff --git a/src/libre/re.c b/src/libre/re.c index 2923b7d3d..33cc3234c 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -17,7 +17,6 @@ #include "ac.h" #include "class.h" -#include "print.h" #include "ast.h" #include "ast_analysis.h" #include "ast_compile.h" diff --git a/src/lx/Makefile b/src/lx/Makefile index 683341ab0..a8c293b88 100644 --- a/src/lx/Makefile +++ b/src/lx/Makefile @@ -3,6 +3,7 @@ SRC += src/lx/ast.c SRC += src/lx/lexer.c SRC += src/lx/parser.c +SRC += src/lx/print.c SRC += src/lx/main.c SRC += src/lx/tokens.c SRC += src/lx/var.c diff --git a/src/lx/main.c b/src/lx/main.c index cbcfeea76..0a91f4775 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -128,28 +128,28 @@ io(const char *name) exit(EXIT_FAILURE); } -static lx_print * -print_name(const char *name) +static enum lx_print_lang +lang_name(const char *name) { size_t i; struct { const char *name; - lx_print *f; + enum lx_print_lang lang; } a[] = { - { "test", NULL }, - { "dot", lx_print_dot }, - { "dump", lx_print_dump }, - { "zdot", lx_print_zdot }, - { "c", lx_print_c }, - { "h", lx_print_h } + { "test", LX_PRINT_NONE }, + { "dot", LX_PRINT_DOT }, + { "dump", LX_PRINT_DUMP }, + { "zdot", LX_PRINT_ZDOT }, + { "c", LX_PRINT_C }, + { "h", LX_PRINT_H } }; assert(name != NULL); for (i = 0; i < sizeof a / sizeof *a; i++) { if (0 == strcmp(a[i].name, name)) { - return a[i].f; + return a[i].lang; } } @@ -534,10 +534,10 @@ int main(int argc, char *argv[]) { struct ast *ast; - lx_print *print; + enum lx_print_lang lang; int concurrency; - print = lx_print_c; + lang = LX_PRINT_C; keep_nfa = 0; print_progress = 0; concurrency = 1; @@ -579,7 +579,7 @@ main(int argc, char *argv[]) case 'x': api_exclude |= lang_exclude(optarg); break; case 'l': - print = print_name(optarg); + lang = lang_name(optarg); break; case 'n': @@ -610,17 +610,17 @@ main(int argc, char *argv[]) } } - if (keep_nfa && print != lx_print_dot) { + if (keep_nfa && lang != LX_PRINT_DOT) { fprintf(stderr, "-n is for .dot output only\n"); return EXIT_FAILURE; } - if (api_tokbuf && (print != lx_print_c && print != lx_print_h && print != lx_print_dump)) { + if (api_tokbuf && (lang != LX_PRINT_C && lang != LX_PRINT_H && lang != LX_PRINT_DUMP)) { fprintf(stderr, "-b is for .c/.h output only\n"); return EXIT_FAILURE; } - if (api_getc && (print != lx_print_c && print != lx_print_h && print != lx_print_dump)) { + if (api_getc && (lang != LX_PRINT_C && lang != LX_PRINT_H && lang != LX_PRINT_DUMP)) { fprintf(stderr, "-g is for .c/.h output only\n"); return EXIT_FAILURE; } @@ -668,7 +668,7 @@ main(int argc, char *argv[]) * of which end state is associated with each mapping. In other words, * without losing track of which regexp maps to which token. */ - if (print != lx_print_h) { + if (lang != LX_PRINT_H) { if (print_progress) { fprintf(stderr, "-- minimise AST:"); zn = 0; @@ -707,7 +707,7 @@ main(int argc, char *argv[]) * This converts the tree of zones to a DAG. * TODO: Fix */ - if (0 && print != lx_print_h) { + if (0 && lang != LX_PRINT_H) { struct ast_zone *z, **p; int changed; unsigned int zn, zd, zp; @@ -801,7 +801,7 @@ main(int argc, char *argv[]) /* * Semantic checks. */ - if (print != lx_print_h) { + if (lang != LX_PRINT_H) { struct ast_zone *z; unsigned int zn; fsm_state_t i; @@ -910,7 +910,7 @@ main(int argc, char *argv[]) fprintf(stderr, "\n"); } - if (e && print != lx_print_dot) { + if (e && lang != LX_PRINT_DOT) { return EXIT_FAILURE; } } @@ -918,10 +918,10 @@ main(int argc, char *argv[]) /* XXX: can do this before semantic checks */ /* TODO: free ast */ /* TODO: free DFA ast_mappings, created in carryopaque, iff making a DFA. i.e. those which have non-NULL conflict sets */ - if (print == lx_print_h) { + if (lang == LX_PRINT_H) { /* TODO: special case to avoid overhead; free non-minimized NFA */ } - if (!keep_nfa && print != lx_print_h) { + if (!keep_nfa && lang != LX_PRINT_H) { struct ast_zone *z; struct ast_mapping *m; fsm_state_t i; @@ -951,9 +951,7 @@ main(int argc, char *argv[]) fprintf(stderr, "-- output:"); } - if (print != NULL) { - print(stdout, ast); - } + lx_print(stdout, ast, lang); if (print_progress) { fprintf(stderr, "\n"); diff --git a/src/lx/print.c b/src/lx/print.c new file mode 100644 index 000000000..6c65a52c0 --- /dev/null +++ b/src/lx/print.c @@ -0,0 +1,44 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include +#include + +#include "print.h" + +int +lx_print(FILE *f, const struct ast *ast, enum lx_print_lang lang) +{ + lx_print_f *print; + + assert(ast != NULL); + + if (lang == LX_PRINT_NONE) { + return 0; + } + + switch (lang) { + case LX_PRINT_C: print = lx_print_c; break; + case LX_PRINT_DOT: print = lx_print_dot; break; + case LX_PRINT_DUMP: print = lx_print_dump; break; + case LX_PRINT_H: print = lx_print_h; break; + case LX_PRINT_ZDOT: print = lx_print_zdot; break; + + default: + errno = ENOTSUP; + return -1; + } + + // TODO: return int + print(f, ast); + + return 0; +} + diff --git a/src/lx/print.h b/src/lx/print.h index 118ec208c..7452e86d7 100644 --- a/src/lx/print.h +++ b/src/lx/print.h @@ -42,19 +42,31 @@ enum api_exclude { API_BUF = 1 << 4 }; +enum lx_print_lang { + LX_PRINT_NONE, + LX_PRINT_C, + LX_PRINT_DOT, + LX_PRINT_DUMP, + LX_PRINT_H, + LX_PRINT_ZDOT +}; + extern enum api_tokbuf api_tokbuf; extern enum api_getc api_getc; extern enum api_exclude api_exclude; extern struct fsm_options opt; -typedef void (lx_print)(FILE *f, const struct ast *ast); +typedef void (lx_print_f)(FILE *f, const struct ast *ast); + +lx_print_f lx_print_c; +lx_print_f lx_print_dot; +lx_print_f lx_print_dump; +lx_print_f lx_print_h; +lx_print_f lx_print_zdot; -lx_print lx_print_c; -lx_print lx_print_h; -lx_print lx_print_dot; -lx_print lx_print_dump; -lx_print lx_print_zdot; +int +lx_print(FILE *f, const struct ast *ast, enum lx_print_lang lang); #endif diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index 5ad9d6212..00a84914b 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -205,7 +205,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) z->fsm->opt = &opt; - fsm_print_dot(f, z->fsm); + fsm_print(f, z->fsm, FSM_PRINT_DOT); z->fsm->opt = tmp; } diff --git a/src/re/main.c b/src/re/main.c index e164279f7..2fca4ba19 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -33,11 +33,10 @@ #include #include "libfsm/internal.h" /* XXX */ -#include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ #include "libre/ast.h" /* XXX */ #include "libre/ast_new_from_fsm.h" /* XXX */ - +#include "libre/print.h" /* XXX */ #define DEBUG_ESCAPES 0 #define DEBUG_VM_FSM 0 @@ -102,61 +101,75 @@ io(const char *name) exit(EXIT_FAILURE); } +/* TODO: centralise */ static void -print_name(const char *name, - fsm_print **print_fsm, ast_print **print_ast) +lang_name(const char *name, enum fsm_print_lang *fsm_lang, enum ast_print_lang *ast_lang) { size_t i; - struct { + const struct { const char *name; - fsm_print *print_fsm; - ast_print *print_ast; + enum fsm_print_lang lang; } a[] = { - { "api", fsm_print_api, NULL }, - { "awk", fsm_print_awk, NULL }, - { "c", fsm_print_c, NULL }, - { "dot", fsm_print_dot, NULL }, - { "fsm", fsm_print_fsm, NULL }, - { "ir", fsm_print_ir, NULL }, - { "irjson", fsm_print_irjson, NULL }, - { "json", fsm_print_json, NULL }, - { "vmc", fsm_print_vmc, NULL }, - { "vmdot", fsm_print_vmdot, NULL }, - { "rust", fsm_print_rust, NULL }, - { "llvm", fsm_print_llvm, NULL }, - { "sh", fsm_print_sh, NULL }, - { "go", fsm_print_go, NULL }, - - { "vmops_c", fsm_print_vmops_c, NULL }, - { "vmops_h", fsm_print_vmops_h, NULL }, - { "vmops_main", fsm_print_vmops_main, NULL }, - - { "amd64", fsm_print_vmasm, NULL }, - { "amd64_att", fsm_print_vmasm_amd64_att, NULL }, - { "amd64_nasm", fsm_print_vmasm_amd64_nasm, NULL }, - { "amd64_go", fsm_print_vmasm_amd64_go, NULL }, - - { "tree", NULL, ast_print_tree }, - { "abnf", NULL, ast_print_abnf }, - { "ast", NULL, ast_print_dot }, - { "pcre", NULL, ast_print_pcre } + { "amd64", FSM_PRINT_AMD64_NASM }, + { "amd64_att", FSM_PRINT_AMD64_ATT }, + { "amd64_go", FSM_PRINT_AMD64_GO }, + { "amd64_nasm", FSM_PRINT_AMD64_NASM }, + + { "api", FSM_PRINT_API }, + { "awk", FSM_PRINT_AWK }, + { "c", FSM_PRINT_C }, + { "dot", FSM_PRINT_DOT }, + { "fsm", FSM_PRINT_FSM }, + { "go", FSM_PRINT_GO }, + { "ir", FSM_PRINT_IR }, + { "irjson", FSM_PRINT_IRJSON }, + { "json", FSM_PRINT_JSON }, + { "llvm", FSM_PRINT_LLVM }, + { "rust", FSM_PRINT_RUST }, + { "sh", FSM_PRINT_SH }, + { "vmc", FSM_PRINT_VMC }, + + { "vmdot", FSM_PRINT_VMDOT }, + { "vmops_c", FSM_PRINT_VMOPS_C }, + { "vmops_h", FSM_PRINT_VMOPS_H }, + { "vmops_main", FSM_PRINT_VMOPS_MAIN } + }; + + const struct { + const char *name; + enum ast_print_lang lang; + } b[] = { + { "abnf", AST_PRINT_ABNF }, + { "ast", AST_PRINT_DOT }, + { "pcre", AST_PRINT_PCRE }, + { "tree", AST_PRINT_TREE } }; assert(name != NULL); - assert(print_fsm != NULL); - assert(print_ast != NULL); for (i = 0; i < sizeof a / sizeof *a; i++) { if (0 == strcmp(a[i].name, name)) { - *print_fsm = a[i].print_fsm; - *print_ast = a[i].print_ast; + *fsm_lang = a[i].lang; + return; + } + } + + for (i = 0; i < sizeof b / sizeof *b; i++) { + if (0 == strcmp(b[i].name, name)) { + *ast_lang = b[i].lang; return; } } fprintf(stderr, "unrecognised output language; valid languages are: "); + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + ", "); + } + for (i = 0; i < sizeof a / sizeof *a; i++) { fprintf(stderr, "%s%s", a[i].name, @@ -670,14 +683,15 @@ parse_flags(const char *arg, enum re_flags *flags) int main(int argc, char *argv[]) { + struct fsm *fsm; struct fsm *(*join)(struct fsm *, struct fsm *, struct fsm_combine_info *); int (*query)(const struct fsm *, const struct fsm *); - fsm_print *print_fsm; - ast_print *print_ast; + enum fsm_print_lang fsm_lang; + enum ast_print_lang ast_lang; enum re_dialect dialect; - struct fsm *fsm; enum re_flags flags; + size_t generate_bounds = 0; int xfiles, yfiles; int fsmfiles; int example; @@ -686,7 +700,6 @@ main(int argc, char *argv[]) int patterns; int ambig; int makevm; - size_t generate_bounds = 0; struct fsm_dfavm *vm; @@ -709,8 +722,8 @@ main(int argc, char *argv[]) patterns = 0; ambig = 0; makevm = 0; - print_fsm = NULL; - print_ast = NULL; + fsm_lang = FSM_PRINT_NONE; + ast_lang = AST_PRINT_NONE; query = NULL; join = fsm_union; dialect = RE_NATIVE; @@ -738,14 +751,14 @@ main(int argc, char *argv[]) break; case 'l': - print_name(optarg, &print_fsm, &print_ast); + lang_name(optarg, &fsm_lang, &ast_lang); break; case 'F': parse_flags(optarg, &flags); break; - case 'p': print_fsm = fsm_print_fsm; break; + case 'p': fsm_lang = FSM_PRINT_FSM; break; case 'q': query = comparison(optarg); break; case 'r': dialect = dialect_name(optarg); break; @@ -787,12 +800,12 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if (!!print_fsm + !!print_ast + example + isliteral + !!query > 1) { + if ((fsm_lang != FSM_PRINT_NONE) + (ast_lang != AST_PRINT_NONE) + example + isliteral + !!query > 1) { fprintf(stderr, "-m, -p, -q and -t are mutually exclusive\n"); return EXIT_FAILURE; } - if (!!print_fsm + !!print_ast + example + isliteral + !!query && xfiles) { + if ((fsm_lang != FSM_PRINT_NONE) + (ast_lang != AST_PRINT_NONE) + example + isliteral + !!query && xfiles) { fprintf(stderr, "-x applies only when executing\n"); return EXIT_FAILURE; } @@ -817,7 +830,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if (print_fsm == NULL) { + if (fsm_lang == FSM_PRINT_NONE) { keep_nfa = 0; } @@ -910,7 +923,7 @@ main(int argc, char *argv[]) } /* XXX: repetitive */ - if (print_ast != NULL) { + if (ast_lang != AST_PRINT_NONE) { struct ast *ast; struct re_err err; @@ -966,7 +979,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - print_ast(stdout, &opt, flags, ast); + ast_print(stdout, ast, &opt, flags, ast_lang); ast_free(ast); @@ -986,7 +999,7 @@ main(int argc, char *argv[]) { int i; - for (i = 0; i < argc - !(print_fsm || example || isliteral || !!query || argc <= 1); i++) { + for (i = 0; i < argc - !(fsm_lang != FSM_PRINT_NONE || example || isliteral || !!query || argc <= 1); i++) { struct re_err err; struct fsm *new, *q; @@ -1112,7 +1125,7 @@ main(int argc, char *argv[]) return EXIT_SUCCESS; } - if ((print_fsm || example) && argc > 0) { + if ((fsm_lang != FSM_PRINT_NONE || example) && argc > 0) { fprintf(stderr, "too many arguments\n"); return EXIT_FAILURE; } @@ -1180,7 +1193,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if (!patterns && example && print_fsm != fsm_print_c) { + if (!patterns && example && fsm_lang != FSM_PRINT_C) { if (!fsm_minimise(fsm)) { perror("fsm_minimise"); return EXIT_FAILURE; @@ -1230,27 +1243,43 @@ main(int argc, char *argv[]) return fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt); } - if (print_fsm != NULL) { + if (fsm_lang != FSM_PRINT_NONE) { /* TODO: print examples in comments for end states; * patterns in comments for the whole FSM */ - if (print_fsm == fsm_print_c || print_fsm == fsm_print_vmc) { + switch (fsm_lang) { + case FSM_PRINT_C: + case FSM_PRINT_VMC: opt.endleaf = endleaf_c; - } else if (print_fsm == fsm_print_rust) { + break; + + case FSM_PRINT_RUST: opt.endleaf = endleaf_rust; - } else if (print_fsm == fsm_print_llvm) { + break; + + case FSM_PRINT_LLVM: opt.endleaf = endleaf_llvm; - } else if (print_fsm == fsm_print_dot || print_fsm == fsm_print_vmdot) { + break; + + case FSM_PRINT_DOT: + case FSM_PRINT_VMDOT: opt.endleaf = patterns ? endleaf_dot : NULL; - } else if (print_fsm == fsm_print_json) { + break; + + case FSM_PRINT_JSON: opt.endleaf = patterns ? endleaf_json : NULL; + break; + + default: + opt.endleaf = NULL; + break; } - if (-1 == print_fsm(stdout, fsm)) { + if (-1 == fsm_print(stdout, fsm, fsm_lang)) { if (errno == ENOTSUP) { fprintf(stderr, "unsupported IO API\n"); } else { - perror("print_fsm"); + perror("fsm_print"); } exit(EXIT_FAILURE); } diff --git a/src/retest/main.c b/src/retest/main.c index a5309e6c3..ec60f5ca5 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -34,7 +34,6 @@ #include #include "libfsm/internal.h" /* XXX */ -#include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ #include "libre/ast.h" /* XXX */ @@ -1049,11 +1048,11 @@ process_test_file(const char *filename, #if DEBUG_VM_FSM fprintf(stderr, "FSM:\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fprintf(stderr, "---\n"); { FILE *f = fopen("dump.fsm", "w"); - fsm_print_fsm(f, fsm); + fsm_print(f, fsm, FSM_PRINT_FSM); fclose(f); } #endif /* DEBUG_VM_FSM */ diff --git a/src/retest/reperf.c b/src/retest/reperf.c index d722a93a4..c826391b5 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -31,7 +31,6 @@ #include #include "libfsm/internal.h" /* XXX */ -#include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ #include "libre/ast.h" /* XXX */ @@ -734,11 +733,11 @@ perf_case_run(struct perf_case *c, enum halt halt, #if DEBUG_VM_FSM fprintf(stderr, "FSM:\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fprintf(stderr, "---\n"); { FILE *f = fopen("dump.fsm", "w"); - fsm_print_fsm(f, fsm); + fsm_print(f, fsm, FSM_PRINT_FSM); fclose(f); } #endif /* DEBUG_VM_FSM */ diff --git a/src/retest/runner.c b/src/retest/runner.c index b45b6b7e0..d6b364d43 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -84,18 +84,18 @@ print(const struct fsm *fsm, enum implementation impl, int e; switch (impl) { - case IMPL_C: e = fsm_print_c(f, fsm); break; - case IMPL_RUST: e = fsm_print_rust(f, fsm); break; - case IMPL_LLVM: e = fsm_print_llvm(f, fsm); break; - case IMPL_VMC: e = fsm_print_vmc(f, fsm); break; - case IMPL_GOASM: e = fsm_print_vmasm_amd64_go(f, fsm); break; - case IMPL_VMASM: e = fsm_print_vmasm_amd64_att(f, fsm); break; - case IMPL_GO: e = fsm_print_go(f, fsm); break; + case IMPL_C: e = fsm_print(f, fsm, FSM_PRINT_C); break; + case IMPL_RUST: e = fsm_print(f, fsm, FSM_PRINT_RUST); break; + case IMPL_LLVM: e = fsm_print(f, fsm, FSM_PRINT_LLVM); break; + case IMPL_VMC: e = fsm_print(f, fsm, FSM_PRINT_VMC); break; + case IMPL_GOASM: e = fsm_print(f, fsm, FSM_PRINT_AMD64_GO); break; + case IMPL_VMASM: e = fsm_print(f, fsm, FSM_PRINT_AMD64_ATT); break; + case IMPL_GO: e = fsm_print(f, fsm, FSM_PRINT_GO); break; case IMPL_VMOPS: - e = fsm_print_vmops_h(f, fsm) - | fsm_print_vmops_c(f, fsm) - | fsm_print_vmops_main(f, fsm); + e = fsm_print(f, fsm, FSM_PRINT_VMOPS_H) + | fsm_print(f, fsm, FSM_PRINT_VMOPS_C) + | fsm_print(f, fsm, FSM_PRINT_VMOPS_MAIN); break; case IMPL_INTERPRET: diff --git a/tests/aho_corasick/actest.c b/tests/aho_corasick/actest.c index facf8209b..bf922fa0b 100644 --- a/tests/aho_corasick/actest.c +++ b/tests/aho_corasick/actest.c @@ -102,7 +102,7 @@ void wordlist_finalize(struct word_list *words) int main(int argc, char *argv[]) { struct fsm *fsm; char s[BUFSIZ]; - fsm_print *print; + enum fsm_print_lang lang; const char *pname = argv[0]; @@ -116,7 +116,7 @@ int main(int argc, char *argv[]) { opt.anonymous_states = 1; opt.consolidate_edges = 1; - print = fsm_print_fsm; + lang = FSM_PRINT_FSM; { int c; @@ -129,7 +129,7 @@ int main(int argc, char *argv[]) { case 'l': anchored_left = 1; break; case 'r': anchored_right = 1; break; - case 'd': print = fsm_print_dot; break; + case 'd': lang = FSM_PRINT_DOT; break; case 'h': case '?': @@ -173,9 +173,7 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } - if (print != NULL) { - print(stdout, fsm); - } + fsm_print(stdout, fsm, lang); fsm_free(fsm); diff --git a/tests/capture/captest.c b/tests/capture/captest.c index d1b5c510a..18c25dd08 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -80,7 +80,7 @@ captest_run_single(const struct captest_single_fsm_test_info *info) } #if CAPTEST_RUN_SINGLE_LOG - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fsm_capture_dump(stderr, "fsm", fsm); #endif diff --git a/tests/capture/capture3.c b/tests/capture/capture3.c index 9d4d284ab..919716e41 100644 --- a/tests/capture/capture3.c +++ b/tests/capture/capture3.c @@ -83,15 +83,15 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "\n=== f_ab...\n"); - fsm_print_fsm(stderr, f_ab); + fsm_print(stderr, f_ab, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_ab", f_ab); fprintf(stderr, "\n=== f_cde...\n"); - fsm_print_fsm(stderr, f_cde); + fsm_print(stderr, f_cde, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_cde", f_cde); fprintf(stderr, "\n=== f_fghi...\n"); - fsm_print_fsm(stderr, f_fghi); + fsm_print(stderr, f_fghi, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_fghi", f_fghi); #endif @@ -111,14 +111,14 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "=== unioned f_ab with f_cde... (CB ab: %u, cde: %u)\n", bases[0].capture, bases[1].capture); - fsm_print_fsm(stderr, f_all); + fsm_print(stderr, f_all, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_all", f_all); #endif #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "=== unioned f_all with f_fghi... (CB fghi: %u), %u captures\n", bases[2].capture, fsm_countcaptures(f_all)); - fsm_print_fsm(stderr, f_all); + fsm_print(stderr, f_all, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_all #2", f_all); #endif @@ -129,7 +129,7 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, f_all); + fsm_print(stderr, f_all, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_all", f_all); #endif @@ -164,7 +164,7 @@ det_and_min(const char *tag, struct fsm *fsm) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fsm_capture_dump(stderr, tag, fsm); #endif diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 5ba8c941c..c6d0a25ce 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -90,11 +90,11 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== abc \n"); - fsm_print_fsm(stderr, f_abc); + fsm_print(stderr, f_abc, FSM_PRINT_FSM); fsm_capture_dump(stderr, "abc", f_abc); fprintf(stderr, "==================== ab*c \n"); - fsm_print_fsm(stderr, f_ab_c); + fsm_print(stderr, f_ab_c, FSM_PRINT_FSM); fsm_capture_dump(stderr, "ab*c", f_ab_c); #endif @@ -110,7 +110,7 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== post-union \n"); - fsm_print_fsm(stderr, f_all); + fsm_print(stderr, f_all, FSM_PRINT_FSM); fsm_capture_dump(stderr, "capture_actions", f_all); fprintf(stderr, "====================\n"); #endif @@ -121,7 +121,7 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== post-det \n"); - fsm_print_fsm(stderr, f_all); + fsm_print(stderr, f_all, FSM_PRINT_FSM); fsm_capture_dump(stderr, "capture_actions", f_all); fprintf(stderr, "====================\n"); #endif @@ -144,7 +144,7 @@ det_and_min(const char *tag, struct fsm *fsm) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fsm_capture_dump(stderr, tag, fsm); #endif diff --git a/tests/capture/capture5.c b/tests/capture/capture5.c index b3a4be3ee..0da103114 100644 --- a/tests/capture/capture5.c +++ b/tests/capture/capture5.c @@ -77,7 +77,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== built\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fsm_capture_dump(stderr, "built", fsm); #endif @@ -88,7 +88,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fsm_capture_dump(stderr, "after det", fsm); #endif @@ -99,7 +99,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after min\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); fsm_capture_dump(stderr, "after min", fsm); #endif return fsm; diff --git a/tests/capture/capture_concat1.c b/tests/capture/capture_concat1.c index ee9c8aaab..084b666ea 100644 --- a/tests/capture/capture_concat1.c +++ b/tests/capture/capture_concat1.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after concat: cb_ab %u, cb_cde %u\n", *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); + fsm_print(stderr, abcde, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### after concat", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); + fsm_print(stderr, abcde, FSM_PRINT_FSM); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/capture/capture_concat2.c b/tests/capture/capture_concat2.c index a8f070c7e..4f8743244 100644 --- a/tests/capture/capture_concat2.c +++ b/tests/capture/capture_concat2.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after concat: cb_abc %u, cb_de %u\n", *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); + fsm_print(stderr, abcde, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### after concat", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); + fsm_print(stderr, abcde, FSM_PRINT_FSM); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/capture/capture_union1.c b/tests/capture/capture_union1.c index 5d9bd2920..704544997 100644 --- a/tests/capture/capture_union1.c +++ b/tests/capture/capture_union1.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after union: cb_ab %u, cb_cde %u\n", *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); + fsm_print(stderr, abcde, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### after union", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); + fsm_print(stderr, abcde, FSM_PRINT_FSM); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/gen/gen3.c b/tests/gen/gen3.c index 33cbcba66..56b158f41 100644 --- a/tests/gen/gen3.c +++ b/tests/gen/gen3.c @@ -61,7 +61,7 @@ build(void) } if (getenv("PRINT")) { - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); } return fsm; diff --git a/tests/minimise/minimise_test_case_list.c b/tests/minimise/minimise_test_case_list.c index 1386f0dcc..395004703 100644 --- a/tests/minimise/minimise_test_case_list.c +++ b/tests/minimise/minimise_test_case_list.c @@ -110,10 +110,10 @@ check_minimisation(const char *pattern) __func__, pattern, expected_state_count, state_count_min); fprintf(stderr, "== expected:\n"); - fsm_print_fsm(stderr, oracle_min); + fsm_print(stderr, oracle_min, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, FSM_PRINT_FSM); } fsm_free(oracle_min); diff --git a/theft/fuzz_capture_string_set.c b/theft/fuzz_capture_string_set.c index 7326356c2..31cb741b3 100644 --- a/theft/fuzz_capture_string_set.c +++ b/theft/fuzz_capture_string_set.c @@ -162,7 +162,7 @@ check_capstring_set(struct capture_env *env, if (verbosity > 2) { fprintf(stderr, "==== cs '%s'\n", cs->string); - fsm_print_fsm(stderr, dfa); + fsm_print(stderr, dfa, FSM_PRINT_FSM); } struct fsm *cp = fsm_clone(dfa); @@ -175,7 +175,7 @@ check_capstring_set(struct capture_env *env, const size_t cp_capture_count = fsm_countcaptures(cp); if (verbosity > 2) { fprintf(stderr, "==== min(det(cp))\n"); - fsm_print_fsm(stderr, cp); + fsm_print(stderr, cp, FSM_PRINT_FSM); fsm_capture_dump(stderr, "min(det(cp))", cp); fprintf(stderr, "capture_count: %lu\n", cp_capture_count); } @@ -209,7 +209,7 @@ check_capstring_set(struct capture_env *env, if (verbosity > 2) { fprintf(stderr, "==== combined, pre-det\n"); - fsm_print_fsm(stderr, combined); + fsm_print(stderr, combined, FSM_PRINT_FSM); fsm_capture_dump(stderr, "combined, pre-det", combined); } @@ -227,7 +227,7 @@ check_capstring_set(struct capture_env *env, if (verbosity > 2) { fprintf(stderr, "==== det(combined)\n"); - fsm_print_fsm(stderr, combined); + fsm_print(stderr, combined, FSM_PRINT_FSM); fsm_capture_dump(stderr, "det(combined)", combined); } diff --git a/theft/fuzz_nfa.c b/theft/fuzz_nfa.c index 6ff698c98..bdc22745d 100644 --- a/theft/fuzz_nfa.c +++ b/theft/fuzz_nfa.c @@ -109,7 +109,7 @@ nfa_operations_should_not_impact_matching(theft_seed seed) do { \ if (verbosity > 2) { \ fprintf(stderr, "==== %s ====\n", BANNER); \ - fsm_print_fsm(stderr, nfa); \ + fsm_print(stderr, nfa, FSM_PRINT_FSM); \ } \ } while(0) From 481af9fd03f61923217bfde992aa51266d696bfb Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 10 Jul 2024 11:00:55 +0100 Subject: [PATCH 199/373] Missing targets. --- examples/bm/Makefile | 3 +++ examples/rpn/Makefile | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/bm/Makefile b/examples/bm/Makefile index c0689f362..65c1b1ce2 100644 --- a/examples/bm/Makefile +++ b/examples/bm/Makefile @@ -10,3 +10,6 @@ pcre: pcre.c libfsm: libfsm.c gcc -o libfsm -O3 -Wall -std=c99 ${BM_CFLAGS} libfsm.c -I ../../include ../../build/lib/libre.a ../../build/lib/libfsm.a +clean: + rm -f pcre libfsm + diff --git a/examples/rpn/Makefile b/examples/rpn/Makefile index 698ee63d6..16018c35c 100644 --- a/examples/rpn/Makefile +++ b/examples/rpn/Makefile @@ -19,13 +19,13 @@ lexer.h: lexer.lx ${LX} -g fgetc -l h < lexer.lx > lexer.h lexer.o: lexer.c - ${CC} -c -DLX_HEADER='"lexer.h"' ${CFLAGS} ${.ALLSRC} + ${CC} -c -DLX_HEADER='"lexer.h"' ${CFLAGS} $> main.o: main.c - ${CC} -c ${CFLAGS} ${.ALLSRC} + ${CC} -c ${CFLAGS} $> rpn: lexer.o main.o - ${CC} -o ${.TARGET} ${.ALLSRC} + ${CC} -o ${.TARGET} $> lexer.c: lexer.h From 0adad698cfb714c4e8ba5bdfb3bc4e05e1f38770 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 10 Jul 2024 11:37:40 +0100 Subject: [PATCH 200/373] Hoist up ir and vm opcode compilation to fsm_print(). This cuts down a lot of repetition for the various print functions, and also helps make clear what they actually use. --- src/libfsm/print.c | 119 ++++++++++++++++++++++++++------- src/libfsm/print.h | 40 ++++++----- src/libfsm/print/Makefile | 8 +-- src/libfsm/print/api.c | 4 -- src/libfsm/print/awk.c | 120 ++++++++++++--------------------- src/libfsm/print/c.c | 74 ++++++++------------- src/libfsm/print/dot.c | 93 +++++++++++++------------- src/libfsm/print/fsm.c | 54 ++++++++------- src/libfsm/print/go.c | 122 ++++++++++++---------------------- src/libfsm/print/irdot.c | 32 ++++----- src/libfsm/print/irjson.c | 22 ++----- src/libfsm/print/json.c | 33 +++++----- src/libfsm/print/llvm.c | 110 ++++++++++--------------------- src/libfsm/print/rust.c | 135 ++++++++++++++------------------------ src/libfsm/print/sh.c | 68 ++++--------------- src/libfsm/print/vmasm.c | 81 +++++++---------------- src/libfsm/print/vmc.c | 112 +++++++++++-------------------- src/libfsm/print/vmdot.c | 113 +++++++++++-------------------- src/libfsm/print/vmops.c | 101 ++++++++++++---------------- 19 files changed, 589 insertions(+), 852 deletions(-) diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 2c99ac572..1f3a461ef 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -5,54 +5,125 @@ */ #include +#include #include #include #include #include +#include #include "print.h" +#include "internal.h" + +#include "vm/vm.h" +#include "print/ir.h" + int fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) { - int (*print)(FILE *f, const struct fsm *fsm); + struct dfavm_assembler_ir a; + struct ir *ir; + int r; + + static const struct dfavm_assembler_ir zero; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; + + fsm_print_f *print_fsm = NULL; + ir_print_f *print_ir = NULL; + vm_print_f *print_vm = NULL; + assert(f != NULL); assert(fsm != NULL); + assert(fsm->opt != NULL); if (lang == FSM_PRINT_NONE) { return 0; } switch (lang) { - case FSM_PRINT_AMD64_ATT: print = fsm_print_amd64_att; break; - case FSM_PRINT_AMD64_GO: print = fsm_print_amd64_go; break; - case FSM_PRINT_AMD64_NASM: print = fsm_print_amd64_nasm; break; - - case FSM_PRINT_API: print = fsm_print_api; break; - case FSM_PRINT_AWK: print = fsm_print_awk; break; - case FSM_PRINT_C: print = fsm_print_c; break; - case FSM_PRINT_DOT: print = fsm_print_dot; break; - case FSM_PRINT_FSM: print = fsm_print_fsm; break; - case FSM_PRINT_GO: print = fsm_print_go; break; - case FSM_PRINT_IR: print = fsm_print_ir; break; - case FSM_PRINT_IRJSON: print = fsm_print_irjson; break; - case FSM_PRINT_JSON: print = fsm_print_json; break; - case FSM_PRINT_LLVM: print = fsm_print_llvm; break; - case FSM_PRINT_RUST: print = fsm_print_rust; break; - case FSM_PRINT_SH: print = fsm_print_sh; break; - case FSM_PRINT_VMC: print = fsm_print_vmc; break; - case FSM_PRINT_VMDOT: print = fsm_print_vmdot; break; - - case FSM_PRINT_VMOPS_C: print = fsm_print_vmops_c; break; - case FSM_PRINT_VMOPS_H: print = fsm_print_vmops_h; break; - case FSM_PRINT_VMOPS_MAIN: print = fsm_print_vmops_main; break; + case FSM_PRINT_AMD64_ATT: print_vm = fsm_print_amd64_att; break; + case FSM_PRINT_AMD64_GO: print_vm = fsm_print_amd64_go; break; + case FSM_PRINT_AMD64_NASM: print_vm = fsm_print_amd64_nasm; break; + + case FSM_PRINT_API: print_fsm = fsm_print_api; break; + case FSM_PRINT_AWK: print_vm = fsm_print_awk; break; + case FSM_PRINT_C: print_ir = fsm_print_c; break; + case FSM_PRINT_DOT: print_ir = fsm_print_dot; break; + case FSM_PRINT_FSM: print_fsm = fsm_print_fsm; break; + case FSM_PRINT_GO: print_vm = fsm_print_go; break; + case FSM_PRINT_IR: print_ir = fsm_print_ir; break; + case FSM_PRINT_IRJSON: print_ir = fsm_print_irjson; break; + case FSM_PRINT_JSON: print_fsm = fsm_print_json; break; + case FSM_PRINT_LLVM: print_vm = fsm_print_llvm; break; + case FSM_PRINT_RUST: print_vm = fsm_print_rust; break; + case FSM_PRINT_SH: print_vm = fsm_print_sh; break; + case FSM_PRINT_VMC: print_vm = fsm_print_vmc; break; + case FSM_PRINT_VMDOT: print_vm = fsm_print_vmdot; break; + + case FSM_PRINT_VMOPS_C: print_vm = fsm_print_vmops_c; break; + case FSM_PRINT_VMOPS_H: print_vm = fsm_print_vmops_h; break; + case FSM_PRINT_VMOPS_MAIN: print_vm = fsm_print_vmops_main; break; default: errno = ENOTSUP; return -1; } - return print(f, fsm); + ir = NULL; + + if (print_fsm != NULL) { + r = print_fsm(f, fsm->opt, fsm); + goto done; + } + + ir = make_ir(fsm); + if (ir == NULL) { + return -1; + } + + if (print_ir != NULL) { + r = print_ir(f, fsm->opt, ir); + goto done; + } + + a = zero; + + /* TODO: non-const a */ + if (!dfavm_compile_ir(&a, ir, vm_opts)) { + free_ir(fsm, ir); + return -1; + } + + if (print_vm != NULL) { + /* + * TODO: I'm passing ir here for sake of emitting a state index by + * op->ir_state - ir_states for returning from the generated code. + * + * I want to remove this after moving the generated API to just + * return a boolean, and for callers to think in terms of endid + * sets instead. + */ + r = print_vm(f, fsm->opt, ir, a.linked); + } + + dfavm_opasm_finalize_op(&a); + +done: + + if (ir != NULL) { + free_ir(fsm, ir); + } + + if (ferror(f)) { + return -1; + } + + return r; } diff --git a/src/libfsm/print.h b/src/libfsm/print.h index a0c63e513..7c4a6a976 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -8,30 +8,34 @@ #define FSM_INTERNAL_PRINT_H struct fsm; +struct ir; +struct dfavm_op_ir; -typedef int fsm_print_f(FILE *f, const struct fsm *fsm); +typedef int fsm_print_f(FILE *f, const struct fsm_options *opt, const struct fsm *fsm); +typedef int ir_print_f(FILE *f, const struct fsm_options *opt, const struct ir *ir); +typedef int vm_print_f(FILE *f, const struct fsm_options *opt, const struct ir *ir, struct dfavm_op_ir *ops); -fsm_print_f fsm_print_amd64_att; -fsm_print_f fsm_print_amd64_go; -fsm_print_f fsm_print_amd64_nasm; +vm_print_f fsm_print_amd64_att; +vm_print_f fsm_print_amd64_go; +vm_print_f fsm_print_amd64_nasm; fsm_print_f fsm_print_api; -fsm_print_f fsm_print_awk; -fsm_print_f fsm_print_c; -fsm_print_f fsm_print_dot; +vm_print_f fsm_print_awk; +ir_print_f fsm_print_c; +ir_print_f fsm_print_dot; fsm_print_f fsm_print_fsm; -fsm_print_f fsm_print_go; -fsm_print_f fsm_print_ir; -fsm_print_f fsm_print_irjson; +vm_print_f fsm_print_go; +ir_print_f fsm_print_ir; +ir_print_f fsm_print_irjson; fsm_print_f fsm_print_json; -fsm_print_f fsm_print_llvm; -fsm_print_f fsm_print_rust; -fsm_print_f fsm_print_sh; -fsm_print_f fsm_print_vmc; +vm_print_f fsm_print_llvm; +vm_print_f fsm_print_rust; +vm_print_f fsm_print_sh; +vm_print_f fsm_print_vmc; -fsm_print_f fsm_print_vmdot; -fsm_print_f fsm_print_vmops_c; -fsm_print_f fsm_print_vmops_h; -fsm_print_f fsm_print_vmops_main; +vm_print_f fsm_print_vmdot; +vm_print_f fsm_print_vmops_c; +vm_print_f fsm_print_vmops_h; +vm_print_f fsm_print_vmops_main; #endif diff --git a/src/libfsm/print/Makefile b/src/libfsm/print/Makefile index f128a62d9..c0015c899 100644 --- a/src/libfsm/print/Makefile +++ b/src/libfsm/print/Makefile @@ -3,20 +3,20 @@ SRC += src/libfsm/print/api.c SRC += src/libfsm/print/awk.c SRC += src/libfsm/print/c.c -SRC += src/libfsm/print/vmops.c SRC += src/libfsm/print/dot.c SRC += src/libfsm/print/fsm.c +SRC += src/libfsm/print/go.c SRC += src/libfsm/print/ir.c SRC += src/libfsm/print/irdot.c SRC += src/libfsm/print/irjson.c SRC += src/libfsm/print/json.c -SRC += src/libfsm/print/rust.c SRC += src/libfsm/print/llvm.c +SRC += src/libfsm/print/rust.c SRC += src/libfsm/print/sh.c -SRC += src/libfsm/print/go.c +SRC += src/libfsm/print/vmasm.c SRC += src/libfsm/print/vmc.c SRC += src/libfsm/print/vmdot.c -SRC += src/libfsm/print/vmasm.c +SRC += src/libfsm/print/vmops.c .for src in ${SRC:Msrc/libfsm/print/*.c} CFLAGS.${src} += -I src # XXX: for internal.h diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index f25a1d97a..58de4572c 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -240,10 +240,6 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fsm_free(fsm); - if (ferror(f)) { - return -1; - } - return 0; error: diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index 462a3ab7f..eb62ec6a8 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -32,7 +32,7 @@ #define START UINT32_MAX static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -90,10 +90,8 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) + enum dfavm_op_end end_bits) { - (void) ir; - if (end_bits == VM_END_FAIL) { fprintf(f, "return -1"); return 0; @@ -144,29 +142,19 @@ print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) /* TODO: eventually to be non-static */ static int -fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, +fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, + struct dfavm_op_ir *ops, const char *cp, const char *prefix, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); assert(cp != NULL); assert(prefix != NULL); - a = zero; - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. * We'll need that in order to be able to use the leaf callback here. */ (void) leaf; @@ -175,10 +163,6 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -189,8 +173,8 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = START; - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops|| op->num_incoming > 0) { op->index = l++; } } @@ -198,9 +182,9 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " switch (l) {\n"); - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { - if (op != a.linked) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops|| op->num_incoming > 0) { + if (op != ops) { fprintf(f, "\n"); } @@ -222,7 +206,7 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { return -1; } fprintf(f, ";"); @@ -230,7 +214,7 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, case VM_OP_FETCH: { fprintf(f, "if (s == \"\") "); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, op->u.fetch.end_bits)) { return -1; } fprintf(f, "\n"); @@ -261,22 +245,44 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " }\n"); - dfavm_opasm_finalize_op(&a); - return 0; } -static int -fsm_print_awk_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *cp) +int +fsm_print_awk(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); + const char *prefix; + const char *cp; + assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); + + (void) ir; + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (opt->leaf != NULL) { + leaf = opt->leaf; + } else { + leaf = print_leaf; + } + + if (opt->cp != NULL) { + cp = opt->cp; + } else { + cp = "c"; /* XXX */ + } if (opt->fragment) { - if (-1 == fsm_print_awkfrag(f, ir, opt, cp, prefix, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix, + leaf, opt->leaf_opaque)) { return -1; } @@ -299,8 +305,8 @@ fsm_print_awk_complete(FILE *f, const struct ir *ir, fprintf(f, ", l, c) {\n"); - if (-1 == fsm_print_awkfrag(f, ir, opt, cp, prefix, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix, + leaf, opt->leaf_opaque)) { return -1; } @@ -309,46 +315,6 @@ fsm_print_awk_complete(FILE *f, const struct ir *ir, fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_awk(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *cp; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - if (fsm->opt->cp != NULL) { - cp = fsm->opt->cp; - } else { - cp = "c"; /* XXX */ - } - - r = fsm_print_awk_complete(f, ir, fsm->opt, prefix, cp); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 00e71adb1..a25ce42d0 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -53,7 +53,7 @@ ir_hasend(const struct ir *ir) } static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -134,7 +134,7 @@ print_groups(FILE *f, const struct fsm_options *opt, } static int -print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, +print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, struct ir_state *cs, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), @@ -337,7 +337,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } fprintf(f, "\n"); - if (-1 == print_singlecase(f, ir, opt, cp, &ir->states[i], leaf, leaf_opaque)) { + if (-1 == print_case(f, ir, opt, cp, &ir->states[i], leaf, leaf_opaque)) { return -1; } @@ -355,7 +355,9 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } static int -fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) +fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt, + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), + const void *leaf_opaque) { const char *cp; @@ -396,7 +398,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) } if (-1 == fsm_print_cfrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + leaf, leaf_opaque)) { return -1; } @@ -412,16 +414,31 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) return 0; } -static int -fsm_print_c_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix) +int +fsm_print_c(FILE *f, const struct fsm_options *opt, + const struct ir *ir) { + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); + const char *prefix; + assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (opt->leaf != NULL) { + leaf = opt->leaf; + } else { + leaf = print_leaf; + } if (opt->fragment) { - if (-1 == fsm_print_c_body(f, ir, opt)) { + if (-1 == fsm_print_c_body(f, ir, opt, leaf, opt->leaf_opaque)) { return -1; } } else { @@ -461,7 +478,7 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, if (ir->n == 0) { fprintf(f, "\treturn -1; /* no matches */\n"); } else { - if (-1 == fsm_print_c_body(f, ir, opt)) { + if (-1 == fsm_print_c_body(f, ir, opt, leaf, opt->leaf_opaque)) { return -1; } } @@ -470,41 +487,6 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_c(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - r = fsm_print_c_complete(f, ir, fsm->opt, prefix); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 54d151a96..892287747 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -25,7 +25,8 @@ #include static int -singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) +print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, + const char *prefix, fsm_state_t s) { struct fsm_edge e; struct edge_iter it; @@ -33,21 +34,20 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) struct state_set *unique; assert(f != NULL); + assert(opt != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(prefix != NULL); assert(s < fsm->statecount); - if (!fsm->opt->consolidate_edges) { + if (!opt->consolidate_edges) { { struct state_iter jt; fsm_state_t st; for (state_set_reset(fsm->states[s].epsilons, &jt); state_set_next(&jt, &st); ) { fprintf(f, "\t%sS%-2u -> %sS%-2u [ label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - st); + prefix, s, + prefix, st); fprintf(f, "ε"); @@ -57,12 +57,10 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) for (edge_set_ordered_iter_reset(fsm->states[s].edges, &eoi); edge_set_ordered_iter_next(&eoi, &e); ) { fprintf(f, "\t%sS%-2u -> %sS%-2u [ label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - e.state); + prefix, s, + prefix, e.state); - dot_escputc_html(f, fsm->opt, (char) e.symbol); + dot_escputc_html(f, opt, (char) e.symbol); fprintf(f, "> ];\n"); } @@ -73,7 +71,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) unique = NULL; for (edge_set_reset(fsm->states[s].edges, &it); edge_set_next(&it, &e); ) { - if (!state_set_add(&unique, fsm->opt->alloc, e.state)) { + if (!state_set_add(&unique, opt->alloc, e.state)) { return -1; } } @@ -109,17 +107,15 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) } fprintf(f, "\t%sS%-2u -> %sS%-2u [ ", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - e.state); + prefix, s, + prefix, e.state); if (bm_count(&bm) > 4) { fprintf(f, "weight = 3, "); } fprintf(f, "label = <"); - bm_print(f, fsm->opt, &bm, 0, dot_escputc_html); + bm_print(f, opt, &bm, 0, dot_escputc_html); fprintf(f, "> ];\n"); } @@ -132,10 +128,8 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) for (state_set_reset(fsm->states[s].epsilons, &jt); state_set_next(&jt, &st); ) { fprintf(f, "\t%sS%-2u -> %sS%-2u [ weight = 1, label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - st); + prefix, s, + prefix, st); fprintf(f, "ε"); @@ -151,20 +145,21 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) } static int -print_dotfrag(FILE *f, const struct fsm *fsm) +print_dotfrag(FILE *f, const struct fsm_options *opt, + const struct fsm *fsm, const char *prefix) { fsm_state_t s; assert(f != NULL); + assert(opt != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(prefix != NULL); for (s = 0; s < fsm->statecount; s++) { if (!fsm_isend(fsm, s)) { - if (!fsm->opt->anonymous_states) { + if (!opt->anonymous_states) { fprintf(f, "\t%sS%-2u [ label = <%u> ];\n", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, s); + prefix, s, s); } } else { fsm_end_id_t *ids; @@ -175,7 +170,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) if (count > 0) { int res; - ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + ids = f_malloc(opt->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } @@ -185,7 +180,7 @@ print_dotfrag(FILE *f, const struct fsm *fsm) } fprintf(f, "\t%sS%-2u [ shape = doublecircle", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", s); + prefix, s); assert(f != NULL); @@ -193,18 +188,18 @@ print_dotfrag(FILE *f, const struct fsm *fsm) fprintf(f, "label = <"); - if (fsm->opt->endleaf != NULL) { - if (-1 == fsm->opt->endleaf(f, + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, ids, count, - fsm->opt->endleaf_opaque)) + opt->endleaf_opaque)) { return -1; } - } else if (!fsm->opt->anonymous_states) { + } else if (!opt->anonymous_states) { fprintf(f, "%u", s); } - if (fsm->opt->endleaf != NULL || !fsm->opt->anonymous_states) { + if (opt->endleaf != NULL || !opt->anonymous_states) { if (count > 0) { fprintf(f, "
"); } @@ -222,12 +217,12 @@ print_dotfrag(FILE *f, const struct fsm *fsm) fprintf(f, " ];\n"); - f_free(fsm->opt->alloc, ids); + f_free(opt->alloc, ids); } /* TODO: show example here, unless !opt->comments */ - if (-1 == singlestate(f, fsm, s)) { + if (-1 == print_state(f, opt, fsm, prefix, s)) { return -1; } } @@ -236,14 +231,22 @@ print_dotfrag(FILE *f, const struct fsm *fsm) } int -fsm_print_dot(FILE *f, const struct fsm *fsm) +fsm_print_dot(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) { + const char *prefix; + assert(f != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = ""; + } - if (fsm->opt->fragment) { - if (-1 == print_dotfrag(f, fsm)) { + if (opt->fragment) { + if (-1 == print_dotfrag(f, opt, fsm, prefix)) { return -1; } } else { @@ -255,7 +258,7 @@ fsm_print_dot(FILE *f, const struct fsm *fsm) fprintf(f, "\tnode [ shape = circle ];\n"); fprintf(f, "\tedge [ weight = 2 ];\n"); - if (fsm->opt->anonymous_states) { + if (opt->anonymous_states) { fprintf(f, "\tnode [ label = \"\", width = 0.3 ];\n"); } @@ -266,13 +269,13 @@ fsm_print_dot(FILE *f, const struct fsm *fsm) if (fsm_getstart(fsm, &start)) { fprintf(f, "\tstart -> %sS%u;\n", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", + opt->prefix != NULL ? opt->prefix : "", start); fprintf(f, "\n"); } - if (-1 == print_dotfrag(f, fsm)) { + if (-1 == print_dotfrag(f, opt, fsm, prefix)) { return -1; } @@ -280,10 +283,6 @@ fsm_print_dot(FILE *f, const struct fsm *fsm) fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index a3cb471cf..6bd441882 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -88,10 +88,15 @@ findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) } static int -print_state_comments(FILE *f, const struct fsm *fsm, fsm_state_t dst) +print_state_comments(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, + fsm_state_t dst) { fsm_state_t start; + assert(f != NULL); + assert(opt != NULL); + assert(fsm != NULL); + if (!fsm_getstart(fsm, &start)) { return 0; } @@ -109,7 +114,7 @@ print_state_comments(FILE *f, const struct fsm *fsm, fsm_state_t dst) if (n > 0) { fprintf(f, " # e.g. \""); - escputs(f, fsm->opt, fsm_escputc, buf); + escputs(f, opt, fsm_escputc, buf); fprintf(f, "%s\"", n >= (int) sizeof buf - 1 ? "..." : ""); } @@ -121,6 +126,9 @@ print_state_comments(FILE *f, const struct fsm *fsm, fsm_state_t dst) static void print_char_range(FILE *f, const struct fsm_options *opt, char lower, char upper) { + assert(f != NULL); + assert(opt != NULL); + if (lower == upper) { fprintf(f, "\""); fsm_escputc(f, opt, (char) lower); @@ -135,8 +143,12 @@ print_char_range(FILE *f, const struct fsm_options *opt, char lower, char upper) } static int -print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) +print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, + fsm_state_t s) { + assert(f != NULL); + assert(opt != NULL); + { struct state_iter jt; fsm_state_t st; @@ -156,7 +168,7 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) } assert(s < fsm->statecount); - if (fsm->opt->group_edges) { + if (opt->group_edges) { struct edge_group_iter egi; struct edge_group_iter_info info; @@ -180,7 +192,7 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) fprintf(f, ", "); } - print_char_range(f, fsm->opt, (char) lower, (char) i - 1); + print_char_range(f, opt, (char) lower, (char) i - 1); ranges++; } lower = 256; @@ -191,13 +203,13 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) fprintf(f, ", "); } - print_char_range(f, fsm->opt, (char) lower, (char) 255); + print_char_range(f, opt, (char) lower, (char) 255); } fprintf(f, ";"); - if (fsm->opt->comments) { - if (-1 == print_state_comments(f, fsm, info.to)) { + if (opt->comments) { + if (-1 == print_state_comments(f, opt, fsm, info.to)) { return -1; } } @@ -215,13 +227,13 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) fprintf(f, "%-2u -> %2u", s, e.state); fprintf(f, " \""); - fsm_escputc(f, fsm->opt, (char) e.symbol); + fsm_escputc(f, opt, (char) e.symbol); putc('\"', f); fprintf(f, ";"); - if (fsm->opt->comments) { - if (-1 == print_state_comments(f, fsm, e.state)) { + if (opt->comments) { + if (-1 == print_state_comments(f, opt, fsm, e.state)) { return -1; } } @@ -234,7 +246,7 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) } int -fsm_print_fsm(FILE *f, const struct fsm *fsm) +fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) { fsm_state_t s, start; size_t end; @@ -242,7 +254,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) assert(f != NULL); assert(fsm != NULL); - if (!fsm->opt->anonymous_states) { + if (!opt->anonymous_states) { /* * States are output in order here so as to force ordering when * parsing the .fsm format and creating new states. This ensures @@ -257,7 +269,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) } for (s = 0; s < fsm->statecount; s++) { - if (-1 == print_state(f, fsm, s)) { + if (-1 == print_state(f, opt, fsm, s)) { return -1; } } @@ -291,7 +303,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) if (count > 0) { int res; - ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + ids = f_malloc(opt->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } @@ -300,10 +312,10 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) assert(res == 1); } - if (fsm->opt->endleaf != NULL) { - if (-1 == fsm->opt->endleaf(f, + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, ids, count, - fsm->opt->endleaf_opaque)) + opt->endleaf_opaque)) { return -1; } @@ -324,7 +336,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) fprintf(f, "]"); - f_free(fsm->opt->alloc, ids); + f_free(opt->alloc, ids); } fprintf(f, "%s", end > 0 ? ", " : ";\n"); @@ -332,10 +344,6 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) done: - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 7b3616f9d..63fb30ba0 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -29,7 +29,7 @@ #include "ir.h" static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -91,7 +91,8 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) + const struct ir_state *ir_states, + enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { fprintf(f, "{\n\t\treturn -1\n\t}\n"); @@ -109,7 +110,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return %td", op->ir_state - ir->states); + fprintf(f, "return %td", op->ir_state - ir_states); } fprintf(f, "\n\t}\n"); @@ -139,28 +140,19 @@ print_fetch(FILE *f, const struct fsm_options *opt) /* TODO: eventually to be non-static */ static int -fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, +fsm_print_gofrag(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); assert(cp != NULL); - a = zero; - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. * We'll need that in order to be able to use the leaf callback here. */ (void) leaf; @@ -169,10 +161,6 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -183,7 +171,7 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = 0; - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { op->index = l++; } @@ -193,8 +181,8 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* * Only declare variables if we're actually going to use them. */ - if (a.linked->cmp == VM_CMP_ALWAYS && a.linked->instr == VM_OP_STOP) { - assert(a.linked->next == NULL); + if (ops->cmp == VM_CMP_ALWAYS && ops->instr == VM_OP_STOP) { + assert(ops->next == NULL); } else { switch (opt->io) { case FSM_IO_PAIR: @@ -219,7 +207,7 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } } - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { print_label(f, op, opt); fprintf(f, "\n"); @@ -230,12 +218,12 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - print_end(f, op, opt, op->u.stop.end_bits, ir); + print_end(f, op, opt, ir->states, op->u.stop.end_bits); break; case VM_OP_FETCH: print_fetch(f, opt); - print_end(f, op, opt, op->u.fetch.end_bits, ir); + print_end(f, op, opt, ir->states, op->u.fetch.end_bits); break; case VM_OP_BRANCH: @@ -251,27 +239,45 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\n"); } - dfavm_opasm_finalize_op(&a); - return 0; } -static int -fsm_print_go_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *package_prefix) +int +fsm_print_go(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); + const char *prefix; + const char *package_prefix; + /* TODO: currently unused, but must be non-NULL */ const char *cp = ""; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - assert(prefix != NULL); - assert(package_prefix != NULL); + assert(ir != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (opt->leaf != NULL) { + leaf = opt->leaf; + } else { + leaf = print_leaf; + } + + if (opt->package_prefix != NULL) { + package_prefix = opt->package_prefix; + } else { + package_prefix = prefix; + } if (opt->fragment) { - if (-1 == fsm_print_gofrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + if (-1 == fsm_print_gofrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque)) { return -1; } @@ -295,8 +301,8 @@ fsm_print_go_complete(FILE *f, const struct ir *ir, return -1; } - if (-1 == fsm_print_gofrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + if (-1 == fsm_print_gofrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque)) { return -1; } @@ -304,48 +310,6 @@ fsm_print_go_complete(FILE *f, const struct ir *ir, fprintf(f, "}\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_go(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *package_prefix; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - if (fsm->opt->package_prefix != NULL) { - package_prefix = fsm->opt->package_prefix; - } else { - package_prefix = prefix; - } - - r = fsm_print_go_complete(f, ir, fsm->opt, prefix, package_prefix); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 8e91029ca..ac6a6f24a 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -61,7 +61,7 @@ print_endpoint(FILE *f, const struct fsm_options *opt, unsigned char c) } static void -print_state(FILE *f, unsigned to, unsigned self) +print_name(FILE *f, unsigned to, unsigned self) { assert(f != NULL); @@ -144,7 +144,7 @@ print_grouprows(FILE *f, const struct fsm_options *opt, } else { fprintf(f, ""); } @@ -174,7 +174,7 @@ print_grouplinks(FILE *f, unsigned self, } static int -print_cs(FILE *f, const struct fsm_options *opt, +print_state(FILE *f, const struct fsm_options *opt, const struct ir *ir, const struct ir_state *cs) { assert(f != NULL); @@ -232,7 +232,7 @@ print_cs(FILE *f, const struct fsm_options *opt, case IR_SAME: fprintf(f, "\t\t \n"); break; @@ -246,14 +246,14 @@ print_cs(FILE *f, const struct fsm_options *opt, case IR_DOMINANT: fprintf(f, "\t\t \n"); print_grouprows(f, opt, ir_indexof(ir, cs), cs->u.dominant.groups, cs->u.dominant.n); break; case IR_ERROR: fprintf(f, "\t\t \n"); print_errorrows(f, opt, &cs->u.error.error); print_grouprows(f, opt, ir_indexof(ir, cs), cs->u.error.groups, cs->u.error.n); @@ -322,18 +322,14 @@ print_cs(FILE *f, const struct fsm_options *opt, } int -fsm_print_ir(FILE *f, const struct fsm *fsm) +fsm_print_ir(FILE *f, const struct fsm_options *opt, + const struct ir *ir) { - struct ir *ir; size_t i; assert(f != NULL); - assert(fsm != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } + assert(opt != NULL); + assert(ir != NULL); fprintf(f, "digraph G {\n"); @@ -349,19 +345,13 @@ fsm_print_ir(FILE *f, const struct fsm *fsm) } for (i = 0; i < ir->n; i++) { - if (-1 == print_cs(f, fsm->opt, ir, &ir->states[i])) { + if (-1 == print_state(f, opt, ir, &ir->states[i])) { return -1; } } fprintf(f, "}\n"); - free_ir(fsm, ir); - - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 189d4b571..1848e5102 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -113,7 +113,7 @@ print_groups(FILE *f, const struct fsm_options *opt, } static int -print_cs(FILE *f, const struct fsm_options *opt, +print_state(FILE *f, const struct fsm_options *opt, const struct ir_state *cs) { assert(f != NULL); @@ -212,18 +212,14 @@ print_cs(FILE *f, const struct fsm_options *opt, } int -fsm_print_irjson(FILE *f, const struct fsm *fsm) +fsm_print_irjson(FILE *f, const struct fsm_options *opt, + const struct ir *ir) { - struct ir *ir; size_t i; assert(f != NULL); - assert(fsm != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } + assert(opt != NULL); + assert(ir != NULL); fprintf(f, "{\n"); @@ -231,7 +227,7 @@ fsm_print_irjson(FILE *f, const struct fsm *fsm) fprintf(f, "\t\"states\": [\n"); for (i = 0; i < ir->n; i++) { - if (-1 == print_cs(f, fsm->opt, &ir->states[i])) { + if (-1 == print_state(f, opt, &ir->states[i])) { return -1; } @@ -245,12 +241,6 @@ fsm_print_irjson(FILE *f, const struct fsm *fsm) fprintf(f, "}\n"); - free_ir(fsm, ir); - - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 11c7a5f24..9e4f0172d 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -114,7 +114,8 @@ print_edge_label(FILE *f, int *notfirst, } static int -singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) +print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, + fsm_state_t s, int *notfirst) { struct fsm_edge e; struct edge_iter it; @@ -122,12 +123,12 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) struct state_set *unique; assert(f != NULL); + assert(opt != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); assert(s < fsm->statecount); assert(notfirst != NULL); - if (!fsm->opt->consolidate_edges) { + if (!opt->consolidate_edges) { struct state_iter jt; fsm_state_t st; @@ -136,7 +137,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) } for (edge_set_ordered_iter_reset(fsm->states[s].edges, &eoi); edge_set_ordered_iter_next(&eoi, &e); ) { - print_edge_symbol(f, notfirst, fsm->opt, + print_edge_symbol(f, notfirst, opt, s, e.state, e.symbol); } @@ -146,7 +147,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) unique = NULL; for (edge_set_reset(fsm->states[s].edges, &it); edge_set_next(&it, &e); ) { - if (!state_set_add(&unique, fsm->opt->alloc, e.state)) { + if (!state_set_add(&unique, opt->alloc, e.state)) { return -1; } } @@ -181,7 +182,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) } } - print_edge_bitmap(f, notfirst, fsm->opt, s, e.state, &bm); + print_edge_bitmap(f, notfirst, opt, s, e.state, &bm); } state_set_free(unique); @@ -202,14 +203,14 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) } int -fsm_print_json(FILE *f, const struct fsm *fsm) +fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) { fsm_state_t start; fsm_state_t i; assert(f != NULL); + assert(opt != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); fprintf(f, "{\n"); @@ -240,7 +241,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) } /* showing endleaf in addition to existing content */ - if (fsm->opt->endleaf != NULL) { + if (opt->endleaf != NULL) { int notfirst; notfirst = 0; @@ -257,7 +258,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) count = fsm_endid_count(fsm, i); - ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + ids = f_malloc(opt->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } @@ -271,16 +272,16 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, "{ %u, ", i); - if (-1 == fsm->opt->endleaf(f, + if (-1 == opt->endleaf(f, ids, count, - fsm->opt->endleaf_opaque)) + opt->endleaf_opaque)) { return -1; } fprintf(f, " }"); - f_free(fsm->opt->alloc, ids); + f_free(opt->alloc, ids); notfirst = 1; } @@ -294,7 +295,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " \"edges\": [\n"); for (i = 0; i < fsm->statecount; i++) { - if (-1 == singlestate(f, fsm, i, ¬first)) { + if (-1 == print_state(f, opt, fsm, i, ¬first)) { return -1; } } @@ -304,10 +305,6 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, "}\n"); fprintf(f, "\n"); - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 90828ce95..bbc61d97f 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -69,7 +69,7 @@ decl(unsigned *n) } static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -158,7 +158,7 @@ print_target(FILE *f, const struct dfavm_op_ir *op, } static void -print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_cond(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *op, struct frame *frame) { assert(frame != NULL); @@ -455,8 +455,8 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) /* TODO: eventually to be non-static */ static int -fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, - const struct fsm_options *opt, const struct ir *ir, +fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -465,8 +465,8 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, struct dfavm_op_ir *op; assert(f != NULL); - assert(a != NULL); assert(opt != NULL); + assert(ir != NULL); assert(cp != NULL); /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. @@ -482,14 +482,14 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, l = 0; - for (op = a->linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { op->index = l++; } } { retlist.count = 0; - build_retlist(&retlist, a->linked); + build_retlist(&retlist, ops); /* sort for both dedup and bsearch */ qsort(retlist.a, retlist.count, sizeof *retlist.a, @@ -520,7 +520,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, assert(retlist.count > 0); } - print_jump(f, a->linked); + print_jump(f, ops); /* * We're jumping to ret*: labels, and having each jump @@ -591,7 +591,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, } struct frame frame = { 0, 0, 0 }; - for (op = a->linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->instr != VM_OP_STOP || op->cmp != VM_CMP_ALWAYS || op->u.stop.end_bits != VM_END_FAIL) { print_label(f, true, "l%" PRIu32, op->index); } @@ -612,7 +612,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, unsigned b = decl(&frame.b); // TODO: our ret%u: label goes in place of t%u here, which means we're replacing the NULL - print_cond(f, op, opt, &frame); + print_cond(f, opt, op, &frame); print_branch(f, &frame, op->u.stop.end_bits == VM_END_FAIL ? &fail : NULL, op->next); @@ -663,7 +663,7 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->cmp == VM_CMP_ALWAYS) { print_jump(f, dest); } else { - print_cond(f, op, opt, &frame); + print_cond(f, opt, op, &frame); print_branch(f, &frame, dest, op->next); @@ -684,32 +684,38 @@ fsm_print_llvmfrag(FILE *f, const struct dfavm_assembler_ir *a, return 0; } -static int -fsm_print_llvm_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *cp) +int +fsm_print_llvm(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); + const char *prefix; + const char *cp; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - a = zero; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (opt->leaf != NULL) { + leaf = opt->leaf; + } else { + leaf = print_leaf; + } - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; + if (opt->cp != NULL) { + cp = opt->cp; + } else { + cp = "c"; /* XXX */ } if (opt->fragment) { - fsm_print_llvmfrag(f, &a, opt, ir, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + fsm_print_llvmfrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque); goto error; } @@ -765,60 +771,16 @@ fsm_print_llvm_complete(FILE *f, const struct ir *ir, exit(EXIT_FAILURE); } - fsm_print_llvmfrag(f, &a, opt, ir, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + fsm_print_llvmfrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque); fprintf(f, "}\n"); fprintf(f, "\n"); - dfavm_opasm_finalize_op(&a); - - if (ferror(f)) { - return -1; - } - return 0; error: - dfavm_opasm_finalize_op(&a); - return -1; } -int -fsm_print_llvm(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *cp; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - if (fsm->opt->cp != NULL) { - cp = fsm->opt->cp; - } else { - cp = "c"; /* XXX */ - } - - r = fsm_print_llvm_complete(f, ir, fsm->opt, prefix, cp); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index af2b0f302..0adb8e71f 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -32,7 +32,7 @@ #define START UINT32_MAX static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -67,9 +67,11 @@ cmp_operator(int cmp) } static int -has_op(const struct dfavm_op_ir *op, enum dfavm_op_instr instr) +has_op(const struct dfavm_op_ir *ops, enum dfavm_op_instr instr) { - for ( ; op != NULL; op = op->next) { + const struct dfavm_op_ir *op; + + for (op = ops; op != NULL; op = op->next) { if (op->instr == instr) { return 1; } @@ -102,7 +104,8 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) + const struct ir_state *ir_states, + enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { fprintf(f, "return None"); @@ -117,7 +120,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return Some(%td)", op->ir_state - ir->states); + fprintf(f, "return Some(%td)", op->ir_state - ir_states); } return 0; @@ -145,8 +148,8 @@ print_fetch(FILE *f) /* TODO: eventually to be non-static */ static int -fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, - const struct ir *ir, const struct fsm_options *opt, +fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -155,8 +158,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, bool fallthrough; assert(f != NULL); - assert(a != NULL); assert(opt != NULL); + assert(ir != NULL); assert(cp != NULL); /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. @@ -177,8 +180,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, l = START; - for (op = a->linked; op != NULL; op = op->next) { - if (op == a->linked || op->num_incoming > 0) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops || op->num_incoming > 0) { op->index = l++; } } @@ -187,8 +190,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, /* * Only declare variables if we're actually going to use them. */ - if (a->linked->cmp == VM_CMP_ALWAYS && a->linked->instr == VM_OP_STOP) { - assert(a->linked->next == NULL); + if (ops->cmp == VM_CMP_ALWAYS && ops->instr == VM_OP_STOP) { + assert(ops->next == NULL); fprintf(f, "\n"); } else { switch (opt->io) { @@ -212,8 +215,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, } fprintf(f, " pub enum Label {\n "); - for (op = a->linked; op != NULL; op = op->next) { - if (op == a->linked || op->num_incoming > 0) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops || op->num_incoming > 0) { fprintf(f, " "); print_label(f, op); fprintf(f, ","); @@ -223,7 +226,7 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, fprintf(f, " }\n"); fprintf(f, "\n"); - fprintf(f, " let %sl = Ls;\n", has_op(a->linked, VM_OP_BRANCH) ? "mut " : ""); + fprintf(f, " let %sl = Ls;\n", has_op(ops, VM_OP_BRANCH) ? "mut " : ""); fprintf(f, "\n"); fprintf(f, " loop {\n"); @@ -231,9 +234,9 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, fallthrough = true; - for (op = a->linked; op != NULL; op = op->next) { - if (op == a->linked || op->num_incoming > 0) { - if (op != a->linked) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops || op->num_incoming > 0) { + if (op != ops) { if (fallthrough) { fprintf(f, " "); print_jump(f, op); @@ -268,7 +271,7 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, "{ "); } - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, op->u.stop.end_bits)) { return -1; } fprintf(f, ";"); @@ -317,7 +320,7 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, fprintf(f, " "); fprintf(f, "None => "); - print_end(f, op, opt, op->u.fetch.end_bits, ir); + print_end(f, op, opt, ir->states, op->u.fetch.end_bits); fprintf(f, ",\n"); fprintf(f, " "); @@ -361,32 +364,38 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, return 0; } -static int -fsm_print_rust_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *cp) +int +fsm_print_rust(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); + const char *prefix; + const char *cp; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - a = zero; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; + if (opt->leaf != NULL) { + leaf = opt->leaf; + } else { + leaf = print_leaf; + } + + if (opt->cp != NULL) { + cp = opt->cp; + } else { + cp = "c"; /* XXX */ } if (opt->fragment) { - fsm_print_rustfrag(f, &a, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + fsm_print_rustfrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque); goto error; } @@ -404,14 +413,14 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, case FSM_IO_STR: /* e.g. dbg!(fsm_main("xabces")); */ fprintf(f, "(%sinput: &str) -> Option {\n", - has_op(a.linked, VM_OP_FETCH) ? "" : "_"); + has_op(ops, VM_OP_FETCH) ? "" : "_"); fprintf(f, " use Label::*;\n"); break; case FSM_IO_PAIR: /* e.g. dbg!(fsm_main("xabces".as_bytes())); */ fprintf(f, "(%sinput: &[u8]) -> Option {\n", - has_op(a.linked, VM_OP_FETCH) ? "" : "_"); + has_op(ops, VM_OP_FETCH) ? "" : "_"); fprintf(f, " use Label::*;\n"); break; @@ -420,60 +429,16 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, &a, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + fsm_print_rustfrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque); fprintf(f, "}\n"); fprintf(f, "\n"); - dfavm_opasm_finalize_op(&a); - - if (ferror(f)) { - return -1; - } - return 0; error: - dfavm_opasm_finalize_op(&a); - return -1; } -int -fsm_print_rust(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *cp; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - if (fsm->opt->cp != NULL) { - cp = fsm->opt->cp; - } else { - cp = "c"; /* XXX */ - } - - r = fsm_print_rust_complete(f, ir, fsm->opt, prefix, cp); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 04e84663e..21a8b6105 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -119,7 +119,8 @@ print_cond(FILE *f, const struct dfavm_op_ir *op) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) + const struct ir_state *ir_states, + enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { fprintf(f, "fail"); @@ -134,7 +135,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "matched %td", op->ir_state - ir->states); + fprintf(f, "matched %td", op->ir_state - ir_states); } return 0; @@ -152,26 +153,18 @@ print_fetch(FILE *f) fprintf(f, "read -rn 1 c || "); } -static int -fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) +int +fsm_print_sh(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); - a = zero; - - if (!dfavm_compile_ir(&a, ir, vm_opts)) { + if (opt->io != FSM_IO_STR) { + errno = ENOTSUP; return -1; } @@ -185,7 +178,7 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) l = 0; - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { op->index = l++; } @@ -206,9 +199,9 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "\tcase $l in\n"); fprintf(f, "\tstart)\n"); - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { - if (op != a.linked) { + if (op != ops) { fprintf(f, "\t\t;&\n"); } @@ -222,14 +215,14 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) switch (op->instr) { case VM_OP_STOP: print_cond(f, op); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, op->u.stop.end_bits)) { return -1; } break; case VM_OP_FETCH: print_fetch(f); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, op->u.fetch.end_bits)) { return -1; } break; @@ -251,39 +244,6 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "\tesac\n"); fprintf(f, "done\n"); - dfavm_opasm_finalize_op(&a); - - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_sh(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - if (fsm->opt->io != FSM_IO_STR) { - errno = ENOTSUP; - return -1; - } - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - r = fsm_print_shfrag(f, ir, fsm->opt); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 5cfa43fc3..6597f2c53 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -34,9 +34,10 @@ enum asm_dialect { }; static int -print_asm_amd64(FILE *f, const char *prefix, - const struct ir *ir, const struct fsm_options *opt, - const struct dfavm_assembler_ir *a, enum asm_dialect dialect) +print_asm_amd64(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, + const char *prefix, + enum asm_dialect dialect) { // const char *sst_reg = NULL; // state struct: not currently used const char *stp_reg = "rdi"; // string pointer @@ -60,9 +61,8 @@ print_asm_amd64(FILE *f, const char *prefix, #endif assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - assert(a != NULL); + assert(ir != NULL); switch (dialect) { case AMD64_ATT: comment = "#"; break; @@ -140,7 +140,7 @@ print_asm_amd64(FILE *f, const char *prefix, fprintf(f, "\tADDQ %s, %s\n\n", stp_reg, stn_reg); } - for (op = a->linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (curr_st != op->ir_state) { unsigned state = op->ir_state - ir->states; if (op->num_incoming > 0) { @@ -366,91 +366,58 @@ print_asm_amd64(FILE *f, const char *prefix, break; } - if (ferror(f)) { - return -1; - } - return 0; } static int -print_vmasm_encoding(FILE *f, const struct fsm *fsm, enum asm_dialect dialect) +print_vmasm_encoding(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, + enum asm_dialect dialect) { - struct ir *ir; const char *prefix; - int r; - - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); if (dialect == AMD64_GO) { - if (fsm->opt->io != FSM_IO_STR && fsm->opt->io != FSM_IO_PAIR) { + if (opt->io != FSM_IO_STR && opt->io != FSM_IO_PAIR) { errno = ENOTSUP; return -1; } } else { - if (fsm->opt->io != FSM_IO_PAIR) { + if (opt->io != FSM_IO_PAIR) { errno = ENOTSUP; return -1; } } - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - assert(f != NULL); - assert(ir != NULL); - - a = zero; - - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - free_ir(fsm, ir); - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; + if (opt->prefix != NULL) { + prefix = opt->prefix; } else { prefix = "fsm_"; } - r = print_asm_amd64(f, prefix, ir, fsm->opt, &a, dialect); - - dfavm_opasm_finalize_op(&a); - free_ir(fsm, ir); - - return r; + return print_asm_amd64(f, opt, ir, ops, prefix, dialect); } int -fsm_print_amd64_att(FILE *f, const struct fsm *fsm) +fsm_print_amd64_att(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, fsm, AMD64_ATT); + return print_vmasm_encoding(f, opt, ir, ops, AMD64_ATT); } int -fsm_print_amd64_nasm(FILE *f, const struct fsm *fsm) +fsm_print_amd64_nasm(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, fsm, AMD64_NASM); + return print_vmasm_encoding(f, opt, ir, ops, AMD64_NASM); } int -fsm_print_amd64_go(FILE *f, const struct fsm *fsm) +fsm_print_amd64_go(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, fsm, AMD64_GO); + return print_vmasm_encoding(f, opt, ir, ops, AMD64_GO); } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index f94cee60d..9e22fe3bc 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -29,7 +29,7 @@ #include "ir.h" static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -89,7 +89,8 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) + const struct ir_state *ir_states, + enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { fprintf(f, "return -1;"); @@ -104,7 +105,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return %td;", op->ir_state - ir->states); + fprintf(f, "return %td;", op->ir_state - ir_states); } return 0; @@ -281,28 +282,19 @@ walk_sequence(struct dfavm_op_ir *op, /* TODO: eventually to be non-static */ static int -fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, +fsm_print_cfrag(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); assert(cp != NULL); - a = zero; - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. * We'll need that in order to be able to use the leaf callback here. */ (void) leaf; @@ -311,10 +303,6 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -325,20 +313,20 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = 0; - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { op->index = l++; } } } - assert(a.linked != NULL); + assert(ops != NULL); /* * Only declare variables if we're actually going to use them. */ - if (a.linked->cmp == VM_CMP_ALWAYS && a.linked->instr == VM_OP_STOP) { - assert(a.linked->next == NULL); + if (ops->cmp == VM_CMP_ALWAYS && ops->instr == VM_OP_STOP) { + assert(ops->next == NULL); } else { switch (opt->io) { case FSM_IO_GETC: @@ -361,7 +349,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } } - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { fprintf(f, "\n"); print_label(f, op, opt); @@ -373,7 +361,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, op->u.stop.end_bits)) { return -1; } break; @@ -391,7 +379,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "if (e - p < %zu || 0 != memcmp(p, \"", n); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, end_bits, ir)) { + if (-1 == print_end(f, NULL, opt, ir->states, end_bits)) { return -1; } fprintf(f, "\n"); @@ -404,7 +392,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "if (0 != strncmp(p, \""); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, end_bits, ir)) { + if (-1 == print_end(f, NULL, opt, ir->states, end_bits)) { return -1; } fprintf(f, "\n"); @@ -415,7 +403,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, op = tail; } else { print_fetch(f, opt); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, op->u.fetch.end_bits)) { return -1; } if (opt->io == FSM_IO_PAIR) { @@ -448,25 +436,38 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\n"); } - dfavm_opasm_finalize_op(&a); - return 0; } -static int -fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix) +int +fsm_print_vmc(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); + const char *prefix; + /* TODO: currently unused, but must be non-NULL */ const char *cp = ""; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - assert(prefix != NULL); + assert(ir != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (opt->leaf != NULL) { + leaf = opt->leaf; + } else { + leaf = print_leaf; + } if (opt->fragment) { - if (-1 == fsm_print_cfrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + if (-1 == fsm_print_cfrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque)) { return -1; } @@ -490,8 +491,8 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt break; } - if (-1 == fsm_print_cfrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) + if (-1 == fsm_print_cfrag(f, opt, ir, ops, cp, + leaf, opt->leaf_opaque)) { return -1; } @@ -500,41 +501,6 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_vmc(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - r = fsm_print_c_complete(f, ir, fsm->opt, prefix); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index bdcb2e3de..460cd6e72 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -70,7 +70,7 @@ op_can_fallthrough(const struct dfavm_op_ir *op) } static void -print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) +print_cond(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op) { if (op->cmp == VM_CMP_ALWAYS) { fprintf(f, "always"); @@ -83,8 +83,9 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct fsm_options *opt, + const struct ir_state *ir_states, + const struct dfavm_op_ir *op, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { fprintf(f, "fail"); @@ -99,7 +100,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "ret %td", op->ir_state - ir->states); + fprintf(f, "ret %td", op->ir_state - ir_states); if (op->ir_state->endids.count > 0) { fprintf(f, " / "); @@ -124,20 +125,24 @@ print_branch(FILE *f, const struct dfavm_op_ir *op) } static int -fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const struct dfavm_assembler_ir *a) +fsm_print_nodes(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - const struct dfavm_op_ir *op; + struct dfavm_op_ir *op; const struct ir_state *ir_state; unsigned block; + assert(f != NULL); + assert(opt != NULL); + assert(ir != NULL); + block = 0; ir_state = NULL; - for (op = a->linked; op != NULL; op = op->next) { - if (op->num_incoming > 0 || op == a->linked) { - if (op != a->linked) { + for (op = ops; op != NULL; op = op->next) { + if (op->num_incoming > 0 || op == ops) { + if (op != ops) { fprintf(f, "\t\t
S%u%s
"); + if (-1 == opt->endleaf(f, + cs->endids.ids, cs->endids.count, + opt->endleaf_opaque)) + { + return -1; + } + fprintf(f, "
", (unsigned) j); - print_state(f, groups[j].to, self); + print_name(f, groups[j].to, self); fprintf(f, "
to"); - print_state(f, cs->u.same.to, ir_indexof(ir, cs)); + print_name(f, cs->u.same.to, ir_indexof(ir, cs)); fprintf(f, "
mode"); - print_state(f, cs->u.dominant.mode, ir_indexof(ir, cs)); + print_name(f, cs->u.dominant.mode, ir_indexof(ir, cs)); fprintf(f, "
mode"); - print_state(f, cs->u.error.mode, ir_indexof(ir, cs)); + print_name(f, cs->u.error.mode, ir_indexof(ir, cs)); fprintf(f, "
\n"); fprintf(f, "\t> ];\n"); } @@ -148,7 +153,7 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\t\t\n"); fprintf(f, "\t\t\n"); - if (op == a->linked) { + if (op == ops) { fprintf(f, "\t\t\t\n"); } else { fprintf(f, "\t\t\t\n", block++); @@ -190,10 +195,10 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -202,7 +207,7 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, case VM_OP_FETCH: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -210,7 +215,7 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, case VM_OP_BRANCH: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n", block++); } - fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\n"); } - if (ir_state != op->ir_state || op->num_incoming > 0) { - if (op->ir_state->example != NULL) { + if (example != op->example) { + if (op->example != NULL) { fprintf(f, "\t\t\n"); - fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\n"); } - ir_state = op->ir_state; + example = op->example; } fprintf(f, "\t\t\n"); @@ -184,21 +169,13 @@ fsm_print_nodes(FILE *f, const struct fsm_options *opt, fprintf(f, "#%" PRIu32, op->index); fprintf(f, "\n"); - fprintf(f, "\t\t\t\n"); - switch (op->instr) { case VM_OP_STOP: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -207,7 +184,7 @@ fsm_print_nodes(FILE *f, const struct fsm_options *opt, case VM_OP_FETCH: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -237,8 +214,7 @@ fsm_print_nodes(FILE *f, const struct fsm_options *opt, } static void -fsm_print_edges(FILE *f, const struct fsm_options *opt, - const struct ir *ir, const struct dfavm_op_ir *ops) +fsm_print_edges(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *ops) { const struct dfavm_op_ir *op; unsigned long block; @@ -246,7 +222,6 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); (void) opt; @@ -307,33 +282,29 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, /* TODO: eventually to be non-static */ static int -fsm_print_vmdotfrag(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_vmdotfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); - if (-1 == fsm_print_nodes(f, opt, ir, ops)) { + if (-1 == fsm_print_nodes(f, opt, ops)) { return -1; } fprintf(f, "\n"); - fsm_print_edges(f, opt, ir, ops); + fsm_print_edges(f, opt, ops); return 0; } int -fsm_print_vmdot(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_vmdot(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); if (opt->fragment) { - if (-1 == fsm_print_vmdotfrag(f, opt, ir, ops)) { + if (-1 == fsm_print_vmdotfrag(f, opt, ops)) { return -1; } } else { @@ -351,7 +322,7 @@ fsm_print_vmdot(FILE *f, const struct fsm_options *opt, fprintf(f, "\tstart [ shape = none, label = \"\" ];\n"); fprintf(f, "\tstart -> S0:i0:w [ style = bold ];\n"); - if (-1 == fsm_print_vmdotfrag(f, opt, ir, ops)) { + if (-1 == fsm_print_vmdotfrag(f, opt, ops)) { return -1; } diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index b90b13074..2104edc51 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -26,8 +26,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - enum vmops_dialect { VMOPS_C, VMOPS_H, @@ -73,10 +71,10 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt { fprintf(f, "\t\t/* l%" PRIu32 " */\n", op->index); - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, "\t\t/* e.g. \""); /* Go's string escape rules are a superset of C's. */ - if (-1 == escputs(f, opt, c_escputc_str, op->ir_state->example)) { + if (-1 == escputs(f, opt, c_escputc_str, op->example)) { return -1; } fprintf(f, "\" */\n"); @@ -99,7 +97,6 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - const struct ir_state *ir_states, const char *prefix, enum dfavm_op_end end_bits) { @@ -110,7 +107,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -150,8 +147,7 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) /* TODO: eventually to be non-static */ static int -fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *prefix, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -177,7 +173,7 @@ fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, ir->states, prefix, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, prefix, op->u.stop.end_bits)) { return -1; } break; @@ -186,7 +182,7 @@ fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, if (-1 == print_fetch(f, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, ir->states, prefix, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, prefix, op->u.fetch.end_bits)) { return -1; } break; @@ -210,8 +206,7 @@ fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_vmops(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, enum vmops_dialect dialect) { int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); @@ -219,7 +214,6 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -235,7 +229,7 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, if (opt->fragment) { if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, opt, ir, ops, prefix, + if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix, leaf, opt->leaf_opaque)) { return -1; @@ -249,7 +243,7 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, fprintf(f, "#include \"%svmops.h\"\n", prefix); fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, opt, ir, ops, prefix, + if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix, leaf, opt->leaf_opaque)) { return -1; @@ -406,23 +400,20 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, } int -fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, ir, ops, VMOPS_C); + return fsm_print_vmops(f, opt, ops, VMOPS_C); } int -fsm_print_vmops_h(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_vmops_h(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, ir, ops, VMOPS_H); + return fsm_print_vmops(f, opt, ops, VMOPS_H); } int -fsm_print_vmops_main(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_vmops_main(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, ir, ops, VMOPS_MAIN); + return fsm_print_vmops(f, opt, ops, VMOPS_MAIN); } diff --git a/src/libfsm/vm/ir.c b/src/libfsm/vm/ir.c index 8c03d94f9..f27df508d 100644 --- a/src/libfsm/vm/ir.c +++ b/src/libfsm/vm/ir.c @@ -281,7 +281,6 @@ opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op const struct ir_state *ir_state) { static const struct dfavm_op_ir zero; - struct dfavm_op_ir *op; if (a->freelist != NULL) { @@ -291,18 +290,24 @@ opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op op = pool_newop(&a->pool); } - if (op != NULL) { - *op = zero; + if (op == NULL) { + return NULL; + } + + *op = zero; - op->asm_index = a->count++; - op->index = 0; + op->asm_index = a->count++; + op->index = 0; - op->cmp = cmp; - op->instr = instr; + op->cmp = cmp; + op->instr = instr; - op->cmp_arg = arg; + op->cmp_arg = arg; - op->ir_state = ir_state; + if (ir_state != NULL) { + op->example = ir_state->example; + op->endids.ids = ir_state->endids.ids; + op->endids.count = ir_state->endids.count; } return op; @@ -315,11 +320,13 @@ opasm_new_fetch(struct dfavm_assembler_ir *a, unsigned state, enum dfavm_op_end struct dfavm_op_ir *op; op = opasm_new(a, VM_OP_FETCH, VM_CMP_ALWAYS, 0, ir_state); - if (op != NULL) { - op->u.fetch.state = state; - op->u.fetch.end_bits = end; + if (op == NULL) { + return NULL; } + op->u.fetch.state = state; + op->u.fetch.end_bits = end; + return op; } @@ -330,10 +337,12 @@ opasm_new_stop(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned cha struct dfavm_op_ir *op; op = opasm_new(a, VM_OP_STOP, cmp, arg, ir_state); - if (op != NULL) { - op->u.stop.end_bits = end; + if (op == NULL) { + return NULL; } + op->u.stop.end_bits = end; + return op; } @@ -346,11 +355,13 @@ opasm_new_branch(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned c assert(dest_state < a->nstates); op = opasm_new(a, VM_OP_BRANCH, cmp, arg, ir_state); - if (op != NULL) { - // op->u.br.dest = VM_DEST_FAR; // start with all branches as 'far' - op->u.br.dest_state = dest_state; + if (op == NULL) { + return NULL; } + // op->u.br.dest = VM_DEST_FAR; // start with all branches as 'far' + op->u.br.dest_state = dest_state; + return op; } @@ -372,7 +383,8 @@ dfavm_opasm_finalize_op(struct dfavm_assembler_ir *a) } static int -cmp_mode_dests(const void *a, const void *b) { +cmp_mode_dests(const void *a, const void *b) +{ const long *va = a; const long *vb = b; diff --git a/src/libfsm/vm/v1.c b/src/libfsm/vm/v1.c index de1f6ea93..c9af2061f 100644 --- a/src/libfsm/vm/v1.c +++ b/src/libfsm/vm/v1.c @@ -12,6 +12,8 @@ #include #include +#include + #include "vm.h" enum dfavm_io_result diff --git a/src/libfsm/vm/v2.c b/src/libfsm/vm/v2.c index 07eb12ef4..e217f6d11 100644 --- a/src/libfsm/vm/v2.c +++ b/src/libfsm/vm/v2.c @@ -11,6 +11,8 @@ #include #include +#include + #include "vm.h" enum dfavm_vm_op_v2 { diff --git a/src/libfsm/vm/vm.h b/src/libfsm/vm/vm.h index dfe99dfd7..bfb28a774 100644 --- a/src/libfsm/vm/vm.h +++ b/src/libfsm/vm/vm.h @@ -20,7 +20,6 @@ #define DFAVM_MAGIC "DFAVM$" struct ir; -struct ir_state; struct fsm_vm_compile_opts; struct dfavm_op_ir_pool; @@ -75,7 +74,15 @@ struct dfavm_op_ir { */ uint32_t index; - const struct ir_state *ir_state; // for void *opaque + + +const char *example; + +struct { + fsm_end_id_t *ids; /* NULL -> 0 */ + size_t count; +} endids; + uint32_t num_incoming; // number of branches to this instruction int in_trace; From 1545d1c7d91d9e31901df8c96095c935b0ef2cec Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 18 Jul 2024 18:31:09 +0100 Subject: [PATCH 203/373] Only qsort a non-empty retlist. --- src/libfsm/print/llvm.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index c94e511aa..9e11a42d0 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -460,8 +460,10 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o retlist.count = 0; build_retlist(&retlist, ops); - /* sort for both dedup and bsearch */ - qsort(retlist.a, retlist.count, sizeof *retlist.a, cmp_ret_by_endid); + if (retlist.count > 0) { + /* sort for both dedup and bsearch */ + qsort(retlist.a, retlist.count, sizeof *retlist.a, cmp_ret_by_endid); + } /* * If we're going by endleaf, we deal with endids. @@ -470,6 +472,9 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o if (opt->endleaf != NULL && retlist.count > 0) { size_t j = 0; + /* sort for both dedup and bsearch */ + qsort(retlist.a, retlist.count, sizeof *retlist.a, cmp_ret_by_endid); + /* deduplicate based on endids only. * j is the start of a run; i increments until we find * the start of the next run */ From eef6c003f697a1bc5cfefa58805915de8496824e Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 18 Jul 2024 17:44:16 +0100 Subject: [PATCH 204/373] Introduce ambig enum, and replace leaf/endleaf callbacks with reject/accept hooks. The accept callback gets a set of end ids. These hooks default to outputting those ids verbatim, you needn't override them per-program. I think that's clearer for users of the API. I ported the ambig enum over from (unmerged) work on rx(1). This is for how to handle endid ambiguities, which does something appropriate to each language's generated API. So that's now built in to libfsm rather than being implemented per-language per-program. This gets us clearer handling for multiple id output per language, and I think it's also clearer about what's responsible for what, overall. --- include/fsm/options.h | 55 +++++++--- src/libfsm/print.c | 69 +++++++++++- src/libfsm/print.h | 21 +++- src/libfsm/print/awk.c | 155 +++++++++++++++------------ src/libfsm/print/c.c | 215 +++++++++++++++++++++++++++++--------- src/libfsm/print/dot.c | 97 ++++++++++++----- src/libfsm/print/fsm.c | 64 ++++++++---- src/libfsm/print/go.c | 179 ++++++++++++++++++++++--------- src/libfsm/print/irdot.c | 12 ++- src/libfsm/print/irjson.c | 11 +- src/libfsm/print/json.c | 9 +- src/libfsm/print/llvm.c | 211 +++++++++++++++++++++++++------------ src/libfsm/print/rust.c | 140 ++++++++++++++++--------- src/libfsm/print/sh.c | 99 +++++++++++++++--- src/libfsm/print/vmasm.c | 112 +++++++++++--------- src/libfsm/print/vmc.c | 208 ++++++++++++++++++++++++++---------- src/libfsm/print/vmdot.c | 79 +++++++++----- src/libfsm/print/vmops.c | 107 ++++++++++--------- src/lx/print/c.c | 119 +++++++++++---------- src/lx/print/dot.c | 24 +++-- src/re/main.c | 125 +++++++++++----------- 21 files changed, 1428 insertions(+), 683 deletions(-) diff --git a/include/fsm/options.h b/include/fsm/options.h index 0f861baef..a5e090992 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -19,6 +19,16 @@ enum fsm_io { FSM_IO_PAIR }; +// TODO: comment +enum fsm_ambig { + AMBIG_NONE = 0, /* default */ + AMBIG_ERROR = 1 << 0, + AMBIG_EARLIEST = 1 << 1, + AMBIG_MULTIPLE = 1 << 2, + + AMBIG_SINGLE = AMBIG_ERROR | AMBIG_EARLIEST +}; + struct fsm_options { /* boolean: true indicates to omit names for states in output */ unsigned int anonymous_states:1; @@ -56,27 +66,46 @@ struct fsm_options { /* for generated code, what kind of I/O API to generate */ enum fsm_io io; + /* for generated code, how to handle multiple endids on an accepting state */ + enum fsm_ambig ambig; + /* a prefix for namespacing generated identifiers. NULL if not required. */ const char *prefix; /* the name of the enclosing package; NULL to use `prefix` (default). */ const char *package_prefix; - /* character pointer, for C code fragment output. NULL for the default. */ - const char *cp; + /* + * Hooks to override generated code. These give an oportunity to + * emit application-specific constructs, especially based on ids + * attached to end states. + * + * These hooks are optional, and default to whatever makes sense + * for the language. + * + * Placement in the output stream depends on the format. + * This replaces an entire "return xyz;" statement for C-like formats, + * but appends extra information for others. + */ + struct fsm_hooks { + /* character pointer, for C code fragment output. NULL for the default. */ + const char *cp; + + int (*args)(FILE *, const struct fsm_options *opt, + void *leaf_opaque); - /* TODO: explain. for C code fragment output */ - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, - const void *leaf_opaque); - void *leaf_opaque; + /* + * ids[] is sorted and does not have duplicates. + */ + int (*accept)(FILE *, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *leaf_opaque); - /* TODO: explain. for C code fragment output */ - /* Placement in the output stream depends on the format. - * This replaces an entire "return xyz;" statement for C-like formats, - * but appends extra information for others. */ - int (*endleaf)(FILE *, const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque); - void *endleaf_opaque; + int (*reject)(FILE *, const struct fsm_options *opt, + void *leaf_opaque); + + void *hook_opaque; + } hooks; /* custom allocation functions */ const struct fsm_alloc *alloc; diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 789d2825a..427ef5021 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -19,6 +19,73 @@ #include "vm/vm.h" #include "print/ir.h" +int +print_hook_args(FILE *f, const struct fsm_options *opt, + int (*default_args)(FILE *f, const struct fsm_options *opt, + void *lang_opaque), + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + + if (default_args == NULL) { + assert(lang_opaque == NULL); + } + + if (opt->hooks.accept != NULL) { + return opt->hooks.args(f, opt, lang_opaque); + } else if (default_args != NULL) { + return default_args(f, opt, lang_opaque); + } + + return 0; +} + +int +print_hook_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + int (*default_accept)(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque), + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + + if (default_accept == NULL) { + assert(lang_opaque == NULL); + } + + if (opt->hooks.accept != NULL) { + return opt->hooks.accept(f, opt, ids, count, lang_opaque); + } else if (default_accept != NULL) { + return default_accept(f, opt, ids, count, lang_opaque); + } + + return 0; +} + +int +print_hook_reject(FILE *f, const struct fsm_options *opt, + int (*default_reject)(FILE *f, const struct fsm_options *opt, + void *lang_opaque), + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + + if (default_reject == NULL) { + assert(lang_opaque == NULL); + } + + if (opt->hooks.reject != NULL) { + return opt->hooks.reject(f, opt, lang_opaque); + } else if (default_reject != NULL) { + return default_reject(f, opt, lang_opaque); + } + + return 0; +} int fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) @@ -54,7 +121,7 @@ fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) case FSM_PRINT_API: print_fsm = fsm_print_api; break; case FSM_PRINT_AWK: print_vm = fsm_print_awk; break; case FSM_PRINT_C: print_ir = fsm_print_c; break; - case FSM_PRINT_DOT: print_ir = fsm_print_dot; break; + case FSM_PRINT_DOT: print_fsm = fsm_print_dot; break; case FSM_PRINT_FSM: print_fsm = fsm_print_fsm; break; case FSM_PRINT_GO: print_vm = fsm_print_go; break; case FSM_PRINT_IR: print_ir = fsm_print_ir; break; diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 72e028aec..8b6a9555a 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -11,6 +11,25 @@ struct fsm; struct ir; struct dfavm_op_ir; +int +print_hook_args(FILE *f, const struct fsm_options *opt, + int (*default_args)(FILE *f, const struct fsm_options *opt, + void *lang_opaque), + void *lang_opaque); + +int +print_hook_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + int (*default_accept)(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque), + void *lang_opaque); + +int +print_hook_reject(FILE *f, const struct fsm_options *opt, + int (*default_reject)(FILE *f, const struct fsm_options *opt, void *lang_opaque), + void *lang_opaque); + typedef int fsm_print_f(FILE *f, const struct fsm_options *opt, const struct fsm *fsm); typedef int ir_print_f(FILE *f, const struct fsm_options *opt, const struct ir *ir); typedef int vm_print_f(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops); @@ -22,7 +41,7 @@ vm_print_f fsm_print_amd64_nasm; fsm_print_f fsm_print_api; vm_print_f fsm_print_awk; ir_print_f fsm_print_c; -ir_print_f fsm_print_dot; +fsm_print_f fsm_print_dot; fsm_print_f fsm_print_fsm; vm_print_f fsm_print_go; ir_print_f fsm_print_ir; diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index bf06d471f..54684fb93 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -24,26 +24,12 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" #define START UINT32_MAX -static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) leaf_opaque; - (void) ids; - (void) count; - - fprintf(f, "return 0;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -62,6 +48,79 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + switch (opt->ambig) { + case AMBIG_NONE: + /* awk doesn't have a boolean type */ + fprintf(f, "return 1;"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "return %u;", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "return %u;", ids[0]); + break; + + case AMBIG_MULTIPLE: + /* awk can't return an array */ + fprintf(f, "return \""); + + for (size_t i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + + if (i < count - 1) { + fprintf(f, ","); + } + } + + fprintf(f, "\""); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "return 0;"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op) { @@ -88,38 +147,20 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "return 0"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, default_reject, NULL); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + case VM_END_SUCC: + return print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - size_t i; - - assert(f != NULL); + default_accept, + NULL); - /* awk can't return an array */ - fprintf(f, "return \""); - - for (i = 0; i < op->endids.count; i++) { - fprintf(f, "%u", op->endids.ids[i]); - - if (i < op->endids.count - 1) { - fprintf(f, ","); - } - } - - fprintf(f, "\""); + default: + assert(!"unreached"); + abort(); } - - return 0; } static void @@ -140,9 +181,7 @@ print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) static int fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, - const char *cp, const char *prefix, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *cp, const char *prefix) { struct dfavm_op_ir *op; @@ -151,11 +190,6 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, assert(cp != NULL); assert(prefix != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; @@ -247,7 +281,6 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, int fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; const char *cp; @@ -260,22 +293,14 @@ fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - - if (opt->cp != NULL) { - cp = opt->cp; + if (opt->hooks.cp != NULL) { + cp = opt->hooks.cp; } else { cp = "c"; /* XXX */ } if (opt->fragment) { - if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix, - leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix)) { return -1; } } else { @@ -297,9 +322,7 @@ fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) fprintf(f, ", l, c) {\n"); - if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix, - leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix)) { return -1; } diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 1d17abb70..c22435baf 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "ir.h" @@ -52,17 +53,95 @@ ir_hasend(const struct ir *ir) return 0; } +// TODO: centralise vmc/c static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) +{ + switch (ambig) { + case AMBIG_NONE: + // TODO: for C99 we'd return bool + fprintf(f, "return 1;"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "return %u;", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "return %u;", ids[0]); + break; + + case AMBIG_MULTIPLE: + /* + * Here I would like to emit (static unsigned []) { 1, 2, 3 } + * but specifying a storage duration for compound literals + * is a compiler extension. + * So I'm emitting a static const variable declaration instead. + */ + + fprintf(f, "{\n"); + fprintf(f, "\t\tstatic const unsigned a[] = { "); + for (fsm_end_id_t i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, " };\n"); + fprintf(f, "\t\t*ids = a;\n"); + fprintf(f, "\t\t*count = %zu;\n", count); + fprintf(f, "\t\treturn 0;\n"); + fprintf(f, "\t}"); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { assert(f != NULL); - assert(leaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); - (void) ids; - (void) count; - (void) leaf_opaque; + (void) lang_opaque; - fprintf(f, "return 0; /* leaf */"); + if (-1 == print_ids(f, opt->ambig, ids, count)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "return 0;"); return 0; } @@ -129,21 +208,18 @@ print_groups(FILE *f, const struct fsm_options *opt, static int print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *cp, - struct ir_state *cs, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + struct ir_state *cs) { assert(ir != NULL); assert(opt != NULL); assert(cp != NULL); assert(f != NULL); assert(cs != NULL); - assert(leaf != NULL); switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); - if (-1 == leaf(f, cs->endids.ids, cs->endids.count, leaf_opaque)) { + if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -172,7 +248,7 @@ print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_groups(f, opt, ir_indexof(ir, cs), cs->u.partial.groups, cs->u.partial.n); fprintf(f, "\t\t\tdefault: "); - if (-1 == leaf(f, cs->endids.ids, cs->endids.count, leaf_opaque)) { + if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -203,7 +279,7 @@ print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_ranges(f, opt, cs->u.error.error.ranges, cs->u.error.error.n); fprintf(f, " "); - if (-1 == leaf(f, cs->endids.ids, cs->endids.count, leaf_opaque)) { + if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -259,7 +335,7 @@ print_stateenum(FILE *f, size_t n) } static int -endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) +print_endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) { unsigned i; @@ -283,20 +359,23 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) fprintf(f, "\tcase S%u: ", i); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - ir->states[i].endids.ids, ir->states[i].endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "return 1;"); + if (-1 == print_hook_accept(f, opt, + ir->states[i].endids.ids, ir->states[i].endids.count, + default_accept, + NULL)) + { + return -1; } fprintf(f, "\n"); } - fprintf(f, "\tdefault: return 0; /* unexpected EOT */\n"); + + /* unexpected EOT */ + fprintf(f, "\tdefault: "); + if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + return -1; + } + fprintf(f, "\n"); fprintf(f, "\t}\n"); return 0; @@ -304,9 +383,7 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) int fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *cp) { unsigned i; @@ -330,7 +407,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } fprintf(f, "\n"); - if (-1 == print_case(f, ir, opt, cp, &ir->states[i], leaf, leaf_opaque)) { + if (-1 == print_case(f, ir, opt, cp, &ir->states[i])) { return -1; } @@ -348,9 +425,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } static int -fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt, - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) +fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) { const char *cp; @@ -358,8 +433,8 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt, assert(ir != NULL); assert(opt != NULL); - if (opt->cp != NULL) { - cp = opt->cp; + if (opt->hooks.cp != NULL) { + cp = opt->hooks.cp; } else { switch (opt->io) { case FSM_IO_GETC: cp = "c"; break; @@ -390,9 +465,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt, break; } - if (-1 == fsm_print_cfrag(f, ir, opt, cp, - leaf, leaf_opaque)) - { + if (-1 == fsm_print_cfrag(f, ir, opt, cp)) { return -1; } @@ -400,7 +473,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\n"); /* end states */ - if (-1 == endstates(f, opt, ir)) { + if (-1 == print_endstates(f, opt, ir)) { return -1; } @@ -411,7 +484,6 @@ int fsm_print_c(FILE *f, const struct fsm_options *opt, const struct ir *ir) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; assert(f != NULL); @@ -424,25 +496,68 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - if (opt->fragment) { - if (-1 == fsm_print_c_body(f, ir, opt, leaf, opt->leaf_opaque)) { + if (-1 == fsm_print_c_body(f, ir, opt)) { return -1; } } else { fprintf(f, "\n"); fprintf(f, "int\n%smain", prefix); + fprintf(f, "("); + + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "int (*fsm_getc)(void *getc_opaque), void *getc_opaque"); + break; + + case FSM_IO_STR: + fprintf(f, "const char *s"); + break; + + case FSM_IO_PAIR: + fprintf(f, "const char *b, const char *e"); + break; + } + + /* + * unsigned rather than fsm_end_id_t here, so the generated code + * doesn't depend on fsm.h + */ + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(stdout, ",\n"); + fprintf(stdout, "\tconst unsigned *id"); + break; + + case AMBIG_MULTIPLE: + fprintf(stdout, ",\n"); + fprintf(stdout, "\tconst unsigned **ids, size_t *count"); + break; + + default: + assert(!"unreached"); + abort(); + } + + if (opt->hooks.args != NULL) { + fprintf(stdout, ",\n"); + fprintf(stdout, "\t"); + + if (-1 == print_hook_args(f, opt, NULL, NULL)) { + return -1; + } + } + + fprintf(f, ")\n"); + fprintf(f, "{\n"); switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "(int (*fsm_getc)(void *getc_opaque), void *getc_opaque)\n"); - fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tint c;\n"); fprintf(f, "\n"); @@ -450,8 +565,6 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, break; case FSM_IO_STR: - fprintf(f, "(const char *s)\n"); - fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); @@ -459,8 +572,6 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e)\n"); - fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); @@ -471,7 +582,7 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, if (ir->n == 0) { fprintf(f, "\treturn 0; /* no matches */\n"); } else { - if (-1 == fsm_print_c_body(f, ir, opt, leaf, opt->leaf_opaque)) { + if (-1 == fsm_print_c_body(f, ir, opt)) { return -1; } } diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 892287747..caa3fc7f3 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -11,6 +11,7 @@ #include #include "libfsm/internal.h" /* XXX: up here for bitmap.h */ +#include "libfsm/print.h" #include @@ -24,6 +25,61 @@ #include #include +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + fsm_state_t s; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + s = * (fsm_state_t *) lang_opaque; + + fprintf(f, "label = <"); + + if (!opt->anonymous_states) { + fprintf(f, "%u", s); + + if (count > 0) { + fprintf(f, "
"); + } + } + + for (size_t i = 0; i < count; i++) { + fprintf(f, "#%u", ids[i]); + + if (i < count - 1) { + fprintf(f, ","); + } + } + + fprintf(f, ">"); + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + fsm_state_t s; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + s = * (fsm_state_t *) lang_opaque; + + if (!opt->anonymous_states) { + fprintf(f, "label = <%u>", s); + } + + return 0; +} + static int print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, const char *prefix, fsm_state_t s) @@ -158,8 +214,13 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, for (s = 0; s < fsm->statecount; s++) { if (!fsm_isend(fsm, s)) { if (!opt->anonymous_states) { - fprintf(f, "\t%sS%-2u [ label = <%u> ];\n", - prefix, s, s); + fprintf(f, "\t%sS%-2u [ ", prefix, s); + + if (-1 == print_hook_reject(f, opt, default_reject, &s)) { + return -1; + } + + fprintf(f, " ];\n"); } } else { fsm_end_id_t *ids; @@ -186,35 +247,13 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, fprintf(f, ", "); - fprintf(f, "label = <"); - - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - ids, count, - opt->endleaf_opaque)) - { - return -1; - } - } else if (!opt->anonymous_states) { - fprintf(f, "%u", s); - } - - if (opt->endleaf != NULL || !opt->anonymous_states) { - if (count > 0) { - fprintf(f, "
"); - } - } - - for (size_t i = 0; i < count; i++) { - fprintf(f, "#%u", ids[i]); - - if (i < count - 1) { - fprintf(f, ","); - } + if (-1 == print_hook_accept(f, opt, + ids, count, + default_accept, &s)) + { + return -1; } - fprintf(f, ">"); - fprintf(f, " ];\n"); f_free(opt->alloc, ids); diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 6bd441882..1f0adb9c1 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -9,6 +9,7 @@ #include #include "libfsm/internal.h" /* XXX: up here for bitmap.h */ +#include "libfsm/print.h" #include @@ -24,6 +25,34 @@ #include #include +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + if (count == 0) { + return 0; + } + + fprintf(f, " = ["); + + for (size_t i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + + if (i + 1 < count) { + fprintf(f, ", "); + } + } + + fprintf(f, "]"); + + return 0; +} + /* TODO: centralise */ static int findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) @@ -149,6 +178,12 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, assert(f != NULL); assert(opt != NULL); + if (!fsm_isend(fsm, s)) { + if (-1 == print_hook_reject(f, opt, NULL, NULL)) { + return -1; + } + } + { struct state_iter jt; fsm_state_t st; @@ -312,30 +347,17 @@ fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) assert(res == 1); } - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - ids, count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "%u", s); + fprintf(f, "%u", s); + + if (-1 == print_hook_accept(f, opt, + ids, count, + default_accept, + NULL)) + { + return -1; } if (count > 0) { - fprintf(f, " = ["); - - for (size_t id = 0; id < count; id++) { - fprintf(f, "%zu", id); - - if (id + 1 < count) { - fprintf(f, ", "); - } - } - - fprintf(f, "]"); - f_free(opt->alloc, ids); } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 05ce812e6..442d20466 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -23,24 +23,10 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" -static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) count; - (void) leaf_opaque; - - fprintf(f, "return 0;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -59,6 +45,79 @@ cmp_operator(int cmp) } } +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) +{ + switch (ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, ", %u;", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, ", %u;", ids[0]); + break; + + case AMBIG_MULTIPLE: + assert(!"unimplemented"); + abort(); + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "return true"); + + if (-1 == print_ids(f, opt->ambig, ids, count)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "{\n\t\treturn false\n\t}\n"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { @@ -89,28 +148,30 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "{\n\t\treturn false\n\t}\n"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, default_reject, NULL); - fprintf(f, "{\n"); - fprintf(f, "\t\t"); + case VM_END_SUCC: + fprintf(f, "{\n"); + fprintf(f, "\t\t"); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + if (-1 == print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) + default_accept, + NULL)) { return -1; } - } else { - fprintf(f, "return true"); - } - fprintf(f, "\n\t}\n"); + fprintf(f, "\n\t}\n"); - return 0; + return 0; + + default: + assert(!"unreached"); + abort(); + } } static void @@ -136,9 +197,7 @@ print_fetch(FILE *f, const struct fsm_options *opt) /* TODO: eventually to be non-static */ static int fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, - const char *cp, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *cp) { struct dfavm_op_ir *op; @@ -146,11 +205,6 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops assert(opt != NULL); assert(cp != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; @@ -238,7 +292,6 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops int fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; const char *package_prefix; @@ -254,12 +307,6 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - if (opt->package_prefix != NULL) { package_prefix = opt->package_prefix; } else { @@ -267,9 +314,7 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) } if (opt->fragment) { - if (-1 == fsm_print_gofrag(f, opt, ops, cp, - leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_gofrag(f, opt, ops, cp)) { return -1; } } else { @@ -280,11 +325,11 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) switch (opt->io) { case FSM_IO_PAIR: - fprintf(f, "(data []byte) bool {\n"); + fprintf(f, "(data []byte)"); break; case FSM_IO_STR: - fprintf(f, "(data string) bool {\n"); + fprintf(f, "(data string)"); break; default: @@ -292,9 +337,39 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) return -1; } - if (-1 == fsm_print_gofrag(f, opt, ops, cp, - leaf, opt->leaf_opaque)) - { + if (opt->hooks.args != NULL) { + fprintf(stdout, ", "); + + if (-1 == print_hook_args(f, opt, NULL, NULL)) { + return -1; + } + } + + fprintf(f, " "); + + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "bool"); + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(stdout, "(bool, uint)"); + break; + + case AMBIG_MULTIPLE: + // TODO: fprintf(stdout, "(bool, uint[])"); + errno = ENOTSUP; + return -1; + + default: + assert(!"unreached"); + abort(); + } + + fprintf(f, " {\n"); + + if (-1 == fsm_print_gofrag(f, opt, ops, cp)) { return -1; } diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index ac6a6f24a..db0f559ee 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "ir.h" @@ -214,15 +215,18 @@ print_state(FILE *f, const struct fsm_options *opt, /* TODO: leaf callback for dot output */ - /* showing endleaf in addition to existing content */ - if (cs->isend && opt->endleaf != NULL) { + /* showing hook in addition to existing content */ + if (cs->isend && opt->hooks.accept != NULL) { fprintf(f, "\t\t \n"); } diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 1848e5102..a15b47015 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "ir.h" @@ -135,15 +136,17 @@ print_state(FILE *f, const struct fsm_options *opt, fprintf(f, "],\n"); } - /* showing endleaf in addition to existing content */ - if (cs->isend && opt->endleaf != NULL) { + /* showing hook in addition to existing content */ + if (cs->isend && opt->hooks.accept != NULL) { fprintf(f, "\t\t\t\"endleaf\": "); - if (-1 == opt->endleaf(f, + + if (-1 == print_hook_accept(f, opt, cs->endids.ids, cs->endids.count, - opt->endleaf_opaque)) + NULL, NULL)) { return -1; } + fprintf(f, ",\n"); } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 9e4f0172d..18718d215 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -11,6 +11,7 @@ #include #include "libfsm/internal.h" /* XXX: up here for bitmap.h */ +#include "libfsm/print.h" #include @@ -240,8 +241,8 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) fprintf(f, " ],\n"); } - /* showing endleaf in addition to existing content */ - if (opt->endleaf != NULL) { + /* showing hook in addition to existing content */ + if (opt->hooks.accept != NULL) { int notfirst; notfirst = 0; @@ -272,9 +273,9 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) fprintf(f, "{ %u, ", i); - if (-1 == opt->endleaf(f, + if (-1 == print_hook_accept(f, opt, ids, count, - opt->endleaf_opaque)) + NULL, NULL)) { return -1; } diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 9e11a42d0..6cf49db28 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -26,6 +26,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" @@ -43,6 +44,16 @@ static const char *ptr_void = "i8*"; static const struct dfavm_op_ir fail; // used as a unqiue address only +struct ret { + size_t count; + const fsm_end_id_t *ids; +}; + +struct ret_list { + size_t count; + struct ret *a; +}; + /* * If we had a stack, the current set of live values would be a frame. * We're a DFA, so we don't have a stack. But I still think of them as a frame. @@ -66,23 +77,6 @@ decl(unsigned *n) return (*n)++; } -static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) count; - (void) leaf_opaque; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "ret false;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -101,6 +95,110 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + size_t i; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + i = * (const size_t *) lang_opaque; + + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "[true, %%ret%zu],\n", i); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "[{ true, %u }, %%ret%zu],\n", ids[0], i); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "[{ true, i32 %u }, %%ret%zu],\n", ids[0], i); + break; + + case AMBIG_MULTIPLE: + // TODO: probably { i1, ptr_u8 } + assert(!"unimplemented"); + abort(); + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "[false, %%fail]\n"); + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + case AMBIG_MULTIPLE: + fprintf(f, "[{ false, undef }, %%fail]\n"); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +print_rettype(FILE *f, enum fsm_ambig ambig) +{ + switch (ambig) { + case AMBIG_NONE: + fprintf(f, "i1"); + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, "{ i1, u32 }"); + break; + + case AMBIG_MULTIPLE: + // TODO: probably { i1, ptr_u8 } + assert(!"unimplemented"); + abort(); + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + static void print_decl(FILE *f, const char *name, unsigned n) { @@ -229,6 +327,8 @@ print_fetch(FILE *f, const struct fsm_options *opt, print_decl(f, "r", decl(&frame->r)); fprintf(f, "icmp eq i32 %%i%u, -1 ; EOF\n", n); +// XXX: we don't distinguish error from eof +// https://github.com/katef/libfsm/issues/484 print_branch(f, frame, end_bits == VM_END_FAIL ? &fail : NULL, NULL); @@ -307,16 +407,6 @@ print_fetch(FILE *f, const struct fsm_options *opt, } } -struct ret { - size_t count; - const fsm_end_id_t *ids; -}; - -struct ret_list { - size_t count; - struct ret *a; -}; - static bool append_ret(struct ret_list *list, const fsm_end_id_t *ids, size_t count) @@ -427,9 +517,7 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) /* TODO: eventually to be non-static */ static int fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, - const char *cp, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *cp) { struct ret_list retlist; struct dfavm_op_ir *op; @@ -438,11 +526,6 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o assert(opt != NULL); assert(cp != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; @@ -461,15 +544,6 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o build_retlist(&retlist, ops); if (retlist.count > 0) { - /* sort for both dedup and bsearch */ - qsort(retlist.a, retlist.count, sizeof *retlist.a, cmp_ret_by_endid); - } - - /* - * If we're going by endleaf, we deal with endids. - * If we don't have endleaf, we deal with state numbers. - */ - if (opt->endleaf != NULL && retlist.count > 0) { size_t j = 0; /* sort for both dedup and bsearch */ @@ -531,23 +605,30 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o * llvm doesn't find this optimisation for us. */ print_label(f, true, "stop"); - fprintf(f, "\t%%ret = phi i1\n"); + + fprintf(f, "\t%%ret = phi "); + print_rettype(f, opt->ambig); + fprintf(f, "\n"); + for (size_t i = 0; i < retlist.count; i++) { fprintf(f, "\t "); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - retlist.a[i].ids, retlist.a[i].count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "[true, %%ret%zu],\n", i); + if (-1 == print_hook_accept(f, opt, + retlist.a[i].ids, retlist.a[i].count, + default_accept, &i)) + { + return -1; } } - fprintf(f, "\t%s[false, %%fail]\n", " "); - fprintf(f, "\tret i1 %%ret\n"); + + fprintf(f, "\t "); + if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + return -1; + } + + fprintf(f, "\tret "); + print_rettype(f, opt->ambig); + fprintf(f, " %%ret\n"); print_label(f, true, "fail"); fprintf(f, "\tbr "); @@ -657,7 +738,6 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o int fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; const char *cp; @@ -670,25 +750,19 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - - if (opt->cp != NULL) { - cp = opt->cp; + if (opt->hooks.cp != NULL) { + cp = opt->hooks.cp; } else { cp = "c"; /* XXX */ } if (opt->fragment) { - fsm_print_llvmfrag(f, opt, ops, cp, - leaf, opt->leaf_opaque); + fsm_print_llvmfrag(f, opt, ops, cp); return 0; } fprintf(f, "; generated\n"); +//XXX: type depends on ambig fprintf(f, "define dso_local i1 @%smain", prefix); switch (opt->io) { @@ -740,8 +814,7 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) exit(EXIT_FAILURE); } - fsm_print_llvmfrag(f, opt, ops, cp, - leaf, opt->leaf_opaque); + fsm_print_llvmfrag(f, opt, ops, cp); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index da5b15a93..eefe84b11 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -24,26 +24,12 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" #define START UINT32_MAX -static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) count; - (void) leaf_opaque; - - fprintf(f, "return None;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -62,6 +48,78 @@ cmp_operator(int cmp) } } +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) +{ + switch (ambig) { + case AMBIG_NONE: + fprintf(f, "return Some(())"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "return Some(%u)", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "return Some(%u)", ids[0]); + break; + + case AMBIG_MULTIPLE: + assert(!"unimplemented"); + abort(); + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + if (-1 == print_ids(f, opt->ambig, ids, count)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "return None"); + + return 0; +} + static int has_op(const struct dfavm_op_ir *ops, enum dfavm_op_instr instr) { @@ -102,23 +160,20 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "return None"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, default_reject, NULL); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + case VM_END_SUCC: + return print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "return Some(())"); - } + default_accept, + NULL); - return 0; + default: + assert(!"unreached"); + abort(); + } } static void @@ -144,9 +199,7 @@ print_fetch(FILE *f) /* TODO: eventually to be non-static */ static int fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, - const char *cp, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *cp) { struct dfavm_op_ir *op; bool fallthrough; @@ -155,11 +208,6 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o assert(opt != NULL); assert(cp != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; @@ -267,7 +315,6 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { return -1; } - fprintf(f, ";"); if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, " }"); } @@ -360,7 +407,6 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o int fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; const char *cp; @@ -373,21 +419,14 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - - if (opt->cp != NULL) { - cp = opt->cp; + if (opt->hooks.cp != NULL) { + cp = opt->hooks.cp; } else { cp = "c"; /* XXX */ } if (opt->fragment) { - fsm_print_rustfrag(f, opt, ops, cp, - leaf, opt->leaf_opaque); + fsm_print_rustfrag(f, opt, ops, cp); goto error; } @@ -421,8 +460,7 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, opt, ops, cp, - leaf, opt->leaf_opaque); + fsm_print_rustfrag(f, opt, ops, cp); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 89ec3897f..a01369b24 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -24,6 +24,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" @@ -63,6 +64,81 @@ ord_operator(int cmp) } } +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) +{ + switch (ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, " %u", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, " %u", ids[0]); + break; + + case AMBIG_MULTIPLE: + for (size_t i = 0; i < count; i++) { + fprintf(f, " %u", ids[i]); + } + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "matched"); + + if (-1 == print_ids(f, opt->ambig, ids, count)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "fail"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { @@ -119,20 +195,19 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "fail"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, default_reject, NULL); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + case VM_END_SUCC: + return print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "matched"); + default_accept, + NULL); + + default: + assert(!"unreached"); + abort(); } return 0; diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 0ae125f8f..95250d8a0 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" @@ -31,6 +32,56 @@ enum asm_dialect { AMD64_GO, }; +static int +print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, + enum asm_dialect dialect, const char *ret_reg, + enum dfavm_op_end end_bits) +{ + bool r; + + switch (end_bits) { + case VM_END_FAIL: + if (-1 == print_hook_reject(f, opt, NULL, NULL)) { + return -1; + } + break; + + case VM_END_SUCC: + if (-1 == print_hook_accept(f, opt, + op->endids.ids, op->endids.count, + NULL, NULL)) + { + return -1; + } + break; + + default: + assert(!"unreached"); + abort(); + } + + if (opt->ambig != AMBIG_NONE) { + errno = ENOTSUP; + return -1; + } + + r = end_bits == VM_END_SUCC; + + switch (dialect) { + case AMD64_ATT: + fprintf(f, "\tmovl $%d, %%%s\n", r, ret_reg); + break; + case AMD64_NASM: + fprintf(f, "\tMOV %s, %d\n", ret_reg, r); + break; + case AMD64_GO: + fprintf(f, "\tMOVQ $%d, %s\n", r, ret_reg); + break; + } + + return 0; +} + static int print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *prefix, @@ -145,36 +196,18 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, switch (op->instr) { case VM_OP_STOP: { - const char *jmp_op; - bool r = op->u.stop.end_bits == VM_END_SUCC; - - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $%d, %%%s\n", r, ret_reg); - break; - case AMD64_NASM: - fprintf(f, "\tMOV %s, %d\n", ret_reg, r); - break; - case AMD64_GO: - fprintf(f, "\tMOVQ $%d, %s\n", r, ret_reg); - break; - } - - if (r) { - /* endleaf in addition to existing instructions */ - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } + if (-1 == print_end(f, op, opt, + dialect, ret_reg, + op->u.stop.end_bits)) + { + return -1; } if (op->cmp == VM_CMP_ALWAYS && op->next == NULL) { fprintf(f, "\t%s elided jmp to %sfinish\n", comment, label_dot); } else { + const char *jmp_op; + if (op->cmp != VM_CMP_ALWAYS) { switch (dialect) { case AMD64_ATT: @@ -208,30 +241,11 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, case VM_OP_FETCH: { - bool r = op->u.fetch.end_bits == VM_END_SUCC; - - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $%d, %%%s\n", r, ret_reg); - break; - case AMD64_NASM: - fprintf(f, "\tMOV %s, %d\n", ret_reg, r); - break; - case AMD64_GO: - fprintf(f, "\tMOVQ $%d, %s\n", r, ret_reg); - break; - } - - if (r) { - /* endleaf in addition to existing instructions */ - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, - op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } + if (-1 == print_end(f, op, opt, + dialect, ret_reg, + op->u.fetch.end_bits)) + { + return -1; } switch (dialect) { diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index ba5c61995..b69203052 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -23,24 +23,10 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" -static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) count; - (void) leaf_opaque; - - fprintf(f, "return 0;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -59,6 +45,99 @@ cmp_operator(int cmp) } } +// TODO: centralise vmc/c +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) +{ + switch (ambig) { + case AMBIG_NONE: + // TODO: for C99 we'd return bool + fprintf(f, "return 1;"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "return %u;", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "return %u;", ids[0]); + break; + + case AMBIG_MULTIPLE: + /* + * Here I would like to emit (static unsigned []) { 1, 2, 3 } + * but specifying a storage duration for compound literals + * is a compiler extension. + * So I'm emitting a static const variable declaration instead. + */ + + fprintf(f, "{\n"); + fprintf(f, "\t\tstatic const unsigned a[] = { "); + for (fsm_end_id_t i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, " };\n"); + fprintf(f, "\t\t*ids = a;\n"); + fprintf(f, "\t\t*count = %zu;\n", count); + fprintf(f, "\t\treturn 0;\n"); + fprintf(f, "\t}"); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + if (-1 == print_ids(f, opt->ambig, ids, count)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "return 0;"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { @@ -87,23 +166,20 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "return 0;"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, default_reject, NULL); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + case VM_END_SUCC: + return print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "return 1;"); - } + default_accept, + NULL); - return 0; + default: + assert(!"unreached"); + abort(); + } } static void @@ -278,9 +354,7 @@ walk_sequence(struct dfavm_op_ir *op, /* TODO: eventually to be non-static */ static int fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, - const char *cp, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *cp) { struct dfavm_op_ir *op; @@ -288,11 +362,6 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, assert(opt != NULL); assert(cp != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; @@ -435,7 +504,6 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, int fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; /* TODO: currently unused, but must be non-NULL */ @@ -450,41 +518,69 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - if (opt->fragment) { - if (-1 == fsm_print_cfrag(f, opt, ops, cp, - leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_cfrag(f, opt, ops, cp)) { return -1; } } else { fprintf(f, "int\n%smain", prefix); + fprintf(f, "("); switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "(int (*fsm_getc)(void *opaque), void *getc_opaque)\n"); - fprintf(f, "{\n"); + fprintf(f, "int (*fsm_getc)(void *opaque), void *getc_opaque"); break; case FSM_IO_STR: - fprintf(f, "(const char *s)\n"); - fprintf(f, "{\n"); + fprintf(f, "const char *s"); break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e)\n"); - fprintf(f, "{\n"); + fprintf(f, "const char *b, const char *e"); break; + + default: + assert(!"unreached"); + abort(); } - if (-1 == fsm_print_cfrag(f, opt, ops, cp, - leaf, opt->leaf_opaque)) - { + /* + * unsigned rather than fsm_end_id_t here, so the generated code + * doesn't depend on fsm.h + */ + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(stdout, ",\n"); + fprintf(stdout, "\tconst unsigned *id"); + break; + + case AMBIG_MULTIPLE: + fprintf(stdout, ",\n"); + fprintf(stdout, "\tconst unsigned **ids, size_t *count"); + break; + + default: + assert(!"unreached"); + abort(); + } + + if (opt->hooks.args != NULL) { + fprintf(stdout, ",\n"); + fprintf(stdout, "\t"); + + if (-1 == print_hook_args(f, opt, NULL, NULL)) { + return -1; + } + } + + fprintf(f, ")\n"); + fprintf(f, "{\n"); + + if (-1 == fsm_print_cfrag(f, opt, ops, cp)) { return -1; } diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index c647cb988..90a8b0971 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -23,6 +23,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" @@ -44,6 +45,49 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "match"); + + if (count > 0) { + fprintf(f, " / "); + + for (size_t i = 0; i < count; i++) { + fprintf(f, "#%u", ids[i]); + + if (i < count - 1) { + fprintf(f, " "); + } + } + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + + fprintf(f, "fail"); + + return 0; +} + static int op_can_fallthrough(const struct dfavm_op_ir *op) { @@ -75,35 +119,20 @@ static int print_end(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "fail"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, default_reject, NULL); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + case VM_END_SUCC: + return print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) - { - return -1; - } - } else { - fprintf(f, "match"); - - if (op->endids.count > 0) { - fprintf(f, " / "); + default_accept, + NULL); - for (size_t i = 0; i < op->endids.count; i++) { - fprintf(f, "#%u", op->endids.ids[i]); - - if (i < op->endids.count - 1) { - fprintf(f, " "); - } - } - } + default: + assert(!"unreached"); + abort(); } - - return 0; } static void diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 2104edc51..4520dc702 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -23,6 +23,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "libfsm/vm/vm.h" @@ -32,23 +33,6 @@ enum vmops_dialect { VMOPS_MAIN, }; -static int -print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) count; - (void) leaf_opaque; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -66,6 +50,45 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) +{ + const char *prefix; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + prefix = lang_opaque; + + // TODO: print ids + (void) ids; + (void) count; + + fprintf(f, "%sactionRET, 1", prefix); + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + const char *prefix; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + prefix = lang_opaque; + + fprintf(f, "%sactionRET, 0", prefix); + + return 0; +} + static int print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { @@ -100,20 +123,28 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const char *prefix, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "%sactionRET, 0},\n", prefix); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + if (-1 == print_hook_reject(f, opt, default_reject, + (void *) prefix)) + { + return -1; + } + break; - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, + case VM_END_SUCC: + if (-1 == print_hook_accept(f, opt, op->endids.ids, op->endids.count, - opt->endleaf_opaque)) + default_accept, + (void *) prefix)) { return -1; } - } else { - fprintf(f, "%sactionRET, 1", prefix); + break; + + default: + assert(!"unreached"); + abort(); } fprintf(f, "},\n"); @@ -148,20 +179,13 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) /* TODO: eventually to be non-static */ static int fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, - const char *prefix, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *leaf_opaque) + const char *prefix) { struct dfavm_op_ir *op; assert(f != NULL); assert(opt != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { if (-1 == print_label(f, op, opt)) { @@ -209,7 +233,6 @@ int fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, enum vmops_dialect dialect) { - int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; assert(f != NULL); @@ -221,17 +244,9 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, prefix = "fsm_"; } - if (opt->leaf != NULL) { - leaf = opt->leaf; - } else { - leaf = print_leaf; - } - if (opt->fragment) { if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix, - leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix)) { return -1; } } @@ -243,9 +258,7 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, fprintf(f, "#include \"%svmops.h\"\n", prefix); fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix, - leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix)) { return -1; } fprintf(f, "\t};\n"); diff --git a/src/lx/print/c.c b/src/lx/print/c.c index a8ae67eec..bc5e5e91a 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -28,9 +28,7 @@ /* XXX: abstraction */ int fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), - const void *opaque); + const char *cp); static int skip(const struct fsm *fsm, fsm_state_t state) @@ -138,41 +136,23 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, } static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +accept_c(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct ast *ast; const struct ast_mapping *m; - ast = leaf_opaque; + ast = opt->hooks.hook_opaque; - assert(ast != NULL); - - if (ids == NULL) { - m = NULL; - } else { - assert(count > 0); - m = ast_getendmappingbyendid(ids[0]); - } - - if (m == NULL) { - /* XXX: don't need this if complete */ - switch (opt.io) { - case FSM_IO_GETC: - fprintf(f, "lx->lgetc = NULL; "); - break; - - case FSM_IO_STR: - fprintf(f, "lx->p = NULL; "); - break; - - case FSM_IO_PAIR: - fprintf(f, "lx->p = NULL; "); - break; - } + assert(f != NULL); + assert(opt != NULL); + assert(opt->hooks.hook_opaque != NULL); + assert(ids != NULL); + assert(count > 0); + assert(lang_opaque == NULL); - fprintf(f, "return %sUNKNOWN;", prefix.tok); - return 0; - } + m = ast_getendmappingbyendid(ids[0]); /* XXX: don't need this if complete */ fprintf(f, "%sungetc(lx, c); ", prefix.api); @@ -191,6 +171,37 @@ leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) return 0; } +static int +reject_c(FILE *f, const struct fsm_options *opt, + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(opt->hooks.hook_opaque != NULL); + assert(lang_opaque == NULL); + + (void) opt->hooks.hook_opaque; + + /* XXX: don't need this if complete */ + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "lx->lgetc = NULL; "); + break; + + case FSM_IO_STR: + fprintf(f, "lx->p = NULL; "); + break; + + case FSM_IO_PAIR: + fprintf(f, "lx->p = NULL; "); + break; + } + + fprintf(f, "return %sUNKNOWN;", prefix.tok); + + return 0; +} + static void print_proto(FILE *f, const struct ast *ast, const struct ast_zone *z) { @@ -393,14 +404,14 @@ print_io(FILE *f) fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); fprintf(f, "\t\tlx->end.col = 1;\n"); - if (opt.io == FSM_IO_STR) { /* ignore terminating '\0' */ - fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); - fprintf(f, "\t\tlx->end.byte--;\n"); - fprintf(f, "\t\tlx->end.col--;\n"); - fprintf(f, "\t}\n"); - } else { - fprintf(f, "\t}\n"); - } + if (opt.io == FSM_IO_STR) { /* ignore terminating '\0' */ + fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); + fprintf(f, "\t\tlx->end.byte--;\n"); + fprintf(f, "\t\tlx->end.col--;\n"); + fprintf(f, "\t}\n"); + } else { + fprintf(f, "\t}\n"); + } fprintf(f, "\n"); } fprintf(f, "\treturn c;\n"); @@ -683,14 +694,15 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) tmp = z->fsm->opt; - o.comments = z->fsm->opt->comments; - o.case_ranges = z->fsm->opt->case_ranges; - o.leaf = leaf; - o.leaf_opaque = (void *) ast; + o.comments = z->fsm->opt->comments; + o.case_ranges = z->fsm->opt->case_ranges; + o.hooks.accept = accept_c; + o.hooks.reject = reject_c; + o.hooks.hook_opaque = (void *) ast; z->fsm->opt = &o; - assert(opt.cp != NULL); + assert(opt.hooks.cp != NULL); ir = make_ir(z->fsm); if (ir == NULL) { @@ -698,8 +710,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } /* XXX: abstraction */ - (void) fsm_print_cfrag(f, ir, &o, opt.cp, - z->fsm->opt->leaf != NULL ? z->fsm->opt->leaf : leaf, z->fsm->opt->leaf_opaque); + (void) fsm_print_cfrag(f, ir, &o, opt.hooks.cp); free_ir(z->fsm, ir); @@ -754,7 +765,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\t\tdefault:\n"); fprintf(f, "\t\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.cp); + fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.hooks.cp); fprintf(f, "\t\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t\t}\n"); fprintf(f, "\t\t\t}\n"); @@ -765,7 +776,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } else { fprintf(f, "\n"); fprintf(f, "\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.cp); + fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.hooks.cp); fprintf(f, "\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t}\n"); fprintf(f, "\t\t}\n"); @@ -958,9 +969,9 @@ lx_print_c(FILE *f, const struct ast *ast) assert(ast != NULL); switch (opt.io) { - case FSM_IO_GETC: opt.cp = "c"; break; - case FSM_IO_STR: opt.cp = "c"; break; - case FSM_IO_PAIR: opt.cp = "c"; break; + case FSM_IO_GETC: opt.hooks.cp = "c"; break; + case FSM_IO_STR: opt.hooks.cp = "c"; break; + case FSM_IO_PAIR: opt.hooks.cp = "c"; break; } for (z = ast->zl; z != NULL; z = z->next) { @@ -1112,7 +1123,7 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\n"); - if (opt.io == FSM_IO_STR) { + if (opt.io == FSM_IO_STR) { fprintf(f, "void\n"); fprintf(f, "%sinput_str(struct %slx *lx, const char *p)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); @@ -1123,5 +1134,5 @@ lx_print_c(FILE *f, const struct ast *ast) } fprintf(f, "\tlx->p = p;\n"); fprintf(f, "}\n"); - } + } } diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index 00a84914b..d67a5e5ca 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -65,29 +65,31 @@ mapping(FILE *f, const struct ast_mapping *m, const struct ast *ast) } static int -endleaf_dot(FILE *f, const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) +accept_dot(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct ast_mapping *m; const struct ast *ast; + fsm_state_t s; assert(f != NULL); - assert(endleaf_opaque != NULL); - + assert(opt != NULL); + assert(opt->hooks.hook_opaque != NULL); assert(ids != NULL); assert(count > 0); - m = ast_getendmappingbyendid(ids[0]); + assert(lang_opaque != NULL); - ast = endleaf_opaque; + ast = opt->hooks.hook_opaque; + s = * (fsm_state_t *) lang_opaque; + + m = ast_getendmappingbyendid(ids[0]); fprintf(f, "label = <"); -/* TODO: would need the fsm for indexing here */ -#if 0 if (!anonymous_states) { fprintf(f, "%u
", s); } -#endif if (m->conflict != NULL) { const struct mapping_set *p; @@ -200,8 +202,8 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) opt.consolidate_edges = z->fsm->opt->consolidate_edges; opt.comments = z->fsm->opt->comments; opt.prefix = p; - opt.endleaf = endleaf_dot; - opt.endleaf_opaque = (void *) ast; + opt.hooks.accept = accept_dot; + opt.hooks.hook_opaque = (void *) ast; z->fsm->opt = &opt; diff --git a/src/re/main.c b/src/re/main.c index 2fca4ba19..d6ee29f85 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -426,18 +426,19 @@ printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) } static int -endleaf_c(FILE *f, - const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) +accept_c(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct match *m; unsigned n; size_t i; - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); - (void) f; - (void) endleaf_opaque; + (void) opt; + (void) lang_opaque; n = 0; @@ -467,18 +468,19 @@ endleaf_c(FILE *f, } static int -endleaf_rust(FILE *f, - const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) +accept_rust(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct match *m; unsigned n; size_t i; - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); - (void) f; - (void) endleaf_opaque; + (void) opt; + (void) lang_opaque; n = 0; @@ -508,28 +510,18 @@ endleaf_rust(FILE *f, } static int -endleaf_llvm(FILE *f, - const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) +accept_llvm(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct match *m; unsigned n; size_t i; - /* - * XXX: Hack for llvm codegen only, we don't have a way to pass this - * value from fsm_print_llvm(). We're working around that by making - * an assumption about the ordering for ret label numbers, that - * they're output sequentially. - * - * And also it's not very forward-thinking to keep this in static storage. - */ - static size_t ret = 0; - - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); - (void) f; - (void) endleaf_opaque; + (void) opt; n = 0; @@ -539,7 +531,9 @@ endleaf_llvm(FILE *f, } } - fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, ret++); + i = * (const size_t *) lang_opaque; + + fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, i); fprintf(f, " ; "); @@ -559,20 +553,31 @@ endleaf_llvm(FILE *f, } static int -endleaf_dot(FILE *f, - const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) +accept_dot(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct match *m; + fsm_state_t s; size_t i; - assert(f != NULL); - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + (void) opt; - (void) endleaf_opaque; + s = * (fsm_state_t *) lang_opaque; fprintf(f, "label = <"); + if (!opt->anonymous_states) { + fprintf(f, "%u", s); + + if (count > 0) { + fprintf(f, "
"); + } + } + for (i = 0; i < count; i++) { for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { fprintf(f, "#%u", m->id); @@ -585,40 +590,39 @@ endleaf_dot(FILE *f, fprintf(f, ">"); - /* TODO: only if comments */ /* TODO: centralise to libfsm/print/dot.c */ + if (opt->comments) { + fprintf(f, " # "); -#if 0 - fprintf(f, " # "); - - for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + for (i = 0; i < count; i++) { + for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { + fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ - if (m->next != NULL || i < count - 1) { - fprintf(f, ", "); + if (m->next != NULL || i < count - 1) { + fprintf(f, ", "); + } } } - } - fprintf(f, "\n"); -#endif + fprintf(f, "\n"); + } return 0; } static int -endleaf_json(FILE *f, - const fsm_end_id_t *ids, size_t count, - const void *endleaf_opaque) +accept_json(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque) { const struct match *m; size_t i; - assert(f != NULL); - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); - (void) endleaf_opaque; + (void) opt; + (void) lang_opaque; fprintf(f, "[ "); @@ -634,9 +638,6 @@ endleaf_json(FILE *f, fprintf(f, " ]"); - /* TODO: only if comments */ - /* TODO: centralise to libfsm/print/json.c */ - return 0; } @@ -1250,28 +1251,28 @@ main(int argc, char *argv[]) switch (fsm_lang) { case FSM_PRINT_C: case FSM_PRINT_VMC: - opt.endleaf = endleaf_c; + opt.hooks.accept = accept_c; break; case FSM_PRINT_RUST: - opt.endleaf = endleaf_rust; + opt.hooks.accept = accept_rust; break; case FSM_PRINT_LLVM: - opt.endleaf = endleaf_llvm; + opt.hooks.accept = accept_llvm; break; case FSM_PRINT_DOT: case FSM_PRINT_VMDOT: - opt.endleaf = patterns ? endleaf_dot : NULL; + opt.hooks.accept = patterns ? accept_dot : NULL; break; case FSM_PRINT_JSON: - opt.endleaf = patterns ? endleaf_json : NULL; + opt.hooks.accept = patterns ? accept_json : NULL; break; default: - opt.endleaf = NULL; + opt.hooks.accept = NULL; break; } From b4538cf587bd1a6ecde9dfeedc39ec20a6063eaa Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 Jul 2024 10:17:59 +0100 Subject: [PATCH 205/373] Cruft. --- include/fsm/fsm.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index de5de136e..1d462f2a1 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -7,14 +7,6 @@ #ifndef FSM_H #define FSM_H -/* - * TODO: This API needs quite some refactoring. Mostly we ought to operate - * in-place, else the user would only free() everything. Having an explicit - * clone interface leaves the option for duplicating, if they wish. However be - * careful about leaving things in an indeterminate state; everything ought to - * be atomic. - */ - #include struct fsm; From 61fb10edcaac3559ce915638f0bc887d054e6d3e Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 19 Jul 2024 13:52:07 +0100 Subject: [PATCH 206/373] Move alloc hooks to struct fsm. This is a step towards moving fsm_options to the print interfaces only. --- fuzz/target.c | 8 +- include/fsm/alloc.h | 5 + include/fsm/fsm.h | 5 +- include/fsm/options.h | 4 - include/fsm/parser.h | 5 +- include/re/re.h | 2 + include/re/strings.h | 9 +- src/fsm/main.c | 15 +- src/libfsm/capture.c | 22 +- src/libfsm/charset.c | 4 +- src/libfsm/clone.c | 8 +- src/libfsm/closure.c | 12 +- src/libfsm/concat.c | 5 + src/libfsm/consolidate.c | 17 +- src/libfsm/determinise.c | 24 +- src/libfsm/edge.c | 4 +- src/libfsm/endids.c | 16 +- src/libfsm/epsilons.c | 18 +- src/libfsm/equal.c | 5 + src/libfsm/example.c | 2 +- src/libfsm/fsm.c | 61 ++-- src/libfsm/gen.c | 2 +- src/libfsm/internal.h | 1 + src/libfsm/merge.c | 9 +- src/libfsm/mergestates.c | 4 +- src/libfsm/minimise.c | 44 +-- src/libfsm/parser.act | 10 +- src/libfsm/parser.c | 75 ++-- src/libfsm/parser.h | 11 +- src/libfsm/pred/hasnondeterminism.c | 2 +- src/libfsm/print/api.c | 8 +- src/libfsm/print/dot.c | 6 +- src/libfsm/print/fsm.c | 4 +- src/libfsm/print/ir.c | 54 +-- src/libfsm/print/json.c | 6 +- src/libfsm/reverse.c | 32 +- src/libfsm/shortest.c | 20 +- src/libfsm/shuffle.c | 4 +- src/libfsm/state.c | 14 +- src/libfsm/trim.c | 30 +- src/libfsm/union.c | 5 + src/libfsm/vacuum.c | 2 +- src/libfsm/walk/reachable.c | 4 +- src/libfsm/walk2.c | 23 +- src/libre/ast_compile.c | 15 +- src/libre/ast_compile.h | 3 + src/libre/ast_rewrite.c | 15 +- src/libre/re.c | 3 +- src/libre/re_strings.c | 13 +- src/lx/ast.c | 4 +- src/lx/main.c | 8 +- src/lx/parser.act | 19 +- src/lx/parser.c | 344 +++++++++--------- src/lx/parser.h | 11 +- src/re/main.c | 22 +- src/retest/cvtpcre.c | 9 +- src/retest/main.c | 11 +- src/retest/reperf.c | 5 +- tests/aho_corasick/actest.c | 2 +- tests/capture/captest.c | 14 +- tests/capture/captest.h | 3 - tests/capture/capture4.c | 2 +- tests/capture/capture5.c | 2 +- tests/endids/endids0.c | 2 +- tests/endids/endids0_many_endids.c | 2 +- .../endids10_minimise_partial_overlap.c | 4 +- tests/endids/endids1_determinise.c | 2 +- .../endids/endids1_determinise_and_minimise.c | 2 +- tests/endids/endids2_union.c | 4 +- tests/endids/endids2_union_many_endids.c | 2 +- tests/endids/endids4.c | 4 +- tests/endids/endids5.c | 4 +- tests/endids/endids6.c | 4 +- tests/endids/endids7.c | 4 +- tests/endids/endids7_with_duplicates.c | 4 +- tests/endids/endids8.c | 4 +- tests/endids/endids9.c | 4 +- tests/gen/gtest.c | 5 +- tests/minimise/minimise_test_case_list.c | 2 +- tests/re_strings/testutil.c | 4 +- 80 files changed, 642 insertions(+), 541 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 6bb6f2772..c9d5d4a6e 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -95,7 +95,7 @@ build(const char *pattern) }; time_get(&pre); - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); time_get(&post); delta_usec = time_diff_usec(&pre, &post); total_usec += delta_usec; @@ -176,7 +176,7 @@ shuffle_minimise(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ @@ -283,11 +283,11 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const .offset = 0 }; - const struct fsm_options options = { + const struct fsm_options opt = { .io = io_mode, }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &options, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ return EXIT_SUCCESS; diff --git a/include/fsm/alloc.h b/include/fsm/alloc.h index ea9c9a885..4388629a0 100644 --- a/include/fsm/alloc.h +++ b/include/fsm/alloc.h @@ -7,6 +7,8 @@ #ifndef FSM_ALLOC_H #define FSM_ALLOC_H +struct fsm; + struct fsm_alloc { void (*free)(void *opaque, void *p); void *(*calloc)(void *opaque, size_t n, size_t sz); @@ -15,5 +17,8 @@ struct fsm_alloc { void *opaque; }; +void +fsm_set_alloc(struct fsm *fsm, const struct fsm_alloc *alloc); + #endif diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 1d462f2a1..c08b16549 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -10,6 +10,7 @@ #include struct fsm; +struct fsm_alloc; struct fsm_options; struct path; /* XXX */ struct fsm_capture; @@ -44,7 +45,7 @@ typedef unsigned int fsm_end_id_t; * TODO: also fsm_parse should create an FSM, not add into an existing one */ struct fsm * -fsm_new(const struct fsm_options *opt); +fsm_new(const struct fsm_alloc *alloc); /* * As fsm_new(), but with an explicit number of pre-allocated states. @@ -53,7 +54,7 @@ fsm_new(const struct fsm_options *opt); * You can still add more states per usual. */ struct fsm * -fsm_new_statealloc(const struct fsm_options *opt, size_t statealloc); +fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc); /* * Free a structure created by fsm_new(), and all of its contents. diff --git a/include/fsm/options.h b/include/fsm/options.h index a5e090992..5254ce2a9 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -10,7 +10,6 @@ #include struct fsm; -struct fsm_alloc; struct fsm_state; enum fsm_io { @@ -106,9 +105,6 @@ struct fsm_options { void *hook_opaque; } hooks; - - /* custom allocation functions */ - const struct fsm_alloc *alloc; }; #endif diff --git a/include/fsm/parser.h b/include/fsm/parser.h index 414126926..9c512ae50 100644 --- a/include/fsm/parser.h +++ b/include/fsm/parser.h @@ -1,12 +1,11 @@ #ifndef FSM_PARSER_H #define FSM_PARSER_H -#include - struct fsm; +struct fsm_alloc; struct fsm_options; struct fsm * -fsm_parse(FILE *, const struct fsm_options *); +fsm_parse(FILE *, const struct fsm_alloc *alloc, const struct fsm_options *); #endif diff --git a/include/re/re.h b/include/re/re.h index deab6caed..33d32b940 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -8,6 +8,7 @@ #define RE_H struct fsm; +struct fsm_alloc; struct fsm_options; enum re_dialect { @@ -133,6 +134,7 @@ re_getchar_fun(void *opaque); struct fsm * re_comp(enum re_dialect dialect, re_getchar_fun *f, void *opaque, + const struct fsm_alloc *alloc, const struct fsm_options *opt, enum re_flags flags, struct re_err *err); diff --git a/include/re/strings.h b/include/re/strings.h index fe4f7adc2..1c43e2e95 100644 --- a/include/re/strings.h +++ b/include/re/strings.h @@ -8,6 +8,8 @@ #define RE_STRINGS_H #include + +struct fsm_alloc; struct fsm_options; struct re_strings; @@ -32,7 +34,8 @@ enum re_strings_flags { }; struct fsm * -re_strings(const struct fsm_options *opt, const char *a[], size_t n, +re_strings(const struct fsm_alloc *alloc, const struct fsm_options *opt, + const char *a[], size_t n, enum re_strings_flags flags); struct re_strings * @@ -49,7 +52,9 @@ re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endi struct fsm * re_strings_build(struct re_strings *g, - const struct fsm_options *opt, enum re_strings_flags flags); + const struct fsm_alloc *alloc, + const struct fsm_options *opt, + enum re_strings_flags flags); #endif diff --git a/src/fsm/main.c b/src/fsm/main.c index 80540ee16..084b147aa 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -60,8 +60,6 @@ static int clock_gettime(int clk_id, struct timespec *ts) extern int optind; extern char *optarg; -static struct fsm_options opt; - #define OP_ARITY 0x1 enum op { @@ -365,10 +363,16 @@ do_fsm_cleanup(void) int main(int argc, char *argv[]) { + static const struct fsm_options zero_options; + + /* TODO: use alloc hooks for -Q accounting */ + struct fsm_alloc *alloc = NULL; + unsigned iterations, i; enum fsm_print_lang lang; enum op op; const char *charset; + struct fsm_options opt; struct fsm *fsm; double elapsed; int xfiles; @@ -381,6 +385,7 @@ main(int argc, char *argv[]) atexit(do_fsm_cleanup); + opt = zero_options; opt.comments = 1; opt.io = FSM_IO_GETC; @@ -480,7 +485,7 @@ main(int argc, char *argv[]) if ((op & OP_ARITY) == 1) { /* argc < 1 is okay */ - q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), &opt); + q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), alloc, &opt); if (q == NULL) { exit(EXIT_FAILURE); } @@ -490,12 +495,12 @@ main(int argc, char *argv[]) exit(EXIT_FAILURE); } - a = fsm_parse(xopen(argv[0]), &opt); + a = fsm_parse(xopen(argv[0]), alloc, &opt); if (a == NULL) { exit(EXIT_FAILURE); } - b = fsm_parse(xopen(argv[1]), &opt); + b = fsm_parse(xopen(argv[1]), alloc, &opt); if (b == NULL) { exit(EXIT_FAILURE); } diff --git a/src/libfsm/capture.c b/src/libfsm/capture.c index 806bb3b12..3f750169e 100644 --- a/src/libfsm/capture.c +++ b/src/libfsm/capture.c @@ -14,7 +14,7 @@ fsm_capture_init(struct fsm *fsm) struct fsm_capture_info *ci = NULL; size_t i; - ci = f_calloc(fsm->opt->alloc, + ci = f_calloc(fsm->alloc, 1, sizeof(*ci)); if (ci == NULL) { goto cleanup; @@ -29,7 +29,7 @@ fsm_capture_init(struct fsm *fsm) cleanup: if (ci != NULL) { - f_free(fsm->opt->alloc, ci); + f_free(fsm->alloc, ci); } return 0; } @@ -41,8 +41,8 @@ fsm_capture_free(struct fsm *fsm) if (ci == NULL) { return; } - f_free(fsm->opt->alloc, ci->buckets); - f_free(fsm->opt->alloc, ci); + f_free(fsm->alloc, ci->buckets); + f_free(fsm->alloc, ci); fsm->capture_info = NULL; } @@ -133,7 +133,7 @@ fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, env.trail = NULL; env.seen = NULL; - env.trail = f_malloc(fsm->opt->alloc, + env.trail = f_malloc(fsm->alloc, DEF_TRAIL_CEIL * sizeof(env.trail[0])); if (env.trail == NULL) { goto cleanup; @@ -141,7 +141,7 @@ fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, env.trail_ceil = DEF_TRAIL_CEIL; seen_words = fsm->statecount/64 + 1; - env.seen = f_malloc(fsm->opt->alloc, + env.seen = f_malloc(fsm->alloc, seen_words * sizeof(env.seen[0])); if (!mark_capture_path(&env)) { @@ -156,8 +156,8 @@ fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, /* fall through */ cleanup: - f_free(fsm->opt->alloc, env.trail); - f_free(fsm->opt->alloc, env.seen); + f_free(fsm->alloc, env.trail); + f_free(fsm->alloc, env.seen); return res; } @@ -177,7 +177,7 @@ init_capture_action_htab(struct fsm *fsm, struct fsm_capture_info *ci) assert(ci->buckets_used == 0); count = DEF_CAPTURE_ACTION_BUCKET_COUNT; - ci->buckets = f_malloc(fsm->opt->alloc, + ci->buckets = f_malloc(fsm->alloc, count * sizeof(ci->buckets[0])); if (ci->buckets == NULL) { return 0; @@ -310,7 +310,7 @@ add_capture_action(struct fsm *fsm, struct fsm_capture_info *ci, return 0; } } else if (ci->buckets_used >= ci->bucket_count/2) { /* grow */ - if (!grow_capture_action_buckets(fsm->opt->alloc, ci)) { + if (!grow_capture_action_buckets(fsm->alloc, ci)) { return 0; } } @@ -404,7 +404,7 @@ grow_trail(struct capture_set_path_env *env) nceil = 2 * env->trail_ceil; assert(nceil > env->trail_ceil); - ntrail = f_realloc(env->fsm->opt->alloc, env->trail, + ntrail = f_realloc(env->fsm->alloc, env->trail, nceil * sizeof(env->trail[0])); if (ntrail == NULL) { return 0; diff --git a/src/libfsm/charset.c b/src/libfsm/charset.c index 662c0ac1f..e7c75fd88 100644 --- a/src/libfsm/charset.c +++ b/src/libfsm/charset.c @@ -36,11 +36,13 @@ fsm_intersect_charset(struct fsm *a, size_t n, const char *charset) { fsm_state_t state; - b = fsm_new_statealloc(a->opt, 1); + b = fsm_new_statealloc(a->alloc, 1); if (b == NULL) { return NULL; } + fsm_setoptions(b, a->opt); + if (!fsm_addstate(b, &state)) { goto error; } diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 69aedaa63..897fe027a 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -35,11 +35,13 @@ fsm_clone(const struct fsm *fsm) assert(fsm != NULL); assert(fsm->opt != NULL); - new = fsm_new_statealloc(fsm->opt, fsm->statecount); + new = fsm_new_statealloc(fsm->alloc, fsm->statecount); if (new == NULL) { return NULL; } + fsm_setoptions(new, fsm->opt); + if (!fsm_addstate_bulk(new, fsm->statecount)) { fsm_free(new); return NULL; @@ -50,12 +52,12 @@ fsm_clone(const struct fsm *fsm) fsm_setend(new, i, 1); } - if (!state_set_copy(&new->states[i].epsilons, new->opt->alloc, fsm->states[i].epsilons)) { + if (!state_set_copy(&new->states[i].epsilons, new->alloc, fsm->states[i].epsilons)) { fsm_free(new); return NULL; } - if (!edge_set_copy(&new->states[i].edges, new->opt->alloc, fsm->states[i].edges)) { + if (!edge_set_copy(&new->states[i].edges, new->alloc, fsm->states[i].edges)) { fsm_free(new); return NULL; } diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index fa2d0c783..a9902b6e4 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -32,7 +32,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s assert(closures != NULL); assert(s < fsm->statecount); - q = queue_new(fsm->opt->alloc, fsm->statecount); + q = queue_new(fsm->alloc, fsm->statecount); if (q == NULL) { return 1; } @@ -72,7 +72,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s * TODO: use partially-constructed (i.e. not readable) sets, and finalize later */ if (p != s && closures[p] != NULL) { - if (!state_set_copy(&closures[s], fsm->opt->alloc, closures[p])) { + if (!state_set_copy(&closures[s], fsm->alloc, closures[p])) { goto error; } } @@ -89,7 +89,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s * multiple threads repeating some of the same work here. */ - if (!state_set_add(&closures[s], fsm->opt->alloc, p)) { + if (!state_set_add(&closures[s], fsm->alloc, p)) { goto error; } @@ -136,7 +136,7 @@ epsilon_closure(struct fsm *fsm) assert(fsm != NULL); assert(fsm->statecount > 0); - closures = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *closures); + closures = f_malloc(fsm->alloc, fsm->statecount * sizeof *closures); if (closures == NULL) { return NULL; } @@ -153,7 +153,7 @@ epsilon_closure(struct fsm *fsm) * avoid the bookkeeping overhead needed for general case. */ if (fsm->states[s].epsilons == NULL) { - if (!state_set_add(&closures[s], fsm->opt->alloc, s)) { + if (!state_set_add(&closures[s], fsm->alloc, s)) { goto error; } } @@ -184,7 +184,7 @@ epsilon_closure(struct fsm *fsm) state_set_free(closures[s]); } - f_free(fsm->opt->alloc, closures); + f_free(fsm->alloc, closures); return NULL; } diff --git a/src/libfsm/concat.c b/src/libfsm/concat.c index 1892ae4ed..980159f12 100644 --- a/src/libfsm/concat.c +++ b/src/libfsm/concat.c @@ -33,6 +33,11 @@ fsm_concat(struct fsm *a, struct fsm *b, assert(a != NULL); assert(b != NULL); + if (a->alloc != b->alloc) { + errno = EINVAL; + return NULL; + } + if (a->opt != b->opt) { errno = EINVAL; return NULL; diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index a000a4c7b..79a7d7f01 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -68,13 +68,14 @@ fsm_consolidate(const struct fsm *src, size_t max_used = 0; assert(src != NULL); - assert(src->opt != NULL); - dst = fsm_new(src->opt); + dst = fsm_new(src->alloc); if (dst == NULL) { goto cleanup; } + fsm_setoptions(dst, src->opt); + for (src_i = 0; src_i < mapping_count; src_i++) { #if LOG_MAPPING fprintf(stderr, "consolidate_mapping[%u]: %u\n", @@ -90,7 +91,7 @@ fsm_consolidate(const struct fsm *src, } assert(dst->statecount == max_used + 1); - seen = f_calloc(src->opt->alloc, + seen = f_calloc(src->alloc, mapping_count/64 + 1, sizeof(seen[0])); if (seen == NULL) { goto cleanup; @@ -114,19 +115,19 @@ fsm_consolidate(const struct fsm *src, SET_DST_SEEN(dst_i); if (!state_set_copy(&dst->states[dst_i].epsilons, - dst->opt->alloc, src->states[src_i].epsilons)) { + dst->alloc, src->states[src_i].epsilons)) { goto cleanup; } state_set_compact(&dst->states[dst_i].epsilons, mapping_cb, &closure); if (!edge_set_copy(&dst->states[dst_i].edges, - dst->opt->alloc, + dst->alloc, src->states[src_i].edges)) { goto cleanup; } edge_set_compact(&dst->states[dst_i].edges, - dst->opt->alloc, mapping_cb, &closure); + dst->alloc, mapping_cb, &closure); if (fsm_isend(src, src_i)) { fsm_setend(dst, dst_i, 1); @@ -151,13 +152,13 @@ fsm_consolidate(const struct fsm *src, } } - f_free(src->opt->alloc, seen); + f_free(src->alloc, seen); return dst; cleanup: - if (seen != NULL) { f_free(src->opt->alloc, seen); } + if (seen != NULL) { f_free(src->alloc, seen); } return NULL; } diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index d89259908..731bc5aef 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -31,7 +31,7 @@ fsm_determinise(struct fsm *nfa) struct analyze_closures_env ac_env = { 0 }; assert(nfa != NULL); - map.alloc = nfa->opt->alloc; + map.alloc = nfa->alloc; /* * This NFA->DFA implementation is for epsilon-free NFA only. This keeps @@ -50,7 +50,7 @@ fsm_determinise(struct fsm *nfa) fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif - issp = interned_state_set_pool_alloc(nfa->opt->alloc); + issp = interned_state_set_pool_alloc(nfa->alloc); if (issp == NULL) { return 0; } @@ -95,12 +95,12 @@ fsm_determinise(struct fsm *nfa) * Our "todo" list. It needn't be a stack; we treat it as an unordered * set where we can consume arbitrary items in turn. */ - stack = stack_init(nfa->opt->alloc); + stack = stack_init(nfa->alloc); if (stack == NULL) { goto cleanup; } - ac_env.alloc = nfa->opt->alloc; + ac_env.alloc = nfa->alloc; ac_env.fsm = nfa; ac_env.issp = issp; @@ -118,7 +118,7 @@ fsm_determinise(struct fsm *nfa) goto cleanup; } - if (!edge_set_advise_growth(&curr->edges, nfa->opt->alloc, ac_env.output_count)) { + if (!edge_set_advise_growth(&curr->edges, nfa->alloc, ac_env.output_count)) { goto cleanup; } @@ -165,7 +165,7 @@ fsm_determinise(struct fsm *nfa) fprintf(stderr, "] -> dfastate %zu on output state %zu\n", m->dfastate, curr->dfastate); #endif - if (!edge_set_add_bulk(&curr->edges, nfa->opt->alloc, output->labels, m->dfastate)) { + if (!edge_set_add_bulk(&curr->edges, nfa->alloc, output->labels, m->dfastate)) { goto cleanup; } } @@ -180,11 +180,13 @@ fsm_determinise(struct fsm *nfa) struct mapping *m; struct fsm *dfa; - dfa = fsm_new(nfa->opt); + dfa = fsm_new(nfa->alloc); if (dfa == NULL) { goto cleanup; } + fsm_setoptions(dfa, nfa->opt); + #if DUMP_MAPPING { fprintf(stderr, "#### fsm_determinise: mapping\n"); @@ -661,7 +663,7 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, * We could probably filter this somehow, at the very least by * checking reachability from every X, but the actual path * handling later will also check reachability. */ - reverse_mappings = f_calloc(dst_dfa->opt->alloc, src_nfa->statecount, sizeof(reverse_mappings[0])); + reverse_mappings = f_calloc(dst_dfa->alloc, src_nfa->statecount, sizeof(reverse_mappings[0])); if (reverse_mappings == NULL) { return 0; } @@ -675,7 +677,7 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, ss = interned_state_set_get_state_set(issp, iss_id); for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { - if (!add_reverse_mapping(dst_dfa->opt->alloc, + if (!add_reverse_mapping(dst_dfa->alloc, reverse_mappings, m->dfastate, state)) { goto cleanup; @@ -705,10 +707,10 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, cleanup: for (i = 0; i < src_nfa->statecount; i++) { if (reverse_mappings[i].list != NULL) { - f_free(dst_dfa->opt->alloc, reverse_mappings[i].list); + f_free(dst_dfa->alloc, reverse_mappings[i].list); } } - f_free(dst_dfa->opt->alloc, reverse_mappings); + f_free(dst_dfa->alloc, reverse_mappings); return res; } diff --git a/src/libfsm/edge.c b/src/libfsm/edge.c index d48ca141c..379a19752 100644 --- a/src/libfsm/edge.c +++ b/src/libfsm/edge.c @@ -24,7 +24,7 @@ fsm_addedge_epsilon(struct fsm *fsm, assert(from < fsm->statecount); assert(to < fsm->statecount); - if (!state_set_add(&fsm->states[from].epsilons, fsm->opt->alloc, to)) { + if (!state_set_add(&fsm->states[from].epsilons, fsm->alloc, to)) { return 0; } @@ -60,7 +60,7 @@ fsm_addedge_literal(struct fsm *fsm, assert(from < fsm->statecount); assert(to < fsm->statecount); - if (!edge_set_add(&fsm->states[from].edges, fsm->opt->alloc, (unsigned char)c, to)) { + if (!edge_set_add(&fsm->states[from].edges, fsm->alloc, (unsigned char)c, to)) { return 0; } diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index c86277865..b93f6f9f1 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -87,15 +87,15 @@ fsm_endid_init(struct fsm *fsm) { struct endid_info_bucket *buckets = NULL; size_t i; - struct endid_info *res = f_calloc(fsm->opt->alloc, 1, sizeof(*res)); + struct endid_info *res = f_calloc(fsm->alloc, 1, sizeof(*res)); if (res == NULL) { return 0; } - buckets = f_malloc(fsm->opt->alloc, + buckets = f_malloc(fsm->alloc, DEF_BUCKET_COUNT * sizeof(buckets[0])); if (buckets == NULL) { - f_free(fsm->opt->alloc, res); + f_free(fsm->alloc, res); return 0; } @@ -131,10 +131,10 @@ fsm_endid_free(struct fsm *fsm) if (b->state == BUCKET_NO_STATE) { continue; } - f_free(fsm->opt->alloc, fsm->endid_info->buckets[i].ids); + f_free(fsm->alloc, fsm->endid_info->buckets[i].ids); } - f_free(fsm->opt->alloc, fsm->endid_info->buckets); - f_free(fsm->opt->alloc, fsm->endid_info); + f_free(fsm->alloc, fsm->endid_info->buckets); + f_free(fsm->alloc, fsm->endid_info); } static int @@ -236,7 +236,7 @@ endid_find_bucket(struct fsm *fsm, fsm_state_t state) LOG_2(" -- growing buckets [%u/%u used], then rehashing\n", ei->buckets_used, ei->bucket_count); - if (!grow_endid_buckets(fsm->opt->alloc, ei)) { + if (!grow_endid_buckets(fsm->alloc, ei)) { return NULL; } has_grown = 1; @@ -364,7 +364,7 @@ allocate_ids(const struct fsm *fsm, struct end_info_ids *prev, size_t n) assert(n > 0); id_alloc_size = sizeof(*ids) + (n - 1) * sizeof(ids->ids[0]); - return f_realloc(fsm->opt->alloc, prev, id_alloc_size); + return f_realloc(fsm->alloc, prev, id_alloc_size); } int diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index a5bb4da48..8dc049eba 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -134,7 +134,7 @@ fsm_remove_epsilons(struct fsm *nfa) fprintf(stderr, "%s: bulk-copying edges leading to state %d onto state %d (from state %d)\n", __func__, info.to, s, es_id); #endif - if (!edge_set_add_bulk(&nfa->states[s].edges, nfa->opt->alloc, + if (!edge_set_add_bulk(&nfa->states[s].edges, nfa->alloc, info.symbols, info.to)) { goto cleanup; } @@ -182,10 +182,10 @@ remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) struct state_iter si; fsm_state_t si_s; struct remap_env env = { 'R', NULL, NULL, 1, 0, 0, NULL }; - env.alloc = nfa->opt->alloc; + env.alloc = nfa->alloc; /* build a reverse mapping */ - rmap = f_calloc(nfa->opt->alloc, nfa->statecount, sizeof(rmap[0])); + rmap = f_calloc(nfa->alloc, nfa->statecount, sizeof(rmap[0])); if (rmap == NULL) { goto cleanup; } @@ -200,7 +200,7 @@ remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) fprintf(stderr, "remap_capture_actions: %u <- %u\n", s, si_s); #endif - if (!state_set_add(&rmap[si_s], nfa->opt->alloc, s)) { + if (!state_set_add(&rmap[si_s], nfa->alloc, s)) { goto cleanup; } } @@ -229,14 +229,14 @@ remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) cleanup: if (env.actions != NULL) { - f_free(nfa->opt->alloc, env.actions); + f_free(nfa->alloc, env.actions); } if (rmap != NULL) { for (i = 0; i < nfa->statecount; i++) { state_set_free(rmap[i]); } - f_free(nfa->opt->alloc, rmap); + f_free(nfa->alloc, rmap); } return res; @@ -381,10 +381,10 @@ carry_endids(struct fsm *fsm, struct state_set *states, struct collect_env env; env.tag = 'E'; /* for fsm_remove_epsilons */ - env.alloc = fsm->opt->alloc; + env.alloc = fsm->alloc; env.count = 0; env.ceil = DEF_CARRY_ENDIDS_COUNT; - env.ids = f_malloc(fsm->opt->alloc, + env.ids = f_malloc(fsm->alloc, env.ceil * sizeof(*env.ids)); if (env.ids == NULL) { return 0; @@ -412,7 +412,7 @@ carry_endids(struct fsm *fsm, struct state_set *states, } cleanup: - f_free(fsm->opt->alloc, env.ids); + f_free(fsm->alloc, env.ids); return env.ok; } diff --git a/src/libfsm/equal.c b/src/libfsm/equal.c index f683a95e2..21dd12e3a 100644 --- a/src/libfsm/equal.c +++ b/src/libfsm/equal.c @@ -121,6 +121,11 @@ fsm_equal(const struct fsm *a, const struct fsm *b) assert(a != NULL); assert(b != NULL); + if (a->alloc != b->alloc) { + errno = EINVAL; + return -1; + } + if (a->opt != b->opt) { errno = EINVAL; return -1; diff --git a/src/libfsm/example.c b/src/libfsm/example.c index ca9ce9571..9b30ad91f 100644 --- a/src/libfsm/example.c +++ b/src/libfsm/example.c @@ -92,7 +92,7 @@ fsm_example(const struct fsm *fsm, fsm_state_t goal, bufsz--; } - path_free(fsm->opt->alloc, path); + path_free(fsm->alloc, path); return n; } diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index 0e46962d1..e99e7ba96 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -34,61 +34,61 @@ free_contents(struct fsm *fsm) for (i = 0; i < fsm->statecount; i++) { state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); } fsm_capture_free(fsm); fsm_endid_free(fsm); - f_free(fsm->opt->alloc, fsm->states); + f_free(fsm->alloc, fsm->states); } struct fsm * -fsm_new_statealloc(const struct fsm_options *opt, size_t statealloc) +fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc) { - static const struct fsm_options defaults; - struct fsm *new, f; + struct fsm *new; - if (statealloc == 0) { - return fsm_new(opt); + if (alloc != NULL) { + assert(alloc->free != NULL); + assert(alloc->calloc != NULL); + assert(alloc->malloc != NULL); + assert(alloc->realloc != NULL); } - if (opt == NULL) { - opt = &defaults; + if (statealloc == 0) { + return fsm_new(alloc); } - f.opt = opt; - - new = f_malloc(f.opt->alloc, sizeof *new); + new = f_malloc(alloc, sizeof *new); if (new == NULL) { return NULL; } - new->statealloc = statealloc; - new->statecount = 0; - new->endcount = 0; + new->alloc = alloc; + new->opt = NULL; + new->statealloc = statealloc; + new->statecount = 0; + new->endcount = 0; new->capture_info = NULL; - new->endid_info = NULL; + new->endid_info = NULL; - new->states = f_malloc(f.opt->alloc, new->statealloc * sizeof *new->states); + new->states = f_malloc(new->alloc, new->statealloc * sizeof *new->states); if (new->states == NULL) { - f_free(f.opt->alloc, new); + f_free(new->alloc, new); return NULL; } fsm_clearstart(new); - new->opt = opt; - if (!fsm_capture_init(new)) { - f_free(f.opt->alloc, new->states); - f_free(f.opt->alloc, new); + f_free(new->alloc, new->states); + f_free(new->alloc, new); return NULL; } if (!fsm_endid_init(new)) { - f_free(f.opt->alloc, new->states); - f_free(f.opt->alloc, new); + f_free(new->alloc, new->states); + f_free(new->alloc, new); fsm_capture_free(new); return NULL; } @@ -97,9 +97,9 @@ fsm_new_statealloc(const struct fsm_options *opt, size_t statealloc) } struct fsm * -fsm_new(const struct fsm_options *opt) +fsm_new(const struct fsm_alloc *alloc) { - return fsm_new_statealloc(opt, FSM_DEFAULT_STATEALLOC); + return fsm_new_statealloc(alloc, FSM_DEFAULT_STATEALLOC); } void @@ -109,7 +109,7 @@ fsm_free(struct fsm *fsm) free_contents(fsm); - f_free(fsm->opt->alloc, fsm); + f_free(fsm->alloc, fsm); } const struct fsm_options * @@ -130,6 +130,11 @@ fsm_move(struct fsm *dst, struct fsm *src) assert(src != NULL); assert(dst != NULL); + if (dst->alloc != src->alloc) { + errno = EINVAL; + return; /* XXX */ + } + if (dst->opt != src->opt) { errno = EINVAL; return; /* XXX */ @@ -147,7 +152,7 @@ fsm_move(struct fsm *dst, struct fsm *src) dst->capture_info = src->capture_info; dst->endid_info = src->endid_info; - f_free(src->opt->alloc, src); + f_free(src->alloc, src); } unsigned int diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 89597f7ae..730b40724 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -211,7 +211,7 @@ gen_init_outer(struct fsm *fsm, size_t max_length, } struct gen_ctx ctx = { - .alloc = fsm->opt->alloc, + .alloc = fsm->alloc, .fsm = fsm, .max_length = max_length, .cb = cb, diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 6e77510a7..3ab8545ef 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -65,6 +65,7 @@ struct fsm_state { struct fsm { struct fsm_state *states; /* array */ + const struct fsm_alloc *alloc; size_t statealloc; /* number of elements allocated */ size_t statecount; /* number of elements populated */ diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index 69834a5bd..b7e8a5993 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -53,7 +53,7 @@ merge(struct fsm *dst, struct fsm *src, /* TODO: round up to next power of two here? * or let realloc do that internally */ - tmp = f_realloc(dst->opt->alloc, dst->states, newalloc * sizeof *dst->states); + tmp = f_realloc(dst->alloc, dst->states, newalloc * sizeof *dst->states); if (tmp == NULL) { return NULL; } @@ -110,7 +110,7 @@ merge(struct fsm *dst, struct fsm *src, return NULL; } - f_free(src->opt->alloc, src->states); + f_free(src->alloc, src->states); src->states = NULL; src->statealloc = 0; src->statecount = 0; @@ -199,6 +199,11 @@ fsm_mergeab(struct fsm *a, struct fsm *b, assert(b != NULL); assert(base_b != NULL); + if (a->alloc != b->alloc) { + errno = EINVAL; + return NULL; + } + if (a->opt != b->opt) { errno = EINVAL; return NULL; diff --git a/src/libfsm/mergestates.c b/src/libfsm/mergestates.c index d525cd7ed..77d5e2f6c 100644 --- a/src/libfsm/mergestates.c +++ b/src/libfsm/mergestates.c @@ -28,10 +28,10 @@ fsm_mergestates(struct fsm *fsm, fsm_state_t a, fsm_state_t b, assert(b < fsm->statecount); /* edges from b */ - if (!state_set_copy(&fsm->states[a].epsilons, fsm->opt->alloc, fsm->states[b].epsilons)) { + if (!state_set_copy(&fsm->states[a].epsilons, fsm->alloc, fsm->states[b].epsilons)) { return 0; } - if (!edge_set_copy(&fsm->states[a].edges, fsm->opt->alloc, fsm->states[b].edges)) { + if (!edge_set_copy(&fsm->states[a].edges, fsm->alloc, fsm->states[b].edges)) { return 0; } diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 4ba32d297..576dfeeee 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -100,7 +100,7 @@ fsm_minimise(struct fsm *fsm) goto cleanup; } - mapping = f_malloc(fsm->opt->alloc, + mapping = f_malloc(fsm->alloc, fsm->statecount * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; @@ -165,10 +165,10 @@ fsm_minimise(struct fsm *fsm) cleanup: if (mapping != NULL) { - f_free(fsm->opt->alloc, mapping); + f_free(fsm->alloc, mapping); } if (shortest_end_distance != NULL) { - f_free(fsm->opt->alloc, shortest_end_distance); + f_free(fsm->alloc, shortest_end_distance); } return r; @@ -262,14 +262,14 @@ build_minimised_mapping(const struct fsm *fsm, env.dfa_labels = dfa_labels; env.dfa_label_count = dfa_label_count; - env.state_ecs = f_malloc(fsm->opt->alloc, alloc_size); + env.state_ecs = f_malloc(fsm->alloc, alloc_size); if (env.state_ecs == NULL) { goto cleanup; } env.ec_map_count = fsm->statecount + 1; - env.jump = f_malloc(fsm->opt->alloc, alloc_size); + env.jump = f_malloc(fsm->alloc, alloc_size); if (env.jump == NULL) { goto cleanup; } - env.ecs = f_malloc(fsm->opt->alloc, alloc_size); + env.ecs = f_malloc(fsm->alloc, alloc_size); if (env.ecs == NULL) { goto cleanup; } env.ecs[INIT_EC_NOT_FINAL] = NO_ID; @@ -409,9 +409,9 @@ build_minimised_mapping(const struct fsm *fsm, /* fall through */ cleanup: - f_free(fsm->opt->alloc, env.ecs); - f_free(fsm->opt->alloc, env.state_ecs); - f_free(fsm->opt->alloc, env.jump); + f_free(fsm->alloc, env.ecs); + f_free(fsm->alloc, env.state_ecs); + f_free(fsm->alloc, env.jump); return res; } @@ -507,7 +507,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, assert(fsm != NULL); assert(shortest_end_distance != NULL); - counts = f_calloc(fsm->opt->alloc, + counts = f_calloc(fsm->alloc, DEF_INITIAL_COUNT_CEIL, sizeof(counts[0])); if (counts == NULL) { goto cleanup; @@ -533,7 +533,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, while (sed >= nceil) { nceil *= 2; } - ncounts = f_realloc(fsm->opt->alloc, + ncounts = f_realloc(fsm->alloc, counts, nceil * sizeof(counts[0])); if (ncounts == NULL) { goto cleanup; @@ -562,7 +562,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, /* Build a permutation vector of the counts, such * that counts[pv[i..N]] would return the values * in counts[] in ascending order. */ - pv = permutation_vector(fsm->opt->alloc, + pv = permutation_vector(fsm->alloc, sed_limit, count_max, counts); if (pv == NULL) { goto cleanup; @@ -602,7 +602,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, * [1]: http://www.sudleyplace.com/APL/Anatomy%20of%20An%20Idiom.pdf * [2]: https://bitbucket.org/ngn/k/src */ - ranking = permutation_vector(fsm->opt->alloc, + ranking = permutation_vector(fsm->alloc, sed_limit, sed_limit, pv); if (ranking == NULL) { goto cleanup; @@ -652,9 +652,9 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, res = 1; cleanup: - f_free(fsm->opt->alloc, counts); - f_free(fsm->opt->alloc, pv); - f_free(fsm->opt->alloc, ranking); + f_free(fsm->alloc, counts); + f_free(fsm->alloc, pv); + f_free(fsm->alloc, ranking); return res; #else @@ -724,7 +724,7 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) /* Use the hash table to assign to new groups. */ - end_md = f_calloc(fsm->opt->alloc, + end_md = f_calloc(fsm->alloc, state_count, sizeof(end_md[0])); if (end_md == NULL) { goto cleanup; @@ -736,7 +736,7 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) } const size_t mask = bucket_count - 1; - htab = f_malloc(fsm->opt->alloc, + htab = f_malloc(fsm->alloc, bucket_count * sizeof(htab[0])); if (htab == NULL) { goto cleanup; @@ -923,17 +923,17 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) cleanup: if (htab != NULL) { - f_free(fsm->opt->alloc, htab); + f_free(fsm->alloc, htab); } if (end_md != NULL) { size_t i; for (i = 0; i < state_count; i++) { struct end_metadata *e = &end_md[i]; if (e->end.ids != NULL) { - f_free(fsm->opt->alloc, e->end.ids); + f_free(fsm->alloc, e->end.ids); } } - f_free(fsm->opt->alloc, end_md); + f_free(fsm->alloc, end_md); } return res; @@ -948,7 +948,7 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, return 1; } - e->ids = f_malloc(fsm->opt->alloc, + e->ids = f_malloc(fsm->alloc, e->count * sizeof(e->ids[0])); if (e->ids == NULL) { return 0; diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index 71ee0a89c..9a882a24f 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -157,7 +157,8 @@ typedef struct fsm * fsm; struct fsm * - fsm_parse(FILE *f, const struct fsm_options *opt); + fsm_parse(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt); @}; @@ -413,7 +414,7 @@ %trailer% @{ - struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { + struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -449,13 +450,14 @@ /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - /* TODO: pass in fsm_options */ - new = fsm_new(opt); + new = fsm_new(alloc); if (new == NULL) { perror("fsm_new"); return NULL; } + fsm_setoptions(new, opt); + ADVANCE_LEXER; switch (CURRENT_TERMINAL) { case TOK_UNKNOWN: diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 442449a44..026e621ac 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -179,7 +179,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: CHAR */ { -#line 246 "src/libfsm/parser.act" +#line 247 "src/libfsm/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); @@ -196,7 +196,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: ESC */ { -#line 170 "src/libfsm/parser.act" +#line 171 "src/libfsm/parser.act" assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(2 == strlen(lex_state->buf.a)); @@ -224,7 +224,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: HEX */ { -#line 239 "src/libfsm/parser.act" +#line 240 "src/libfsm/parser.act" unsigned long u; char *e; @@ -263,7 +263,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: OCT */ { -#line 210 "src/libfsm/parser.act" +#line 211 "src/libfsm/parser.act" unsigned long u; char *e; @@ -338,7 +338,7 @@ ZL2_items:; case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 251 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIa = xstrdup(lex_state->buf.a); @@ -489,7 +489,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-start */ { -#line 403 "src/libfsm/parser.act" +#line 404 "src/libfsm/parser.act" err_expected(lex_state, "'start:'"); @@ -507,7 +507,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 251 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); @@ -530,7 +530,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) } /* BEGINNING OF ACTION: add-state */ { -#line 283 "src/libfsm/parser.act" +#line 284 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -588,7 +588,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ { -#line 335 "src/libfsm/parser.act" +#line 336 "src/libfsm/parser.act" fsm_setstart(fsm, (ZIs)); @@ -597,7 +597,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ { -#line 349 "src/libfsm/parser.act" +#line 350 "src/libfsm/parser.act" free((ZIn)); @@ -639,7 +639,7 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-end */ { -#line 407 "src/libfsm/parser.act" +#line 408 "src/libfsm/parser.act" err_expected(lex_state, "'end:'"); @@ -687,7 +687,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 251 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); @@ -705,7 +705,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 283 "src/libfsm/parser.act" +#line 284 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -763,7 +763,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-end */ { -#line 339 "src/libfsm/parser.act" +#line 340 "src/libfsm/parser.act" fsm_setend(fsm, (ZIs), 1); @@ -772,7 +772,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* END OF ACTION: mark-end */ /* BEGINNING OF ACTION: free */ { -#line 349 "src/libfsm/parser.act" +#line 350 "src/libfsm/parser.act" free((ZIn)); @@ -810,7 +810,7 @@ p_fsm(fsm fsm, lex_state lex_state, act_state act_state) ADVANCE_LEXER; /* BEGINNING OF ACTION: free-statelist */ { -#line 365 "src/libfsm/parser.act" +#line 366 "src/libfsm/parser.act" struct act_statelist *p; struct act_statelist *next; @@ -834,7 +834,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-syntax */ { -#line 412 "src/libfsm/parser.act" +#line 413 "src/libfsm/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); @@ -865,7 +865,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-comma */ { -#line 399 "src/libfsm/parser.act" +#line 400 "src/libfsm/parser.act" err_expected(lex_state, "','"); @@ -895,7 +895,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-sep */ { -#line 391 "src/libfsm/parser.act" +#line 392 "src/libfsm/parser.act" err_expected(lex_state, "';'"); @@ -923,7 +923,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 251 "src/libfsm/parser.act" +#line 252 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); @@ -941,7 +941,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 283 "src/libfsm/parser.act" +#line 284 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -999,7 +999,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ { -#line 283 "src/libfsm/parser.act" +#line 284 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIb)); @@ -1057,7 +1057,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 349 "src/libfsm/parser.act" +#line 350 "src/libfsm/parser.act" free((*ZIa)); @@ -1066,7 +1066,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ { -#line 349 "src/libfsm/parser.act" +#line 350 "src/libfsm/parser.act" free((ZIb)); @@ -1081,7 +1081,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; /* BEGINNING OF ACTION: add-edge-any */ { -#line 375 "src/libfsm/parser.act" +#line 376 "src/libfsm/parser.act" if (!fsm_addedge_any(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_any"); @@ -1104,7 +1104,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) } /* BEGINNING OF ACTION: add-edge-literal */ { -#line 368 "src/libfsm/parser.act" +#line 369 "src/libfsm/parser.act" if (!fsm_addedge_literal(fsm, (ZIx), (ZIy), (ZIc))) { perror("fsm_addedge_literal"); @@ -1120,7 +1120,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: add-edge-epsilon */ { -#line 382 "src/libfsm/parser.act" +#line 383 "src/libfsm/parser.act" if (!fsm_addedge_epsilon(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_epsilon"); @@ -1138,7 +1138,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: err-expected-trans */ { -#line 395 "src/libfsm/parser.act" +#line 396 "src/libfsm/parser.act" err_expected(lex_state, "transition"); @@ -1162,7 +1162,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* BEGINNING OF ACTION: add-state */ { -#line 283 "src/libfsm/parser.act" +#line 284 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -1220,7 +1220,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 349 "src/libfsm/parser.act" +#line 350 "src/libfsm/parser.act" free((*ZIa)); @@ -1298,7 +1298,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) case (TOK_ENDID): /* BEGINNING OF EXTRACT: ENDID */ { -#line 276 "src/libfsm/parser.act" +#line 277 "src/libfsm/parser.act" unsigned long u; char *e; @@ -1333,7 +1333,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-end-id */ { -#line 343 "src/libfsm/parser.act" +#line 344 "src/libfsm/parser.act" if (!fsm_endid_set(fsm, (ZIs), (ZIid))) { goto ZL1; @@ -1351,10 +1351,10 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 480 "src/libfsm/parser.act" +#line 482 "src/libfsm/parser.act" - struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { + struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -1390,13 +1390,14 @@ ZL1:; /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - /* TODO: pass in fsm_options */ - new = fsm_new(opt); + new = fsm_new(alloc); if (new == NULL) { perror("fsm_new"); return NULL; } + fsm_setoptions(new, opt); + ADVANCE_LEXER; switch (CURRENT_TERMINAL) { case TOK_UNKNOWN: @@ -1418,6 +1419,6 @@ ZL1:; return new; } -#line 1422 "src/libfsm/parser.c" +#line 1423 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index be8f288ec..e7ca6f330 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 162 "src/libfsm/parser.act" +#line 163 "src/libfsm/parser.act" typedef struct lex_state * lex_state; @@ -17,17 +17,18 @@ typedef struct fsm * fsm; struct fsm * - fsm_parse(FILE *f, const struct fsm_options *opt); + fsm_parse(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt); -#line 23 "src/libfsm/parser.h" +#line 24 "src/libfsm/parser.h" /* BEGINNING OF FUNCTION DECLARATIONS */ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 481 "src/libfsm/parser.act" +#line 483 "src/libfsm/parser.act" -#line 32 "src/libfsm/parser.h" +#line 33 "src/libfsm/parser.h" /* END OF FILE */ diff --git a/src/libfsm/pred/hasnondeterminism.c b/src/libfsm/pred/hasnondeterminism.c index 5a3801968..39a3b59f2 100644 --- a/src/libfsm/pred/hasnondeterminism.c +++ b/src/libfsm/pred/hasnondeterminism.c @@ -51,7 +51,7 @@ state_epsilon_closure(const struct fsm *fsm, fsm_state_t state, return *closure; } - if (!state_set_add(closure, fsm->opt->alloc, state)) { + if (!state_set_add(closure, fsm->alloc, state)) { return NULL; } diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 58de4572c..e7232da42 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -139,7 +139,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fsm_end_id_t *ids; int res; - ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + ids = f_malloc(fsm->alloc, count * sizeof *ids); if (ids == NULL) { /* XXX */ goto error; @@ -155,11 +155,11 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\t}\n"); fprintf(f, "\n"); - f_free(fsm->opt->alloc, ids); + f_free(fsm->alloc, ids); } } - a = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *a); + a = f_malloc(fsm->alloc, fsm->statecount * sizeof *a); if (a == NULL) { /* XXX */ goto error; @@ -226,7 +226,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) } } - f_free(fsm->opt->alloc, a); + f_free(fsm->alloc, a); fprintf(f, "\n"); diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index caa3fc7f3..797bee340 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -127,7 +127,7 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, unique = NULL; for (edge_set_reset(fsm->states[s].edges, &it); edge_set_next(&it, &e); ) { - if (!state_set_add(&unique, opt->alloc, e.state)) { + if (!state_set_add(&unique, fsm->alloc, e.state)) { return -1; } } @@ -231,7 +231,7 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, if (count > 0) { int res; - ids = f_malloc(opt->alloc, count * sizeof *ids); + ids = f_malloc(fsm->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } @@ -256,7 +256,7 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, fprintf(f, " ];\n"); - f_free(opt->alloc, ids); + f_free(fsm->alloc, ids); } /* TODO: show example here, unless !opt->comments */ diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 1f0adb9c1..a029a82c2 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -338,7 +338,7 @@ fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) if (count > 0) { int res; - ids = f_malloc(opt->alloc, count * sizeof *ids); + ids = f_malloc(fsm->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } @@ -358,7 +358,7 @@ fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) } if (count > 0) { - f_free(opt->alloc, ids); + f_free(fsm->alloc, ids); } fprintf(f, "%s", end > 0 ? ", " : ";\n"); diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index f23ce238a..4ef3195e5 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -127,7 +127,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, * together by their common destination states. */ - groups = f_malloc(fsm->opt->alloc, sizeof *groups * n); /* worst case */ + groups = f_malloc(fsm->alloc, sizeof *groups * n); /* worst case */ if (groups == NULL) { return NULL; } @@ -141,7 +141,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, to = ranges[i].to; - p = f_malloc(fsm->opt->alloc, sizeof *p * (n - i)); /* worst case */ + p = f_malloc(fsm->alloc, sizeof *p * (n - i)); /* worst case */ if (p == NULL) { j++; goto error; @@ -163,7 +163,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, { void *tmp; - tmp = f_realloc(fsm->opt->alloc, p, sizeof *p * k); + tmp = f_realloc(fsm->alloc, p, sizeof *p * k); if (tmp == NULL) { j++; goto error; @@ -182,7 +182,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, { void *tmp; - tmp = f_realloc(fsm->opt->alloc, groups, sizeof *groups * j); + tmp = f_realloc(fsm->alloc, groups, sizeof *groups * j); if (tmp == NULL) { goto error; } @@ -195,10 +195,10 @@ make_groups(const struct fsm *fsm, fsm_state_t state, error: for (i = 0; i < j; i++) { - f_free(fsm->opt->alloc, (void *) groups[i].ranges); + f_free(fsm->alloc, (void *) groups[i].ranges); } - f_free(fsm->opt->alloc, groups); + f_free(fsm->alloc, groups); return NULL; } @@ -209,10 +209,10 @@ free_groups(const struct fsm *fsm, struct ir_group *groups, size_t n) size_t j; for (j = 0; j < n; j++) { - f_free(fsm->opt->alloc, (void *) groups[j].ranges); + f_free(fsm->alloc, (void *) groups[j].ranges); } - f_free(fsm->opt->alloc, groups); + f_free(fsm->alloc, groups); } static void @@ -275,7 +275,7 @@ make_holes(const struct fsm *fsm, const struct bm *bm, size_t *n) assert(bm != NULL); assert(n != NULL); - ranges = f_malloc(fsm->opt->alloc, sizeof *ranges * FSM_SIGMA_COUNT); /* worst case */ + ranges = f_malloc(fsm->alloc, sizeof *ranges * FSM_SIGMA_COUNT); /* worst case */ if (ranges == NULL) { return NULL; } @@ -302,7 +302,7 @@ make_holes(const struct fsm *fsm, const struct bm *bm, size_t *n) { void *tmp; - tmp = f_realloc(fsm->opt->alloc, ranges, sizeof *ranges * *n); + tmp = f_realloc(fsm->alloc, ranges, sizeof *ranges * *n); if (tmp == NULL) { goto error; } @@ -314,7 +314,7 @@ make_holes(const struct fsm *fsm, const struct bm *bm, size_t *n) error: - f_free(fsm->opt->alloc, ranges); + f_free(fsm->alloc, ranges); return NULL; } @@ -401,7 +401,7 @@ make_state(const struct fsm *fsm, fsm_state_t state, cs->strategy = IR_ERROR; cs->u.error.mode = groups[max.j].to; - f_free(fsm->opt->alloc, (void *) groups[max.j].ranges); + f_free(fsm->alloc, (void *) groups[max.j].ranges); if (max.j < n) { memmove(groups + max.j, groups + max.j + 1, sizeof *groups * (n - max.j - 1)); n--; @@ -449,15 +449,15 @@ make_ir(const struct fsm *fsm) return NULL; } - ir = f_malloc(fsm->opt->alloc, sizeof *ir); + ir = f_malloc(fsm->alloc, sizeof *ir); if (ir == NULL) { return NULL; } ir->n = fsm_countstates(fsm); - ir->states = f_malloc(fsm->opt->alloc, sizeof *ir->states * ir->n); + ir->states = f_malloc(fsm->alloc, sizeof *ir->states * ir->n); if (ir->states == NULL) { - f_free(fsm->opt->alloc, ir); + f_free(fsm->alloc, ir); return NULL; } @@ -480,7 +480,7 @@ make_ir(const struct fsm *fsm) } else { int res; - ids = f_malloc(fsm->opt->alloc, count * sizeof *ids); + ids = f_malloc(fsm->alloc, count * sizeof *ids); if (ids == NULL) { goto error; } @@ -513,14 +513,14 @@ make_ir(const struct fsm *fsm) * to the number of states in an fsm, and shorter where * the graph branches often. */ - p = f_malloc(fsm->opt->alloc, ir->n + 3 + 1); + p = f_malloc(fsm->alloc, ir->n + 3 + 1); if (p == NULL) { goto error_example; } n = fsm_example(fsm, i, p, ir->n + 1); if (-1 == n) { - f_free(fsm->opt->alloc, p); + f_free(fsm->alloc, p); goto error_example; } @@ -529,16 +529,16 @@ make_ir(const struct fsm *fsm) * no "stray" states, but the caller may not trim an FSM. */ if (n == 0) { - f_free(fsm->opt->alloc, p); + f_free(fsm->alloc, p); p = NULL; } else if ((size_t) n < ir->n + 1) { char *tmp; n = strlen(p); - tmp = f_realloc(fsm->opt->alloc, p, n + 1); + tmp = f_realloc(fsm->alloc, p, n + 1); if (tmp == NULL) { - f_free(fsm->opt->alloc, p); + f_free(fsm->alloc, p); goto error_example; } @@ -564,7 +564,7 @@ make_ir(const struct fsm *fsm) empty: - ir = f_malloc(fsm->opt->alloc, sizeof *ir); + ir = f_malloc(fsm->alloc, sizeof *ir); if (ir == NULL) { return NULL; } @@ -596,8 +596,8 @@ free_ir(const struct fsm *fsm, struct ir *ir) assert(ir != NULL); for (i = 0; i < ir->n; i++) { - f_free(fsm->opt->alloc, (void *) ir->states[i].example); - f_free(fsm->opt->alloc, (void *) ir->states[i].endids.ids); + f_free(fsm->alloc, (void *) ir->states[i].example); + f_free(fsm->alloc, (void *) ir->states[i].endids.ids); switch (ir->states[i].strategy) { case IR_TABLE: @@ -626,15 +626,15 @@ free_ir(const struct fsm *fsm, struct ir *ir) break; case IR_ERROR: - f_free(fsm->opt->alloc, (void *) ir->states[i].u.error.error.ranges); + f_free(fsm->alloc, (void *) ir->states[i].u.error.error.ranges); free_groups(fsm, (void *) ir->states[i].u.error.groups, ir->states[i].u.error.n); break; } } - f_free(fsm->opt->alloc, ir->states); + f_free(fsm->alloc, ir->states); - f_free(fsm->opt->alloc, ir); + f_free(fsm->alloc, ir); } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 18718d215..1c5d9da4e 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -148,7 +148,7 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, unique = NULL; for (edge_set_reset(fsm->states[s].edges, &it); edge_set_next(&it, &e); ) { - if (!state_set_add(&unique, opt->alloc, e.state)) { + if (!state_set_add(&unique, fsm->alloc, e.state)) { return -1; } } @@ -259,7 +259,7 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) count = fsm_endid_count(fsm, i); - ids = f_malloc(opt->alloc, count * sizeof *ids); + ids = f_malloc(fsm->alloc, count * sizeof *ids); if (ids == NULL) { return -1; } @@ -282,7 +282,7 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) fprintf(f, " }"); - f_free(opt->alloc, ids); + f_free(fsm->alloc, ids); notfirst = 1; } diff --git a/src/libfsm/reverse.c b/src/libfsm/reverse.c index b802f364d..b8e6d65f5 100644 --- a/src/libfsm/reverse.c +++ b/src/libfsm/reverse.c @@ -54,11 +54,13 @@ fsm_reverse(struct fsm *fsm) if (fsm->endcount == 0 || !fsm_getstart(fsm, &prevstart)) { struct fsm *new; - new = fsm_new_statealloc(fsm->opt, 1); + new = fsm_new_statealloc(fsm->alloc, 1); if (new == NULL) { return 0; } + fsm_setoptions(new, fsm->opt); + if (!fsm_addstate(new, &new->start)) { fsm_free(new); return 0; @@ -92,7 +94,7 @@ fsm_reverse(struct fsm *fsm) continue; } - if (!state_set_add(&endset, fsm->opt->alloc, i)) { + if (!state_set_add(&endset, fsm->alloc, i)) { state_set_free(endset); return 0; } @@ -119,16 +121,16 @@ fsm_reverse(struct fsm *fsm) fsm_setstart(fsm, start); } - edges = f_malloc(fsm->opt->alloc, sizeof *edges * fsm->statecount); + edges = f_malloc(fsm->alloc, sizeof *edges * fsm->statecount); if (edges == NULL) { state_set_free(endset); return 0; } - epsilons = f_malloc(fsm->opt->alloc, sizeof *epsilons * fsm->statecount); + epsilons = f_malloc(fsm->alloc, sizeof *epsilons * fsm->statecount); if (edges == NULL) { state_set_free(endset); - f_free(fsm->opt->alloc, edges); + f_free(fsm->alloc, edges); return 0; } @@ -165,7 +167,7 @@ fsm_reverse(struct fsm *fsm) } for (state_set_reset(fsm->states[i].epsilons, &jt); state_set_next(&jt, &se); ) { - if (!state_set_add(&epsilons[se], fsm->opt->alloc, i)) { + if (!state_set_add(&epsilons[se], fsm->alloc, i)) { goto error1; } } @@ -173,7 +175,7 @@ fsm_reverse(struct fsm *fsm) for (edge_set_reset(fsm->states[i].edges, &it); edge_set_next(&it, &e); ) { assert(e.state < fsm->statecount); - if (!edge_set_add(&edges[e.state], fsm->opt->alloc, e.symbol, i)) { + if (!edge_set_add(&edges[e.state], fsm->alloc, e.symbol, i)) { return 0; } } @@ -209,7 +211,7 @@ fsm_reverse(struct fsm *fsm) continue; } - if (!state_set_add(&epsilons[start], fsm->opt->alloc, s)) { + if (!state_set_add(&epsilons[start], fsm->alloc, s)) { goto error1; } } @@ -226,7 +228,7 @@ fsm_reverse(struct fsm *fsm) continue; } - if (!edge_set_copy(&edges[start], fsm->opt->alloc, edges[s])) { + if (!edge_set_copy(&edges[start], fsm->alloc, edges[s])) { goto error; } } @@ -267,7 +269,7 @@ fsm_reverse(struct fsm *fsm) for (i = 0; i < fsm->statecount; i++) { state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); fsm->states[i].epsilons = epsilons[i]; fsm->states[i].edges = edges[i]; @@ -276,8 +278,8 @@ fsm_reverse(struct fsm *fsm) state_set_free(endset); - f_free(fsm->opt->alloc, epsilons); - f_free(fsm->opt->alloc, edges); + f_free(fsm->alloc, epsilons); + f_free(fsm->alloc, edges); return 1; @@ -288,7 +290,7 @@ fsm_reverse(struct fsm *fsm) for (i = 0; i < fsm->statecount; i++) { state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); fsm->states[i].epsilons = epsilons[i]; fsm->states[i].edges = edges[i]; @@ -299,8 +301,8 @@ fsm_reverse(struct fsm *fsm) state_set_free(endset); - f_free(fsm->opt->alloc, epsilons); - f_free(fsm->opt->alloc, edges); + f_free(fsm->alloc, epsilons); + f_free(fsm->alloc, edges); return 0; } diff --git a/src/libfsm/shortest.c b/src/libfsm/shortest.c index e484e9667..45a441a08 100644 --- a/src/libfsm/shortest.c +++ b/src/libfsm/shortest.c @@ -72,7 +72,7 @@ fsm_shortest(const struct fsm *fsm, } for (i = 0; i < fsm->statecount; i++) { - u = priq_push(fsm->opt->alloc, &todo, i, + u = priq_push(fsm->alloc, &todo, i, i == qs ? 0 : FSM_COST_INFINITY); if (u == NULL) { goto error; @@ -149,13 +149,13 @@ fsm_shortest(const struct fsm *fsm, done: for (u = priq_find(done, goal); u != NULL; u = u->prev) { - if (!path_push(fsm->opt->alloc, &path, u->state, u->c)) { + if (!path_push(fsm->alloc, &path, u->state, u->c)) { goto error; } } - priq_free(fsm->opt->alloc, todo); - priq_free(fsm->opt->alloc, done); + priq_free(fsm->alloc, todo); + priq_free(fsm->alloc, done); return path; @@ -171,21 +171,21 @@ fsm_shortest(const struct fsm *fsm, * (otherwise we would have been able to reach it). */ - if (!path_push(fsm->opt->alloc, &path, u->state, u->c)) { + if (!path_push(fsm->alloc, &path, u->state, u->c)) { goto error; } - priq_free(fsm->opt->alloc, todo); - priq_free(fsm->opt->alloc, done); + priq_free(fsm->alloc, todo); + priq_free(fsm->alloc, done); return path; error: - priq_free(fsm->opt->alloc, todo); - priq_free(fsm->opt->alloc, done); + priq_free(fsm->alloc, todo); + priq_free(fsm->alloc, done); - path_free(fsm->opt->alloc, path); + path_free(fsm->alloc, path); return NULL; } diff --git a/src/libfsm/shuffle.c b/src/libfsm/shuffle.c index da8e35bfd..0a7269dab 100644 --- a/src/libfsm/shuffle.c +++ b/src/libfsm/shuffle.c @@ -68,7 +68,7 @@ fsm_shuffle(struct fsm *fsm, unsigned seed) return 1; } - mapping = f_malloc(fsm->opt->alloc, state_count * sizeof(mapping[0])); + mapping = f_malloc(fsm->alloc, state_count * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; } populate_mapping(mapping, state_count, (uint32_t)seed); @@ -80,6 +80,6 @@ fsm_shuffle(struct fsm *fsm, unsigned seed) res = 1; cleanup: - f_free(fsm->opt->alloc, mapping); + f_free(fsm->alloc, mapping); return res; } diff --git a/src/libfsm/state.c b/src/libfsm/state.c index acf2bff25..063cc78ab 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -35,7 +35,7 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) struct fsm_state *tmp; size_t i; - tmp = f_realloc(fsm->opt->alloc, fsm->states, n * sizeof *fsm->states); + tmp = f_realloc(fsm->alloc, fsm->states, n * sizeof *fsm->states); if (tmp == NULL) { return 0; } @@ -128,7 +128,7 @@ fsm_removestate(struct fsm *fsm, fsm_state_t state) } state_set_free(fsm->states[state].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[state].edges); + edge_set_free(fsm->alloc, fsm->states[state].edges); if (fsm_getstart(fsm, &start) && start == state) { fsm_clearstart(fsm); @@ -148,7 +148,7 @@ fsm_removestate(struct fsm *fsm, fsm_state_t state) for (i = 0; i < fsm->statecount - 1; i++) { state_set_replace(&fsm->states[i].epsilons, fsm->statecount - 1, state); - if (!edge_set_replace_state(&fsm->states[i].edges, fsm->opt->alloc, fsm->statecount - 1, state)) { + if (!edge_set_replace_state(&fsm->states[i].edges, fsm->alloc, fsm->statecount - 1, state)) { return 0; } } @@ -177,7 +177,7 @@ fsm_compact_states(struct fsm *fsm, size_t kept, removed_count; const fsm_state_t orig_statecount = fsm->statecount; - fsm_state_t *mapping = f_malloc(fsm->opt->alloc, + fsm_state_t *mapping = f_malloc(fsm->alloc, fsm->statecount * sizeof(mapping[0])); if (mapping == NULL) { return 0; @@ -223,7 +223,7 @@ fsm_compact_states(struct fsm *fsm, #endif state_set_compact(&s->epsilons, mapping_cb, mapping); if (fsm->states[i].edges != NULL) { - edge_set_compact(&s->edges, fsm->opt->alloc, mapping_cb, mapping); + edge_set_compact(&s->edges, fsm->alloc, mapping_cb, mapping); } } @@ -232,7 +232,7 @@ fsm_compact_states(struct fsm *fsm, assert(dst <= i); if (mapping[i] == FSM_STATE_REMAP_NO_STATE) { /* dead */ state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); fsm->statecount--; removed_count++; @@ -272,7 +272,7 @@ fsm_compact_states(struct fsm *fsm, /* todo: resize backing array, if significantly smaller? */ } - f_free(fsm->opt->alloc, mapping); + f_free(fsm->alloc, mapping); if (removed != NULL) { *removed = removed_count; diff --git a/src/libfsm/trim.c b/src/libfsm/trim.c index 6a9a25f09..8f7a1da37 100644 --- a/src/libfsm/trim.c +++ b/src/libfsm/trim.c @@ -88,7 +88,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, return 1; /* nothing is reachable */ } - q = queue_new(fsm->opt->alloc, state_count); + q = queue_new(fsm->alloc, state_count); if (q == NULL) { goto cleanup; } @@ -98,13 +98,13 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } if (mode == FSM_TRIM_START_AND_END_REACHABLE) { - edges = f_malloc(fsm->opt->alloc, + edges = f_malloc(fsm->alloc, edge_ceil * sizeof(edges[0])); if (edges == NULL) { goto cleanup; } - ends = f_malloc(fsm->opt->alloc, + ends = f_malloc(fsm->alloc, end_ceil * sizeof(ends[0])); if (ends == NULL) { goto cleanup; @@ -138,7 +138,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (ends && fsm_isend(fsm, s_id)) { if (end_count == end_ceil) { - if (!grow_ends(fsm->opt->alloc, + if (!grow_ends(fsm->alloc, &end_ceil, &ends)) { goto cleanup; } @@ -172,7 +172,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (edges == NULL) { continue; } - if (!save_edge(fsm->opt->alloc, + if (!save_edge(fsm->alloc, &edge_count, &edge_ceil, &edges, s_id, next)) { goto cleanup; @@ -201,7 +201,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (edges == NULL) { continue; } - if (!save_edge(fsm->opt->alloc, + if (!save_edge(fsm->alloc, &edge_count, &edge_ceil, &edges, s_id, next)) { goto cleanup; @@ -262,7 +262,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } /* The ends are no longer needed. */ - f_free(fsm->opt->alloc, ends); + f_free(fsm->alloc, ends); ends = NULL; } @@ -285,7 +285,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, const fsm_state_t max_to = edges[edge_count - 1].to; const size_t offset_count = fsm_countstates(fsm); - offsets = f_calloc(fsm->opt->alloc, + offsets = f_calloc(fsm->alloc, offset_count, sizeof(offsets[0])); if (offsets == NULL) { goto cleanup; @@ -366,9 +366,9 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, res = 1; cleanup: - if (edges != NULL) { f_free(fsm->opt->alloc, edges); } - if (ends != NULL) { f_free(fsm->opt->alloc, ends); } - if (offsets != NULL) { f_free(fsm->opt->alloc, offsets); } + if (edges != NULL) { f_free(fsm->alloc, edges); } + if (ends != NULL) { f_free(fsm->alloc, ends); } + if (offsets != NULL) { f_free(fsm->alloc, offsets); } if (q != NULL) { queue_free(q); } return res; @@ -507,7 +507,7 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, if (shortest_end_distance != NULL && mode == FSM_TRIM_START_AND_END_REACHABLE) { size_t s_i; - sed = f_malloc(fsm->opt->alloc, + sed = f_malloc(fsm->alloc, fsm->statecount * sizeof(sed[0])); if (sed == NULL) { goto cleanup; @@ -546,7 +546,7 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, if (ret < 0) { if (sed != NULL) { - f_free(fsm->opt->alloc, sed); + f_free(fsm->alloc, sed); } return ret; } @@ -562,10 +562,10 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, cleanup: if (marks != NULL) { - f_free(fsm->opt->alloc, marks); + f_free(fsm->alloc, marks); } if (sed != NULL) { - f_free(fsm->opt->alloc, sed); + f_free(fsm->alloc, sed); } return -1; } diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 7a882be01..efc57ca65 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -38,6 +38,11 @@ fsm_union(struct fsm *a, struct fsm *b, assert(a != NULL); assert(b != NULL); + if (a->alloc != b->alloc) { + errno = EINVAL; + return NULL; + } + if (a->opt != b->opt) { errno = EINVAL; return NULL; diff --git a/src/libfsm/vacuum.c b/src/libfsm/vacuum.c index abecf6301..cc819a622 100644 --- a/src/libfsm/vacuum.c +++ b/src/libfsm/vacuum.c @@ -27,7 +27,7 @@ fsm_vacuum(struct fsm *fsm) return true; } - struct fsm_state *nstates = f_realloc(fsm->opt->alloc, + struct fsm_state *nstates = f_realloc(fsm->alloc, fsm->states, nceil * sizeof(nstates[0])); if (nstates == NULL) { return false; diff --git a/src/libfsm/walk/reachable.c b/src/libfsm/walk/reachable.c index f7a961600..ec0f5ee37 100644 --- a/src/libfsm/walk/reachable.c +++ b/src/libfsm/walk/reachable.c @@ -32,7 +32,7 @@ fsm_reachable(const struct fsm *fsm, fsm_state_t state, fsm->states[i].visited = 0; } - q = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *q); + q = f_malloc(fsm->alloc, fsm->statecount * sizeof *q); if (q == NULL) { return -1; } @@ -96,7 +96,7 @@ fsm_reachable(const struct fsm *fsm, fsm_state_t state, fsm->states[i].visited = 0; } - f_free(fsm->opt->alloc, q); + f_free(fsm->alloc, q); return r; } diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index afd8107ca..42db6b389 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -131,7 +131,7 @@ fsm_walk2_data_free(const struct fsm *fsm, struct fsm_walk2_data *data) for (p = data->head; p != NULL; p = next) { next = p->next; - f_free(fsm->opt->alloc, p); + f_free(fsm->alloc, p); } if (data->new) { @@ -159,7 +159,7 @@ alloc_walk2_tuple(struct fsm_walk2_data *data) new_pool: - pool = f_malloc(data->new->opt->alloc, sizeof *pool); + pool = f_malloc(data->new->alloc, sizeof *pool); if (pool == NULL) { return NULL; } @@ -281,9 +281,9 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, int ret; if (fsm_a != NULL) { - alloc = fsm_a->opt->alloc; + alloc = fsm_a->alloc; } else if (fsm_b != NULL) { - alloc = fsm_b->opt->alloc; + alloc = fsm_b->alloc; } else { assert(!"unreached"); } @@ -562,12 +562,14 @@ fsm_walk2(const struct fsm *a, const struct fsm *b, data.edgemask = edgemask; data.endmask = endmask; - data.new = fsm_new(a->opt); + data.new = fsm_new(a->alloc); if (data.new == NULL) { goto error; } - data.states = tuple_set_create(data.new->opt->alloc, cmp_walk2_tuple); + fsm_setoptions(data.new, a->opt); + + data.states = tuple_set_create(data.new->alloc, cmp_walk2_tuple); if (data.states == NULL) { goto error; } @@ -596,7 +598,14 @@ fsm_walk2(const struct fsm *a, const struct fsm *b, empty: - return fsm_new(a->opt); + new = fsm_new(a->alloc); + if (new == NULL) { + return NULL; + } + + fsm_setoptions(new, a->opt); + + return new; error: diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 852b267fc..32dcd8bdf 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -192,13 +192,15 @@ fsm_unionxy(struct fsm *a, struct fsm *b, fsm_state_t x, fsm_state_t y) static struct fsm * expr_compile(struct ast_expr *e, enum re_flags flags, - const struct fsm_options *opt, struct re_err *err) + const struct fsm_alloc *alloc, + const struct fsm_options *opt, + struct re_err *err) { struct ast ast; ast.expr = e; - return ast_compile(&ast, flags, opt, err); + return ast_compile(&ast, flags, alloc, opt, err); } static int @@ -883,13 +885,13 @@ comp_iter(struct comp_env *env, re_flags &= ~(unsigned)RE_REVERSE; a = expr_compile(n->u.subtract.a, re_flags, - fsm_getoptions(env->fsm), env->err); + env->fsm->alloc, fsm_getoptions(env->fsm), env->err); if (a == NULL) { return 0; } b = expr_compile(n->u.subtract.b, re_flags, - fsm_getoptions(env->fsm), env->err); + env->fsm->alloc, fsm_getoptions(env->fsm), env->err); if (b == NULL) { fsm_free(a); return 0; @@ -959,6 +961,7 @@ comp_iter(struct comp_env *env, struct fsm * ast_compile(const struct ast *ast, enum re_flags re_flags, + const struct fsm_alloc *alloc, const struct fsm_options *opt, struct re_err *err) { @@ -967,11 +970,13 @@ ast_compile(const struct ast *ast, assert(ast != NULL); - fsm = fsm_new(opt); + fsm = fsm_new(alloc); if (fsm == NULL) { return NULL; } + fsm_setoptions(fsm, opt); + if (!fsm_addstate(fsm, &x)) { goto error; } diff --git a/src/libre/ast_compile.h b/src/libre/ast_compile.h index 89e1b12a4..86a7bb882 100644 --- a/src/libre/ast_compile.h +++ b/src/libre/ast_compile.h @@ -12,11 +12,14 @@ */ struct fsm; +struct fsm_alloc; +struct fsm_options; struct re_err; struct fsm * ast_compile(const struct ast *ast, enum re_flags re_flags, + const struct fsm_alloc *alloc, const struct fsm_options *opt, struct re_err *err); diff --git a/src/libre/ast_rewrite.c b/src/libre/ast_rewrite.c index d05fc0a82..6f13447af 100644 --- a/src/libre/ast_rewrite.c +++ b/src/libre/ast_rewrite.c @@ -118,7 +118,8 @@ deduplicate_alt(struct ast_expr_pool **poolp, struct ast_expr *n) } static struct fsm * -compile_subexpr(struct ast_expr *e, enum re_flags flags) +compile_subexpr(struct ast_expr *e, const struct fsm_alloc *alloc, + enum re_flags flags) { struct fsm *fsm; struct ast ast; @@ -146,7 +147,7 @@ compile_subexpr(struct ast_expr *e, enum re_flags flags) return 0; } - fsm = ast_compile(&ast, flags | RE_ANCHORED, NULL, NULL); + fsm = ast_compile(&ast, flags | RE_ANCHORED, alloc, NULL, NULL); if (fsm == NULL) { return 0; } @@ -619,17 +620,23 @@ rewrite_subtract(struct ast_expr_pool **poolp, struct ast_expr *n, enum re_flags * sub-expressions may be written to assume particular flags * (for example subtraction of two case-insensitive sets may depend * on case-insensitity to produce an empty result). + * + * It's okay to elide the alloc hooks (provided to re_comp) + * because we immediately destroy the constructed fsm, q. + * So we won't mix up different sources of allocation into the AST. */ { + const struct fsm_alloc *alloc = NULL; + struct fsm *a, *b; struct fsm *q; - a = compile_subexpr(n->u.subtract.a, flags); + a = compile_subexpr(n->u.subtract.a, alloc, flags); if (a == NULL) { return 0; } - b = compile_subexpr(n->u.subtract.b, flags); + b = compile_subexpr(n->u.subtract.b, alloc, flags); if (b == NULL) { fsm_free(a); return 0; diff --git a/src/libre/re.c b/src/libre/re.c index 33cc3234c..3e9ee5753 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -176,6 +176,7 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, struct fsm * re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, + const struct fsm_alloc *alloc, const struct fsm_options *opt, enum re_flags flags, struct re_err *err) { @@ -212,7 +213,7 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, ast->expr = ast_expr_tombstone; } - new = ast_compile(ast, flags, opt, err); + new = ast_compile(ast, flags, alloc, opt, err); ast_free(ast); diff --git a/src/libre/re_strings.c b/src/libre/re_strings.c index ff8b2b4ab..34a9fa0fe 100644 --- a/src/libre/re_strings.c +++ b/src/libre/re_strings.c @@ -19,7 +19,8 @@ #include "ac.h" struct fsm * -re_strings(const struct fsm_options *opt, const char *a[], size_t n, +re_strings(const struct fsm_alloc *alloc, const struct fsm_options *opt, + const char *a[], size_t n, enum re_strings_flags flags) { struct re_strings *g; @@ -37,7 +38,7 @@ re_strings(const struct fsm_options *opt, const char *a[], size_t n, } } - fsm = re_strings_build(g, opt, flags); + fsm = re_strings_build(g, alloc, opt, flags); re_strings_free(g); return fsm; @@ -82,7 +83,9 @@ re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endi struct fsm * re_strings_build(struct re_strings *g, - const struct fsm_options *opt, enum re_strings_flags flags) + const struct fsm_alloc *alloc, + const struct fsm_options *opt, + enum re_strings_flags flags) { struct fsm *fsm; fsm_state_t end; @@ -97,11 +100,13 @@ re_strings_build(struct re_strings *g, } /* TODO: count trie nodes and fsm_new_statealloc() */ - fsm = fsm_new(opt); + fsm = fsm_new(alloc); if (fsm == NULL) { goto error; } + fsm_setoptions(fsm, opt); + have_end = 0; if ((flags & RE_STRINGS_AC_AUTOMATON) == 0 && (flags & RE_STRINGS_ANCHOR_RIGHT) == 0) { diff --git a/src/lx/ast.c b/src/lx/ast.c index 5852c7055..97977b803 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -117,12 +117,14 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, return NULL; } - m->fsm = fsm_new(fsm->opt); + m->fsm = fsm_new(fsm->alloc); if (m->fsm == NULL) { free(m); return NULL; } + fsm_setoptions(m->fsm, fsm->opt); + m->token = token; m->to = to; m->conflict = NULL; diff --git a/src/lx/main.c b/src/lx/main.c index 0a91f4775..572014171 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -44,6 +44,8 @@ struct prefix prefix = { "" }; +/* TODO: use alloc hooks for -Q accounting */ +static struct fsm_alloc *alloc; struct fsm_options opt; int print_progress; @@ -373,7 +375,7 @@ zone_minimise(void *arg) } pthread_mutex_unlock(&zmtx); - z->fsm = fsm_new(&opt); + z->fsm = fsm_new(alloc); if (z->fsm == NULL) { pthread_mutex_lock(&zmtx); zerror = errno; @@ -381,6 +383,8 @@ zone_minimise(void *arg) return "fsm_new"; } + fsm_setoptions(z->fsm, &opt); + if (!fsm_addstate(z->fsm, &start)) { pthread_mutex_lock(&zmtx); zerror = errno; @@ -643,7 +647,7 @@ main(int argc, char *argv[]) fprintf(stderr, "-- parsing:"); } - ast = lx_parse(stdin, &opt); + ast = lx_parse(stdin, alloc, &opt); if (ast == NULL) { return EXIT_FAILURE; } diff --git a/src/lx/parser.act b/src/lx/parser.act index 27975d9d5..04001783f 100644 --- a/src/lx/parser.act +++ b/src/lx/parser.act @@ -69,6 +69,7 @@ enum lx_token lex_tok_save; unsigned int zn; const struct fsm_options *opt; + const struct fsm_alloc *alloc; }; struct lex_state { @@ -135,7 +136,8 @@ typedef struct act_state * act_state; typedef struct ast * ast; - struct ast *lx_parse(FILE *f, const struct fsm_options *opt); + struct ast *lx_parse(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt); @}; @@ -309,12 +311,14 @@ assert(@t != NULL); assert(@t != NULL); - @r = fsm_new(act_state->opt); + @r = fsm_new(act_state->alloc); if (@r == NULL) { perror("fsm_new"); @!; } + fsm_setoptions(@r, act_state->opt); + if (!fsm_addstate(@r, &start)) { perror("fsm_addstate"); @!; @@ -365,7 +369,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_LITERAL, act_agetc, @a, act_state->opt, 0, &err); + @r = re_comp(RE_LITERAL, act_agetc, @a, act_state->alloc, act_state->opt, 0, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -380,7 +384,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->opt, @f, &err); + @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->alloc, act_state->opt, @f, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -393,12 +397,14 @@ : () -> (r :fsm) = @{ fsm_state_t start, end; - @r = fsm_new(act_state->opt); + @r = fsm_new(act_state->alloc); if (@r == NULL) { perror("fsm_new"); @!; } + fsm_setoptions(@r, act_state->opt); + if (!fsm_addstate(@r, &start)) { perror("fsm_addstate"); fsm_free(@r); @@ -818,7 +824,7 @@ %trailer% @{ - struct ast *lx_parse(FILE *f, const struct fsm_options *opt) { + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -851,6 +857,7 @@ act_state = &act_state_s; act_state->opt = opt; + act_state->alloc = alloc; if (print_progress) { act_state->zn = 0; diff --git a/src/lx/parser.c b/src/lx/parser.c index 0933c386f..b492f4cd5 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 128 "src/lx/parser.act" +#line 129 "src/lx/parser.act" #include @@ -55,6 +55,7 @@ enum lx_token lex_tok_save; unsigned int zn; const struct fsm_options *opt; + const struct fsm_alloc *alloc; }; struct lex_state { @@ -111,7 +112,7 @@ return a->len--, (unsigned char) *a->p++; } -#line 115 "src/lx/parser.c" +#line 116 "src/lx/parser.c" #ifndef ERROR_TERMINAL @@ -183,11 +184,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-list */ { -#line 817 "src/lx/parser.act" +#line 823 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 191 "src/lx/parser.c" +#line 192 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -205,17 +206,17 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: IDENT */ { -#line 229 "src/lx/parser.act" +#line 231 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); -#line 213 "src/lx/parser.c" +#line 214 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-var */ { -#line 281 "src/lx/parser.act" +#line 283 "src/lx/parser.act" struct ast_zone *z; @@ -242,7 +243,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) goto ZL1; } -#line 246 "src/lx/parser.c" +#line 247 "src/lx/parser.c" } /* END OF ACTION: deref-var */ } @@ -253,18 +254,18 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: TOKEN */ { -#line 225 "src/lx/parser.act" +#line 227 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ -#line 262 "src/lx/parser.c" +#line 263 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 309 "src/lx/parser.act" +#line 311 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -272,12 +273,14 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) assert((ZIt) != NULL); assert((ZIt) != NULL); - (ZIr) = fsm_new(act_state->opt); + (ZIr) = fsm_new(act_state->alloc); if ((ZIr) == NULL) { perror("fsm_new"); goto ZL1; } + fsm_setoptions((ZIr), act_state->opt); + if (!fsm_addstate((ZIr), &start)) { perror("fsm_addstate"); goto ZL1; @@ -321,7 +324,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) fsm_setstart((ZIr), start); -#line 325 "src/lx/parser.c" +#line 328 "src/lx/parser.c" } /* END OF ACTION: deref-token */ } @@ -375,7 +378,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-reverse */ { -#line 678 "src/lx/parser.act" +#line 684 "src/lx/parser.act" assert((ZI210) != NULL); @@ -384,7 +387,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 388 "src/lx/parser.c" +#line 391 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -399,7 +402,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI231) != NULL); @@ -419,7 +422,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 423 "src/lx/parser.c" +#line 426 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI232, &ZI233); @@ -450,7 +453,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complete */ { -#line 669 "src/lx/parser.act" +#line 675 "src/lx/parser.act" assert((ZI210) != NULL); @@ -459,7 +462,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 463 "src/lx/parser.c" +#line 466 "src/lx/parser.c" } /* END OF ACTION: op-complete */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -474,7 +477,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI239) != NULL); @@ -494,7 +497,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 498 "src/lx/parser.c" +#line 501 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI240, &ZI241); @@ -510,11 +513,11 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: IDENT */ { -#line 229 "src/lx/parser.act" +#line 231 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); -#line 518 "src/lx/parser.c" +#line 521 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; @@ -564,7 +567,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI248) != NULL); @@ -584,7 +587,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 588 "src/lx/parser.c" +#line 591 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI249, &ZI250); @@ -615,7 +618,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complement */ { -#line 660 "src/lx/parser.act" +#line 666 "src/lx/parser.act" assert((ZI210) != NULL); @@ -624,7 +627,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 628 "src/lx/parser.c" +#line 631 "src/lx/parser.c" } /* END OF ACTION: op-complement */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -639,7 +642,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI223) != NULL); @@ -659,7 +662,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 663 "src/lx/parser.c" +#line 666 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI224, &ZI225); @@ -686,18 +689,18 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: TOKEN */ { -#line 225 "src/lx/parser.act" +#line 227 "src/lx/parser.act" /* TODO: submatch addressing */ ZI253 = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ -#line 695 "src/lx/parser.c" +#line 698 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 309 "src/lx/parser.act" +#line 311 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -705,12 +708,14 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z assert((ZI253) != NULL); assert((ZI253) != NULL); - (ZI254) = fsm_new(act_state->opt); + (ZI254) = fsm_new(act_state->alloc); if ((ZI254) == NULL) { perror("fsm_new"); goto ZL1; } + fsm_setoptions((ZI254), act_state->opt); + if (!fsm_addstate((ZI254), &start)) { perror("fsm_addstate"); goto ZL1; @@ -754,7 +759,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z fsm_setstart((ZI254), start); -#line 758 "src/lx/parser.c" +#line 763 "src/lx/parser.c" } /* END OF ACTION: deref-token */ p_197 (lex_state, act_state, ZI254, &ZI210); @@ -770,7 +775,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI259) != NULL); @@ -790,7 +795,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 794 "src/lx/parser.c" +#line 799 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI260, &ZI261); @@ -830,7 +835,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI268) != NULL); @@ -850,7 +855,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 854 "src/lx/parser.c" +#line 859 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI269, &ZI270); @@ -870,11 +875,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-thing */ { -#line 813 "src/lx/parser.act" +#line 819 "src/lx/parser.act" err_expected(lex_state, "mapping, binding or zone"); -#line 878 "src/lx/parser.c" +#line 883 "src/lx/parser.c" } /* END OF ACTION: err-expected-thing */ } @@ -900,11 +905,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-open */ { -#line 801 "src/lx/parser.act" +#line 807 "src/lx/parser.act" err_expected(lex_state, "'{'"); -#line 908 "src/lx/parser.c" +#line 913 "src/lx/parser.c" } /* END OF ACTION: err-expected-open */ } @@ -930,11 +935,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-close */ { -#line 805 "src/lx/parser.act" +#line 811 "src/lx/parser.act" err_expected(lex_state, "'}'"); -#line 938 "src/lx/parser.c" +#line 943 "src/lx/parser.c" } /* END OF ACTION: err-expected-close */ } @@ -956,14 +961,14 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: CHAR */ { -#line 220 "src/lx/parser.act" +#line 222 "src/lx/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 967 "src/lx/parser.c" +#line 972 "src/lx/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -973,7 +978,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: ESC */ { -#line 150 "src/lx/parser.act" +#line 152 "src/lx/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -991,7 +996,7 @@ ZL2_pattern_C_Cbody:; default: ZIc = '\0'; break; /* TODO: handle error */ } -#line 995 "src/lx/parser.c" +#line 1000 "src/lx/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1001,7 +1006,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: HEX */ { -#line 213 "src/lx/parser.act" +#line 215 "src/lx/parser.act" unsigned long u; char *e; @@ -1028,7 +1033,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1032 "src/lx/parser.c" +#line 1037 "src/lx/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1038,7 +1043,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: OCT */ { -#line 186 "src/lx/parser.act" +#line 188 "src/lx/parser.act" unsigned long u; char *e; @@ -1065,7 +1070,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1069 "src/lx/parser.c" +#line 1074 "src/lx/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1078,12 +1083,12 @@ ZL2_pattern_C_Cbody:; /* END OF INLINE: 84 */ /* BEGINNING OF ACTION: pattern-char */ { -#line 250 "src/lx/parser.act" +#line 252 "src/lx/parser.act" /* TODO */ *lex_state->p++ = (ZIc); -#line 1087 "src/lx/parser.c" +#line 1092 "src/lx/parser.c" } /* END OF ACTION: pattern-char */ /* BEGINNING OF INLINE: pattern::body */ @@ -1212,7 +1217,7 @@ ZL2_180:; } /* BEGINNING OF ACTION: op-alt */ { -#line 726 "src/lx/parser.act" +#line 732 "src/lx/parser.act" assert((ZI177) != NULL); assert((ZIb) != NULL); @@ -1223,7 +1228,7 @@ ZL2_180:; goto ZL1; } -#line 1227 "src/lx/parser.c" +#line 1232 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: 180 */ @@ -1292,7 +1297,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-reverse */ { -#line 678 "src/lx/parser.act" +#line 684 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1301,7 +1306,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1305 "src/lx/parser.c" +#line 1310 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ } @@ -1316,7 +1321,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complete */ { -#line 669 "src/lx/parser.act" +#line 675 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1325,7 +1330,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1329 "src/lx/parser.c" +#line 1334 "src/lx/parser.c" } /* END OF ACTION: op-complete */ } @@ -1340,7 +1345,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complement */ { -#line 660 "src/lx/parser.act" +#line 666 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1349,7 +1354,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1353 "src/lx/parser.c" +#line 1358 "src/lx/parser.c" } /* END OF ACTION: op-complement */ } @@ -1399,7 +1404,7 @@ ZL2_186:; } /* BEGINNING OF ACTION: op-intersect */ { -#line 715 "src/lx/parser.act" +#line 721 "src/lx/parser.act" assert((ZI183) != NULL); assert((ZIb) != NULL); @@ -1410,7 +1415,7 @@ ZL2_186:; goto ZL1; } -#line 1414 "src/lx/parser.c" +#line 1419 "src/lx/parser.c" } /* END OF ACTION: op-intersect */ /* BEGINNING OF INLINE: 186 */ @@ -1456,7 +1461,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ } /* BEGINNING OF ACTION: op-alt */ { -#line 726 "src/lx/parser.act" +#line 732 "src/lx/parser.act" assert((ZIold_Hexit) != NULL); assert((ZInew_Hexit) != NULL); @@ -1467,7 +1472,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ goto ZL1; } -#line 1471 "src/lx/parser.c" +#line 1476 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-to-mappings::list-of-zone-to-mappings-x */ @@ -1555,7 +1560,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-cross */ { -#line 576 "src/lx/parser.act" +#line 582 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1600,7 +1605,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1604 "src/lx/parser.c" +#line 1609 "src/lx/parser.c" } /* END OF ACTION: op-cross */ /* BEGINNING OF INLINE: 197 */ @@ -1614,7 +1619,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-qmark */ { -#line 621 "src/lx/parser.act" +#line 627 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1659,7 +1664,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1663 "src/lx/parser.c" +#line 1668 "src/lx/parser.c" } /* END OF ACTION: op-qmark */ /* BEGINNING OF INLINE: 197 */ @@ -1673,7 +1678,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-star */ { -#line 526 "src/lx/parser.act" +#line 532 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1723,7 +1728,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1727 "src/lx/parser.c" +#line 1732 "src/lx/parser.c" } /* END OF ACTION: op-star */ /* BEGINNING OF INLINE: 197 */ @@ -1796,7 +1801,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) } /* BEGINNING OF ACTION: op-subtract */ { -#line 704 "src/lx/parser.act" +#line 710 "src/lx/parser.act" assert((*ZI202) != NULL); assert((ZIb) != NULL); @@ -1807,7 +1812,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) goto ZL1; } -#line 1811 "src/lx/parser.c" +#line 1816 "src/lx/parser.c" } /* END OF ACTION: op-subtract */ } @@ -1871,7 +1876,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) } /* BEGINNING OF ACTION: op-concat */ { -#line 687 "src/lx/parser.act" +#line 693 "src/lx/parser.act" assert((*ZI206) != NULL); assert((ZIb) != NULL); @@ -1882,7 +1887,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) goto ZL1; } -#line 1886 "src/lx/parser.c" +#line 1891 "src/lx/parser.c" } /* END OF ACTION: op-concat */ } @@ -1921,13 +1926,13 @@ p_212(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI210, fsm *ZOq) } /* BEGINNING OF ACTION: op-product */ { -#line 699 "src/lx/parser.act" +#line 705 "src/lx/parser.act" fprintf(stderr, "unimplemented\n"); (ZIq) = NULL; goto ZL1; -#line 1931 "src/lx/parser.c" +#line 1936 "src/lx/parser.c" } /* END OF ACTION: op-product */ } @@ -1961,7 +1966,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* BEGINNING OF EXTRACT: RE */ { -#line 237 "src/lx/parser.act" +#line 239 "src/lx/parser.act" assert(lex_state->buf.a[0] == '/'); @@ -1975,13 +1980,13 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ZIf |= RE_ANCHORED; /* regexps in lx are implicitly anchored */ ZIf |= RE_SINGLE; /* we can't assume we're lexing text */ -#line 1979 "src/lx/parser.c" +#line 1984 "src/lx/parser.c" } /* END OF EXTRACT: RE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 264 "src/lx/parser.act" +#line 266 "src/lx/parser.act" size_t len; @@ -2006,19 +2011,19 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2010 "src/lx/parser.c" +#line 2015 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-regex */ { -#line 380 "src/lx/parser.act" +#line 384 "src/lx/parser.act" struct re_err err; assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->opt, (ZIf), &err); + (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->alloc, act_state->opt, (ZIf), &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -2026,16 +2031,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2030 "src/lx/parser.c" +#line 2035 "src/lx/parser.c" } /* END OF ACTION: compile-regex */ /* BEGINNING OF ACTION: free-arr */ { -#line 767 "src/lx/parser.act" +#line 773 "src/lx/parser.act" free((ZIa)); -#line 2039 "src/lx/parser.c" +#line 2044 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2047,7 +2052,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 264 "src/lx/parser.act" +#line 266 "src/lx/parser.act" size_t len; @@ -2072,19 +2077,19 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2076 "src/lx/parser.c" +#line 2081 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-literal */ { -#line 365 "src/lx/parser.act" +#line 369 "src/lx/parser.act" struct re_err err; assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_LITERAL, act_agetc, (ZIa), act_state->opt, 0, &err); + (ZIr) = re_comp(RE_LITERAL, act_agetc, (ZIa), act_state->alloc, act_state->opt, 0, &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -2092,16 +2097,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2096 "src/lx/parser.c" +#line 2101 "src/lx/parser.c" } /* END OF ACTION: compile-literal */ /* BEGINNING OF ACTION: free-arr */ { -#line 767 "src/lx/parser.act" +#line 773 "src/lx/parser.act" free((ZIa)); -#line 2105 "src/lx/parser.c" +#line 2110 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2165,7 +2170,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2185,12 +2190,12 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } } -#line 2189 "src/lx/parser.c" +#line 2194 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 454 "src/lx/parser.act" +#line 460 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2223,7 +2228,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho nmappings++; } -#line 2227 "src/lx/parser.c" +#line 2232 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-from-mappings::list-of-zone-from-mappings-x */ @@ -2257,16 +2262,16 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* BEGINNING OF ACTION: no-zone */ { -#line 516 "src/lx/parser.act" +#line 522 "src/lx/parser.act" (ZIparent) = NULL; -#line 2265 "src/lx/parser.c" +#line 2270 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: make-ast */ { -#line 425 "src/lx/parser.act" +#line 431 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2274,12 +2279,12 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2278 "src/lx/parser.c" +#line 2283 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: make-zone */ { -#line 433 "src/lx/parser.act" +#line 439 "src/lx/parser.act" assert((ZIa) != NULL); @@ -2297,28 +2302,28 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2301 "src/lx/parser.c" +#line 2306 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: no-exit */ { -#line 512 "src/lx/parser.act" +#line 518 "src/lx/parser.act" (ZIexit) = NULL; -#line 2310 "src/lx/parser.c" +#line 2315 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: set-globalzone */ { -#line 501 "src/lx/parser.act" +#line 507 "src/lx/parser.act" assert((ZIa) != NULL); assert((ZIz) != NULL); (ZIa)->global = (ZIz); -#line 2322 "src/lx/parser.c" +#line 2327 "src/lx/parser.c" } /* END OF ACTION: set-globalzone */ p_list_Hof_Hthings (lex_state, act_state, ZIa, ZIz, ZIexit); @@ -2342,11 +2347,11 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 809 "src/lx/parser.act" +#line 815 "src/lx/parser.act" err_expected(lex_state, "EOF"); -#line 2350 "src/lx/parser.c" +#line 2355 "src/lx/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2359,7 +2364,7 @@ ZL1:; { /* BEGINNING OF ACTION: make-ast */ { -#line 425 "src/lx/parser.act" +#line 431 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2367,17 +2372,17 @@ ZL1:; goto ZL4; } -#line 2371 "src/lx/parser.c" +#line 2376 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: err-syntax */ { -#line 777 "src/lx/parser.act" +#line 783 "src/lx/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 2381 "src/lx/parser.c" +#line 2386 "src/lx/parser.c" } /* END OF ACTION: err-syntax */ } @@ -2434,7 +2439,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ } /* BEGINNING OF ACTION: add-mapping */ { -#line 454 "src/lx/parser.act" +#line 460 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2467,12 +2472,12 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ nmappings++; } -#line 2471 "src/lx/parser.c" +#line 2476 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF ACTION: clone */ { -#line 757 "src/lx/parser.act" +#line 763 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2482,7 +2487,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ goto ZL1; } -#line 2486 "src/lx/parser.c" +#line 2491 "src/lx/parser.c" } /* END OF ACTION: clone */ } @@ -2518,11 +2523,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: err-expected-map */ { -#line 785 "src/lx/parser.act" +#line 791 "src/lx/parser.act" err_expected(lex_state, "'->'"); -#line 2526 "src/lx/parser.c" +#line 2531 "src/lx/parser.c" } /* END OF ACTION: err-expected-map */ } @@ -2533,12 +2538,12 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) case (TOK_TOKEN): /* BEGINNING OF EXTRACT: TOKEN */ { -#line 225 "src/lx/parser.act" +#line 227 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ -#line 2542 "src/lx/parser.c" +#line 2547 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ break; @@ -2552,11 +2557,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: no-token */ { -#line 508 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIt) = NULL; -#line 2560 "src/lx/parser.c" +#line 2565 "src/lx/parser.c" } /* END OF ACTION: no-token */ } @@ -2645,11 +2650,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-semi */ { -#line 793 "src/lx/parser.act" +#line 799 "src/lx/parser.act" err_expected(lex_state, "';'"); -#line 2653 "src/lx/parser.c" +#line 2658 "src/lx/parser.c" } /* END OF ACTION: err-expected-semi */ } @@ -2696,16 +2701,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 516 "src/lx/parser.act" +#line 522 "src/lx/parser.act" (ZIto) = NULL; -#line 2704 "src/lx/parser.c" +#line 2709 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 454 "src/lx/parser.act" +#line 460 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2738,7 +2743,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2742 "src/lx/parser.c" +#line 2747 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -2750,7 +2755,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: make-zone */ { -#line 433 "src/lx/parser.act" +#line 439 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -2768,12 +2773,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 2772 "src/lx/parser.c" +#line 2777 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 454 "src/lx/parser.act" +#line 460 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2806,7 +2811,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2810 "src/lx/parser.c" +#line 2815 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ p_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Hof_Hzone_Hfrom_Hmappings_Hx (lex_state, act_state, *ZIa, *ZIz, ZIchild, *ZIexit); @@ -2830,11 +2835,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-to */ { -#line 797 "src/lx/parser.act" +#line 803 "src/lx/parser.act" err_expected(lex_state, "'..'"); -#line 2838 "src/lx/parser.c" +#line 2843 "src/lx/parser.c" } /* END OF ACTION: err-expected-to */ } @@ -2859,34 +2864,36 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 516 "src/lx/parser.act" +#line 522 "src/lx/parser.act" (ZIx) = NULL; -#line 2867 "src/lx/parser.c" +#line 2872 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: no-token */ { -#line 508 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIy) = NULL; -#line 2876 "src/lx/parser.c" +#line 2881 "src/lx/parser.c" } /* END OF ACTION: no-token */ /* BEGINNING OF ACTION: regex-any */ { -#line 396 "src/lx/parser.act" +#line 400 "src/lx/parser.act" fsm_state_t start, end; - (ZIw) = fsm_new(act_state->opt); + (ZIw) = fsm_new(act_state->alloc); if ((ZIw) == NULL) { perror("fsm_new"); goto ZL5; } + fsm_setoptions((ZIw), act_state->opt); + if (!fsm_addstate((ZIw), &start)) { perror("fsm_addstate"); fsm_free((ZIw)); @@ -2908,12 +2915,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit fsm_setstart((ZIw), start); fsm_setend((ZIw), end, 1); -#line 2912 "src/lx/parser.c" +#line 2919 "src/lx/parser.c" } /* END OF ACTION: regex-any */ /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZIw) != NULL); @@ -2933,12 +2940,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 2937 "src/lx/parser.c" +#line 2944 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 454 "src/lx/parser.act" +#line 460 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2971,7 +2978,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2975 "src/lx/parser.c" +#line 2982 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -2998,11 +3005,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 817 "src/lx/parser.act" +#line 823 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3006 "src/lx/parser.c" +#line 3013 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3018,16 +3025,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: no-exit */ { -#line 512 "src/lx/parser.act" +#line 518 "src/lx/parser.act" (ZIr2) = NULL; -#line 3026 "src/lx/parser.c" +#line 3033 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: make-zone */ { -#line 433 "src/lx/parser.act" +#line 439 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -3045,12 +3052,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3049 "src/lx/parser.c" +#line 3056 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 454 "src/lx/parser.act" +#line 460 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -3083,7 +3090,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 3087 "src/lx/parser.c" +#line 3094 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: 131 */ @@ -3102,11 +3109,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 817 "src/lx/parser.act" +#line 823 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3110 "src/lx/parser.c" +#line 3117 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3150,11 +3157,11 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-bind */ { -#line 789 "src/lx/parser.act" +#line 795 "src/lx/parser.act" err_expected(lex_state, "'='"); -#line 3158 "src/lx/parser.c" +#line 3165 "src/lx/parser.c" } /* END OF ACTION: err-expected-bind */ } @@ -3169,7 +3176,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: add-binding */ { -#line 486 "src/lx/parser.act" +#line 492 "src/lx/parser.act" struct var *v; @@ -3186,7 +3193,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3190 "src/lx/parser.c" +#line 3197 "src/lx/parser.c" } /* END OF ACTION: add-binding */ } @@ -3212,7 +3219,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: deref-var */ { -#line 281 "src/lx/parser.act" +#line 283 "src/lx/parser.act" struct ast_zone *z; @@ -3239,7 +3246,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3243 "src/lx/parser.c" +#line 3250 "src/lx/parser.c" } /* END OF ACTION: deref-var */ p_197 (lex_state, act_state, ZI273, &ZI210); @@ -3255,7 +3262,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: subtract-exit */ { -#line 737 "src/lx/parser.act" +#line 743 "src/lx/parser.act" assert((ZI278) != NULL); @@ -3275,7 +3282,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 3279 "src/lx/parser.c" +#line 3286 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, ZIa, ZIz, ZIexit, &ZI279, &ZI280); @@ -3298,10 +3305,10 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 881 "src/lx/parser.act" +#line 888 "src/lx/parser.act" - struct ast *lx_parse(FILE *f, const struct fsm_options *opt) { + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -3334,6 +3341,7 @@ ZL1:; act_state = &act_state_s; act_state->opt = opt; + act_state->alloc = alloc; if (print_progress) { act_state->zn = 0; @@ -3361,6 +3369,6 @@ ZL1:; return ast; } -#line 3365 "src/lx/parser.c" +#line 3373 "src/lx/parser.c" /* END OF FILE */ diff --git a/src/lx/parser.h b/src/lx/parser.h index 970fe2b0e..53085c140 100644 --- a/src/lx/parser.h +++ b/src/lx/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 140 "src/lx/parser.act" +#line 142 "src/lx/parser.act" #include @@ -20,18 +20,19 @@ typedef struct act_state * act_state; typedef struct ast * ast; - struct ast *lx_parse(FILE *f, const struct fsm_options *opt); + struct ast *lx_parse(FILE *f, + const struct fsm_alloc *alloc, const struct fsm_options *opt); -#line 26 "src/lx/parser.h" +#line 27 "src/lx/parser.h" /* BEGINNING OF FUNCTION DECLARATIONS */ extern void p_lx(lex_state, act_state, ast *); /* BEGINNING OF TRAILER */ -#line 883 "src/lx/parser.act" +#line 890 "src/lx/parser.act" -#line 36 "src/lx/parser.h" +#line 37 "src/lx/parser.h" /* END OF FILE */ diff --git a/src/re/main.c b/src/re/main.c index d6ee29f85..16c152cc0 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -55,8 +55,6 @@ struct match { struct match *next; }; -static struct fsm_options opt; - static void usage(void) { @@ -684,6 +682,12 @@ parse_flags(const char *arg, enum re_flags *flags) int main(int argc, char *argv[]) { + static const struct fsm_options zero_options; + + /* TODO: use alloc hooks for -Q accounting */ + struct fsm_alloc *alloc = NULL; + struct fsm_options opt; + struct fsm *fsm; struct fsm *(*join)(struct fsm *, struct fsm *, struct fsm_combine_info *); @@ -706,6 +710,8 @@ main(int argc, char *argv[]) atexit(do_fsm_cleanup); + opt = zero_options; + /* note these defaults are the opposite than for fsm(1) */ opt.anonymous_states = 1; opt.consolidate_edges = 1; @@ -943,7 +949,7 @@ main(int argc, char *argv[]) } else { struct fsm *fsm; - fsm = fsm_parse(f, &opt); + fsm = fsm_parse(f, alloc, &opt); if (fsm == NULL) { perror("fsm_parse"); return EXIT_FAILURE; @@ -989,12 +995,14 @@ main(int argc, char *argv[]) flags |= RE_MULTI; - fsm = fsm_new(&opt); + fsm = fsm_new(alloc); if (fsm == NULL) { perror("fsm_new"); return EXIT_FAILURE; } + fsm_setoptions(fsm, &opt); + fsm_to_cleanup = fsm; { @@ -1019,9 +1027,9 @@ main(int argc, char *argv[]) f = xopen(argv[i]); if (!fsmfiles) { - new = re_comp(dialect, fsm_fgetc, f, &opt, flags, &err); + new = re_comp(dialect, fsm_fgetc, f, alloc, &opt, flags, &err); } else { - new = fsm_parse(f, &opt); + new = fsm_parse(f, alloc, &opt); if (new == NULL) { perror("fsm_parse"); return EXIT_FAILURE; @@ -1034,7 +1042,7 @@ main(int argc, char *argv[]) s = argv[i]; - new = re_comp(dialect, fsm_sgetc, &s, &opt, flags, &err); + new = re_comp(dialect, fsm_sgetc, &s, alloc, &opt, flags, &err); } if (new == NULL) { diff --git a/src/retest/cvtpcre.c b/src/retest/cvtpcre.c index 633431f7d..3eaa57ad1 100644 --- a/src/retest/cvtpcre.c +++ b/src/retest/cvtpcre.c @@ -427,10 +427,11 @@ state_name(enum state st) return "unk"; } -static const struct fsm_options zero_options; - -int main(int argc, char **argv) +int +main(int argc, char **argv) { + static const struct fsm_options zero_options; + struct str l; enum state state; size_t count, nparsed, linenum, regexp_line; @@ -619,7 +620,7 @@ int main(int argc, char **argv) comp_err = err_zero; - fsm = re_comp(RE_PCRE, fsm_sgetc, &re, &opt, mods, &comp_err); + fsm = re_comp(RE_PCRE, fsm_sgetc, &re, NULL, &opt, mods, &comp_err); re_ok = (fsm != NULL); if (re_ok) { fsm_free(fsm); diff --git a/src/retest/main.c b/src/retest/main.c index ec60f5ca5..c3dd4e74c 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -738,6 +738,7 @@ flagstring(enum re_flags flags, char buf[16]) */ static int process_test_file(const char *filename, + const struct fsm_alloc *alloc, enum re_dialect default_dialect, enum implementation impl, int max_errors, struct error_record *erec) { static const struct fsm_runner init_runner; @@ -977,7 +978,7 @@ process_test_file(const char *filename, } re_str = regexp; - fsm = re_comp(dialect, fsm_sgetc, &re_str, &opt, flags, &err); + fsm = re_comp(dialect, fsm_sgetc, &re_str, alloc, &opt, flags, &err); if (fsm == NULL) { fprintf(stderr, "line %d: error with %s regexp /%s/%s: %s\n", linenum, dialect_name, regexp, flagdesc, re_strerror(err.e)); @@ -1220,6 +1221,9 @@ io(const char *name) int main(int argc, char *argv[]) { + /* TODO: use alloc hooks for -Q accounting as well as watchdog timeouts */ + struct fsm_alloc *alloc = NULL; + enum re_dialect dialect; enum implementation impl; int max_test_errors; @@ -1344,7 +1348,7 @@ main(int argc, char *argv[]) } if (do_watchdog) { - opt.alloc = &watchdog_alloc; + alloc = &watchdog_alloc; } if (argc < 1) { @@ -1380,7 +1384,8 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - nerrs = process_test_file(argv[i], dialect, impl, max_test_errors, &erec); + nerrs = process_test_file(argv[i], alloc, + dialect, impl, max_test_errors, &erec); if (erec.len > 0) { error_record_print(stderr, &erec); diff --git a/src/retest/reperf.c b/src/retest/reperf.c index c826391b5..bcdbc5559 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -651,6 +651,9 @@ static enum error_type perf_case_run(struct perf_case *c, enum halt halt, struct timing *t) { + /* TODO: use alloc hooks for -Q accounting */ + struct fsm_alloc *alloc = NULL; + struct fsm *fsm; struct fsm_runner runner; struct str contents; @@ -685,7 +688,7 @@ perf_case_run(struct perf_case *c, enum halt halt, re = c->regexp.data; - fsm = re_comp(c->dialect, fsm_sgetc, &re, &opt, flags, &comp_err); + fsm = re_comp(c->dialect, fsm_sgetc, &re, alloc, &opt, flags, &comp_err); if (fsm == NULL) { return ERROR_PARSING_REGEXP; } diff --git a/tests/aho_corasick/actest.c b/tests/aho_corasick/actest.c index bf922fa0b..b453060d0 100644 --- a/tests/aho_corasick/actest.c +++ b/tests/aho_corasick/actest.c @@ -164,7 +164,7 @@ int main(int argc, char *argv[]) { flags |= RE_STRINGS_ANCHOR_RIGHT; } - fsm = re_strings(&opt, (const char **)words.list, words.len, flags); + fsm = re_strings(NULL, &opt, (const char **)words.list, words.len, flags); wordlist_finalize(&words); if (fsm == NULL) { diff --git a/tests/capture/captest.c b/tests/capture/captest.c index 18c25dd08..ad2d0c21f 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -133,7 +133,7 @@ captest_run_single(const struct captest_single_fsm_test_info *info) struct fsm * captest_fsm_of_string(const char *string, unsigned end_id) { - struct fsm *fsm = captest_fsm_with_options(); + struct fsm *fsm = fsm_new(NULL); const size_t length = strlen(string); size_t i; @@ -164,18 +164,6 @@ captest_fsm_of_string(const char *string, unsigned end_id) return NULL; } -static struct fsm_options options; - -struct fsm * -captest_fsm_with_options(void) -{ - struct fsm *fsm = NULL; - - /* We currently don't need to set anything custom on this. */ - fsm = fsm_new(&options); - return fsm; -} - int captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, unsigned expected_end_id, const char **msg) diff --git a/tests/capture/captest.h b/tests/capture/captest.h index a9debff6c..60c2944fb 100644 --- a/tests/capture/captest.h +++ b/tests/capture/captest.h @@ -43,9 +43,6 @@ captest_run_single(const struct captest_single_fsm_test_info *info); int captest_getc(void *opaque); -struct fsm * -captest_fsm_with_options(void); - struct fsm * captest_fsm_of_string(const char *string, unsigned end_id); diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index c6d0a25ce..5ee847770 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -153,7 +153,7 @@ det_and_min(const char *tag, struct fsm *fsm) static struct fsm * build_ab_c(void) { - struct fsm *fsm = captest_fsm_with_options(); + struct fsm *fsm = fsm_new(NULL); assert(fsm != NULL); if (!fsm_addstate_bulk(fsm, 4)) { goto fail; } diff --git a/tests/capture/capture5.c b/tests/capture/capture5.c index 0da103114..f103b851e 100644 --- a/tests/capture/capture5.c +++ b/tests/capture/capture5.c @@ -60,7 +60,7 @@ int main(void) { static struct fsm * build(void) { - struct fsm *fsm = captest_fsm_with_options(); + struct fsm *fsm = fsm_new(NULL); if (!fsm_addstate_bulk(fsm, 4)) { goto fail; } diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index 1f5e6f9b6..34e12f116 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -26,7 +26,7 @@ int main(void) size_t nend; size_t nstates, state_ind; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_determinise(fsm); assert(ret == 1); diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index 86a66063c..09ca8e8a3 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -28,7 +28,7 @@ int main(void) size_t i; fsm_state_t end_state; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_determinise(fsm); assert(ret == 1); diff --git a/tests/endids/endids10_minimise_partial_overlap.c b/tests/endids/endids10_minimise_partial_overlap.c index 5a72d05d9..3d7d9e78b 100644 --- a/tests/endids/endids10_minimise_partial_overlap.c +++ b/tests/endids/endids10_minimise_partial_overlap.c @@ -33,14 +33,14 @@ int main(void) const char *regex_ab_star_c = "^ab*c$"; const char *regex_abc = "^abc$"; - struct fsm *fsm_ab_star_c = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_ab_star_c, NULL, 0, NULL); + struct fsm *fsm_ab_star_c = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_ab_star_c, NULL, NULL, 0, NULL); assert(fsm_ab_star_c != NULL); if (!fsm_setendid(fsm_ab_star_c, ENDID_AB_STAR_C)) { assert(!"setendid"); } if (!fsm_determinise(fsm_ab_star_c)) { assert(!"determinise"); } if (!fsm_minimise(fsm_ab_star_c)) { assert(!"minimise"); } - struct fsm *fsm_abc = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_abc, NULL, 0, NULL); + struct fsm *fsm_abc = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_abc, NULL, NULL, 0, NULL); assert(fsm_abc != NULL); if (!fsm_setendid(fsm_abc, ENDID_ABC)) { assert(!"setendid"); } diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 472807eed..5b34da511 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -25,7 +25,7 @@ int main(void) int ret; size_t nstates, state_ind; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index a2feafb18..a8327398a 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -27,7 +27,7 @@ int main(void) int ret; size_t nstates, state_ind; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index 0d6182437..72588015f 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -26,8 +26,8 @@ int main(void) fsm_end_id_t all_endids[2]; unsigned nstates, state_ind; - s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); + s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index a573f1921..e152987c4 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -149,7 +149,7 @@ int main(void) strncpy(&buf[0], patterns[i], sizeof buf); bufp = &buf[0]; - new = re_comp(RE_NATIVE, fsm_sgetc, &bufp, NULL, 0, NULL); + new = re_comp(RE_NATIVE, fsm_sgetc, &bufp, NULL, NULL, 0, NULL); assert(new != NULL); ret = fsm_determinise(new); diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index a4530c93b..a9e095193 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -27,8 +27,8 @@ int main(void) fsm_end_id_t all_endids[2]; unsigned nstates, state_ind; - s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); + s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index 50030ea0d..923e7503d 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -27,8 +27,8 @@ int main(void) fsm_end_id_t all_endids[2]; unsigned nstates, state_ind; - s1 = "(abc|def)"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); + s1 = "(abc|def)"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index e237c2881..37c7baced 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -60,14 +60,14 @@ int main(void) size_t nstates, ninfo, state_ind, info_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index 3021be830..0637b2700 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -72,14 +72,14 @@ int main(void) size_t nstates, ninfo, state_ind, info_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index 939c4b306..975c5c2c5 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -76,14 +76,14 @@ int main(void) size_t nstates, ninfo, state_ind, info_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index 12b36fae0..16db9ac8e 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -27,8 +27,8 @@ int main(void) fsm_end_id_t all_ids[2]; unsigned nstates, state_ind; - s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); + s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index 519399973..0a0d5b66e 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -37,14 +37,14 @@ int main(void) size_t nstates, ninfo, state_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index ae4e94eed..59f677f00 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -9,7 +9,6 @@ #include #include -#include #include /* Note: not "gentest" because then this matches the gen* pattern and @@ -87,8 +86,6 @@ gtest_matches_cb(const struct fsm *fsm, return FSM_GENERATE_MATCHES_CB_RES_HALT; } -static const struct fsm_options options; /* use defaults */ - struct fsm * gtest_fsm_of_matches(const struct exp_matches *ms) { @@ -96,7 +93,7 @@ gtest_fsm_of_matches(const struct exp_matches *ms) for (size_t m_i = 0; m_i < ms->count; m_i++) { const struct exp_match *m = &ms->matches[m_i]; - struct fsm *fsm = fsm_new(&options); + struct fsm *fsm = fsm_new(NULL); assert(fsm != NULL); if (!fsm_addstate_bulk(fsm, m->length + 1)) { diff --git a/tests/minimise/minimise_test_case_list.c b/tests/minimise/minimise_test_case_list.c index 395004703..c3e82a236 100644 --- a/tests/minimise/minimise_test_case_list.c +++ b/tests/minimise/minimise_test_case_list.c @@ -81,7 +81,7 @@ check_minimisation(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); assert(fsm != NULL); if (!fsm_determinise(fsm)) { return 0; diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index 66afa1a38..d72a209f8 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -9,8 +9,6 @@ #include "re/re.h" #include "re/strings.h" -static struct fsm_options opt; - #define MAX_INPUTS 100 static fsm_end_id_t ids[MAX_INPUTS]; @@ -34,7 +32,7 @@ run_test(const char **strings) const int flags = 0; /* not anchored */ - struct fsm *fsm = re_strings_build(s, &opt, flags); + struct fsm *fsm = re_strings_build(s, NULL, NULL, flags); assert(fsm != NULL); /* Each literal string input should match, and the set of From a7d684743c216fc0f4f9bb7054695d034feffbd4 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 20 Jul 2024 11:04:38 +0100 Subject: [PATCH 207/373] Uninitialized value. --- src/libfsm/print/ir.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 4ef3195e5..3703afcba 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -351,6 +351,7 @@ make_state(const struct fsm *fsm, fsm_state_t state, if (fsm_iscomplete(fsm, state)) { mode.state = fsm_findmode(fsm, state, &mode.freq); } else { + mode.state = 0; /* XXX: dummy */ mode.freq = 0; } From 4b2ab25ed08ebc2425f8e37228194170e50c16c1 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 21 Jul 2024 22:17:33 +0100 Subject: [PATCH 208/373] Remove fsm.opt and split print hooks from options. The important part of the diff here is just: ```diff --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -11,7 +11,6 @@ #include #include -#include #include @@ -76,7 +75,6 @@ struct fsm { struct fsm_capture_info *capture_info; struct endid_info *endid_info; - const struct fsm_options *opt; }; ``` Everything else is fallout from not needing to pass around the options struct. This leaves the options struct only used for the print routines. --- fuzz/target.c | 78 ++-- include/fsm/fsm.h | 15 +- include/fsm/options.h | 41 +-- include/fsm/parser.h | 3 +- include/fsm/print.h | 40 ++- include/fsm/vm.h | 4 +- include/re/literal.h | 2 - include/re/re.h | 2 - include/re/strings.h | 4 +- src/fsm/main.c | 8 +- src/libfsm/charset.c | 2 - src/libfsm/clone.c | 4 +- src/libfsm/closure.c | 1 - src/libfsm/concat.c | 6 - src/libfsm/consolidate.c | 3 +- src/libfsm/determinise.c | 2 - src/libfsm/endids.c | 1 + src/libfsm/endids.h | 3 - src/libfsm/epsilons.c | 5 +- src/libfsm/equal.c | 7 +- src/libfsm/fsm.c | 20 +- src/libfsm/internal.h | 2 - src/libfsm/libfsm.syms | 2 - src/libfsm/merge.c | 11 +- src/libfsm/minimise.c | 4 +- src/libfsm/minimise_test_oracle.c | 1 + src/libfsm/parser.act | 7 +- src/libfsm/parser.c | 9 +- src/libfsm/parser.h | 4 +- src/libfsm/print.c | 72 ++-- src/libfsm/print.h | 38 +- src/libfsm/print/api.c | 18 +- src/libfsm/print/awk.c | 37 +- src/libfsm/print/c.c | 57 +-- src/libfsm/print/dot.c | 39 +- src/libfsm/print/fsm.c | 30 +- src/libfsm/print/go.c | 51 +-- src/libfsm/print/ir.c | 7 +- src/libfsm/print/ir.h | 4 +- src/libfsm/print/irdot.c | 15 +- src/libfsm/print/irjson.c | 18 +- src/libfsm/print/json.c | 29 +- src/libfsm/print/llvm.c | 30 +- src/libfsm/print/rust.c | 77 ++-- src/libfsm/print/sh.c | 24 +- src/libfsm/print/vmasm.c | 43 ++- src/libfsm/print/vmc.c | 41 ++- src/libfsm/print/vmdot.c | 40 ++- src/libfsm/print/vmops.c | 56 ++- src/libfsm/reverse.c | 4 - src/libfsm/state.c | 1 + src/libfsm/trim.c | 1 + src/libfsm/union.c | 7 +- src/libfsm/vm.c | 29 +- src/libfsm/walk2.c | 6 +- src/libre/ac.h | 1 - src/libre/ast.h | 2 - src/libre/ast_analysis.h | 2 - src/libre/ast_compile.c | 10 +- src/libre/ast_compile.h | 2 - src/libre/ast_new_from_fsm.c | 1 + src/libre/ast_rewrite.c | 2 +- src/libre/dialect/comp.h | 1 - src/libre/dialect/glob/parser.c | 41 ++- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/like/parser.c | 41 ++- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/literal/parser.c | 29 +- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/native/parser.c | 207 ++++++----- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/pcre/parser.c | 321 +++++++++-------- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/sql/parser.c | 157 ++++---- src/libre/dialect/sql/parser.h | 4 +- src/libre/parser.act | 1 - src/libre/re.c | 11 +- src/libre/re_strings.c | 7 +- src/lx/ast.c | 2 - src/lx/main.c | 20 +- src/lx/parser.act | 16 +- src/lx/parser.c | 339 +++++++++--------- src/lx/parser.h | 11 +- src/lx/print.c | 6 +- src/lx/print.h | 10 +- src/lx/print/c.c | 91 +++-- src/lx/print/dot.c | 39 +- src/lx/print/dump.c | 25 +- src/lx/print/h.c | 7 +- src/lx/print/zdot.c | 5 +- src/lx/tokens.c | 1 + src/re/main.c | 65 ++-- src/retest/cvtpcre.c | 11 +- src/retest/main.c | 4 +- src/retest/reperf.c | 4 +- src/retest/runner.c | 41 ++- src/retest/runner.h | 4 +- tests/aho_corasick/actest.c | 4 +- tests/capture/captest.c | 2 +- tests/capture/capture3.c | 14 +- tests/capture/capture4.c | 10 +- tests/capture/capture5.c | 6 +- tests/capture/capture_concat1.c | 4 +- tests/capture/capture_concat2.c | 4 +- tests/capture/capture_union1.c | 4 +- tests/endids/endids0.c | 2 +- tests/endids/endids0_many_endids.c | 2 +- .../endids10_minimise_partial_overlap.c | 4 +- tests/endids/endids1_determinise.c | 2 +- .../endids/endids1_determinise_and_minimise.c | 2 +- tests/endids/endids2_union.c | 4 +- tests/endids/endids2_union_many_endids.c | 2 +- tests/endids/endids4.c | 4 +- tests/endids/endids5.c | 4 +- tests/endids/endids6.c | 4 +- tests/endids/endids7.c | 4 +- tests/endids/endids7_with_duplicates.c | 4 +- tests/endids/endids8.c | 4 +- tests/endids/endids9.c | 4 +- tests/gen/gen3.c | 2 +- tests/minimise/minimise_test_case_list.c | 6 +- tests/re_strings/testutil.c | 2 +- 122 files changed, 1424 insertions(+), 1306 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index c9d5d4a6e..543891bb9 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -75,8 +75,6 @@ scanner_next(void *opaque) } #endif -static const struct fsm_options opt; - static struct fsm * build(const char *pattern) { @@ -95,7 +93,7 @@ build(const char *pattern) }; time_get(&pre); - fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, RE_MULTI, &err); time_get(&post); delta_usec = time_diff_usec(&pre, &post); total_usec += delta_usec; @@ -133,24 +131,28 @@ build(const char *pattern) } static int -codegen(const struct fsm *fsm) +codegen(const struct fsm *fsm, enum fsm_io io_mode) { + const struct fsm_options opt = { + .io = io_mode, + }; + FILE *dev_null = fopen("/dev/null", "w"); assert(dev_null != NULL); - fsm_print(dev_null, fsm, FSM_PRINT_C); + fsm_print(dev_null, fsm, &opt, NULL, FSM_PRINT_C); fclose(dev_null); return 1; } static int -build_and_codegen(const char *pattern) +build_and_codegen(const char *pattern, enum fsm_io io_mode) { struct fsm *fsm = build(pattern); if (fsm == NULL) { return EXIT_SUCCESS; } - if (!codegen(fsm)) { + if (!codegen(fsm, io_mode)) { return EXIT_SUCCESS; } @@ -176,7 +178,7 @@ shuffle_minimise(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ @@ -245,14 +247,14 @@ shuffle_minimise(const char *pattern) __func__, s_i, pattern, expected_state_count, cp_state_count); fprintf(stderr, "== original input:\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fprintf(stderr, "== expected:\n"); - fsm_print(stderr, oracle_min, FSM_PRINT_FSM); + fsm_print(stderr, oracle_min, NULL, NULL, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print(stderr, cp, FSM_PRINT_FSM); + fsm_print(stderr, cp, NULL, NULL, FSM_PRINT_FSM); fsm_free(cp); fsm_free(oracle_min); @@ -287,7 +289,7 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const .io = io_mode, }; - fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ return EXIT_SUCCESS; @@ -313,28 +315,28 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const /* see if this triggers any asserts */ int r = 0; - r |= fsm_print(f, fsm, FSM_PRINT_AMD64_ATT); - r |= fsm_print(f, fsm, FSM_PRINT_AMD64_GO); - r |= fsm_print(f, fsm, FSM_PRINT_AMD64_NASM); - - r |= fsm_print(f, fsm, FSM_PRINT_API); - r |= fsm_print(f, fsm, FSM_PRINT_AWK); - r |= fsm_print(f, fsm, FSM_PRINT_C); - r |= fsm_print(f, fsm, FSM_PRINT_DOT); - r |= fsm_print(f, fsm, FSM_PRINT_FSM); - r |= fsm_print(f, fsm, FSM_PRINT_GO); - r |= fsm_print(f, fsm, FSM_PRINT_IR); - r |= fsm_print(f, fsm, FSM_PRINT_IRJSON); - r |= fsm_print(f, fsm, FSM_PRINT_JSON); - r |= fsm_print(f, fsm, FSM_PRINT_LLVM); - r |= fsm_print(f, fsm, FSM_PRINT_RUST); - r |= fsm_print(f, fsm, FSM_PRINT_SH); - r |= fsm_print(f, fsm, FSM_PRINT_VMC); - r |= fsm_print(f, fsm, FSM_PRINT_VMDOT); - - r |= fsm_print(f, fsm, FSM_PRINT_VMOPS_C); - r |= fsm_print(f, fsm, FSM_PRINT_VMOPS_H); - r |= fsm_print(f, fsm, FSM_PRINT_VMOPS_MAIN); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AMD64_ATT); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AMD64_GO); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AMD64_NASM); + + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_API); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AWK); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_C); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_DOT); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_FSM); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_GO); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_IR); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_IRJSON); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_JSON); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_LLVM); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_RUST); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_SH); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMC); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMDOT); + + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMOPS_C); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMOPS_H); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMOPS_MAIN); assert(r == 0 || errno != 0); @@ -380,8 +382,12 @@ harness_fuzzer_target(const uint8_t *data, size_t size) const char *pattern = (const char *)data_buf; switch (get_run_mode()) { - case MODE_DEFAULT: - return build_and_codegen(pattern); + case MODE_DEFAULT: { + const uint8_t b0 = data_buf[0]; + const enum fsm_io io_mode = (b0 >> 2) % 3; + + return build_and_codegen(pattern, io_mode); + } case MODE_SHUFFLE_MINIMISE: return shuffle_minimise(pattern); diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index c08b16549..877d5c1bf 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -11,7 +11,6 @@ struct fsm; struct fsm_alloc; -struct fsm_options; struct path; /* XXX */ struct fsm_capture; struct fsm_combine_info; @@ -35,14 +34,12 @@ typedef unsigned int fsm_end_id_t; * from fsm_new() is expected to be passed as the "fsm" argument to the * functions in this API. * - * An options pointer may be passed for control over various details of - * FSM construction and output. This may be NULL, in which case default - * options are used. + * An fsm_alloc pointer may be passed for callbacks on memory allocation. + * This may be NULL, in which case the default libc functions are used. * When non-NULL, the storage pointed to must remain extant until fsm_free(). * * Returns NULL on error; see errno. * TODO: perhaps automatically create a start state, and never have an empty FSM - * TODO: also fsm_parse should create an FSM, not add into an existing one */ struct fsm * fsm_new(const struct fsm_alloc *alloc); @@ -69,14 +66,6 @@ fsm_free(struct fsm *fsm); struct fsm * fsm_clone(const struct fsm *fsm); -/* Returns the options of an FSM */ -const struct fsm_options * -fsm_getoptions(const struct fsm *fsm); - -/* Sets the options of an FSM */ -void -fsm_setoptions(struct fsm *fsm, const struct fsm_options *opts); - /* * Copy the contents of src over dst, and free src. * diff --git a/include/fsm/options.h b/include/fsm/options.h index 5254ce2a9..b5e81fec2 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -7,11 +7,6 @@ #ifndef FSM_OPTIONS_H #define FSM_OPTIONS_H -#include - -struct fsm; -struct fsm_state; - enum fsm_io { FSM_IO_GETC, FSM_IO_STR, @@ -28,6 +23,10 @@ enum fsm_ambig { AMBIG_SINGLE = AMBIG_ERROR | AMBIG_EARLIEST }; +/* + * Print options. + * This is separate to because we also borrow it for libre. + */ struct fsm_options { /* boolean: true indicates to omit names for states in output */ unsigned int anonymous_states:1; @@ -73,38 +72,6 @@ struct fsm_options { /* the name of the enclosing package; NULL to use `prefix` (default). */ const char *package_prefix; - - /* - * Hooks to override generated code. These give an oportunity to - * emit application-specific constructs, especially based on ids - * attached to end states. - * - * These hooks are optional, and default to whatever makes sense - * for the language. - * - * Placement in the output stream depends on the format. - * This replaces an entire "return xyz;" statement for C-like formats, - * but appends extra information for others. - */ - struct fsm_hooks { - /* character pointer, for C code fragment output. NULL for the default. */ - const char *cp; - - int (*args)(FILE *, const struct fsm_options *opt, - void *leaf_opaque); - - /* - * ids[] is sorted and does not have duplicates. - */ - int (*accept)(FILE *, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *leaf_opaque); - - int (*reject)(FILE *, const struct fsm_options *opt, - void *leaf_opaque); - - void *hook_opaque; - } hooks; }; #endif diff --git a/include/fsm/parser.h b/include/fsm/parser.h index 9c512ae50..15e053dd1 100644 --- a/include/fsm/parser.h +++ b/include/fsm/parser.h @@ -3,9 +3,8 @@ struct fsm; struct fsm_alloc; -struct fsm_options; struct fsm * -fsm_parse(FILE *, const struct fsm_alloc *alloc, const struct fsm_options *); +fsm_parse(FILE *, const struct fsm_alloc *alloc); #endif diff --git a/include/fsm/print.h b/include/fsm/print.h index 61a2f7be8..ee889d211 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -7,9 +7,8 @@ #ifndef FSM_PRINT_H #define FSM_PRINT_H -#include - struct fsm; +struct fsm_options; enum fsm_print_lang { FSM_PRINT_NONE, /* No output */ @@ -38,6 +37,38 @@ enum fsm_print_lang { FSM_PRINT_VMOPS_MAIN }; +/* + * Hooks to override generated code. These give an oportunity to + * emit application-specific constructs, especially based on ids + * attached to end states. + * + * These hooks are optional, and default to whatever makes sense + * for the language. + * + * Placement in the output stream depends on the format. + * This replaces an entire "return xyz;" statement for C-like formats, + * but appends extra information for others. + */ +struct fsm_hooks { + /* character pointer, for C code fragment output. NULL for the default. */ + const char *cp; + + int (*args)(FILE *, const struct fsm_options *opt, + void *leaf_opaque, void *hook_opaque); + + /* + * ids[] is sorted and does not have duplicates. + */ + int (*accept)(FILE *, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *leaf_opaque, void *hook_opaque); + + int (*reject)(FILE *, const struct fsm_options *opt, + void *leaf_opaque, void *hook_opaque); + + void *hook_opaque; +}; + /* * Print an FSM to the given file stream. The output is written in the format * specified. @@ -47,7 +78,10 @@ enum fsm_print_lang { * Returns 0, or -1 on error and errno will be set. An errno of ENOTSUP means * the requested IO API is not implemented for this output language. */ -int fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang); +int fsm_print(FILE *f, const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum fsm_print_lang lang); #endif diff --git a/include/fsm/vm.h b/include/fsm/vm.h index b45741f6a..6fab2c1a4 100644 --- a/include/fsm/vm.h +++ b/include/fsm/vm.h @@ -5,6 +5,7 @@ #include struct fsm; +struct fsm_options; struct fsm_dfavm; enum fsm_vm_compile_flags { @@ -34,7 +35,8 @@ struct fsm_dfavm * fsm_vm_compile(const struct fsm *fsm); struct fsm_dfavm * -fsm_vm_compile_with_options(const struct fsm *fsm, struct fsm_vm_compile_opts opts); +fsm_vm_compile_with_options(const struct fsm *fsm, + const struct fsm_options *opt, struct fsm_vm_compile_opts opts); struct fsm_dfavm * fsm_vm_read(FILE *f); diff --git a/include/re/literal.h b/include/re/literal.h index 23d445b2c..a76ceaa07 100644 --- a/include/re/literal.h +++ b/include/re/literal.h @@ -7,7 +7,6 @@ #ifndef RE_LITERAL_H #define RE_LITERAL_H -struct fsm_options; struct re_err; enum re_literal_category { @@ -20,7 +19,6 @@ enum re_literal_category { int re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, enum re_literal_category *category, char **s, size_t *n); diff --git a/include/re/re.h b/include/re/re.h index 33d32b940..20408e98a 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -9,7 +9,6 @@ struct fsm; struct fsm_alloc; -struct fsm_options; enum re_dialect { RE_LIKE, @@ -135,7 +134,6 @@ struct fsm * re_comp(enum re_dialect dialect, re_getchar_fun *f, void *opaque, const struct fsm_alloc *alloc, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err); /* diff --git a/include/re/strings.h b/include/re/strings.h index 1c43e2e95..551ab2eb1 100644 --- a/include/re/strings.h +++ b/include/re/strings.h @@ -10,7 +10,6 @@ #include struct fsm_alloc; -struct fsm_options; struct re_strings; @@ -34,7 +33,7 @@ enum re_strings_flags { }; struct fsm * -re_strings(const struct fsm_alloc *alloc, const struct fsm_options *opt, +re_strings(const struct fsm_alloc *alloc, const char *a[], size_t n, enum re_strings_flags flags); @@ -53,7 +52,6 @@ re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endi struct fsm * re_strings_build(struct re_strings *g, const struct fsm_alloc *alloc, - const struct fsm_options *opt, enum re_strings_flags flags); #endif diff --git a/src/fsm/main.c b/src/fsm/main.c index 084b147aa..9665c9a2c 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -485,7 +485,7 @@ main(int argc, char *argv[]) if ((op & OP_ARITY) == 1) { /* argc < 1 is okay */ - q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), alloc, &opt); + q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), alloc); if (q == NULL) { exit(EXIT_FAILURE); } @@ -495,12 +495,12 @@ main(int argc, char *argv[]) exit(EXIT_FAILURE); } - a = fsm_parse(xopen(argv[0]), alloc, &opt); + a = fsm_parse(xopen(argv[0]), alloc); if (a == NULL) { exit(EXIT_FAILURE); } - b = fsm_parse(xopen(argv[1]), alloc, &opt); + b = fsm_parse(xopen(argv[1]), alloc); if (b == NULL) { exit(EXIT_FAILURE); } @@ -713,7 +713,7 @@ main(int argc, char *argv[]) } } - if (-1 == fsm_print(stdout, fsm, lang)) { + if (-1 == fsm_print(stdout, fsm, &opt, NULL, lang)) { if (errno == ENOTSUP) { fprintf(stderr, "unsupported IO API\n"); } else { diff --git a/src/libfsm/charset.c b/src/libfsm/charset.c index e7c75fd88..3f987eaa8 100644 --- a/src/libfsm/charset.c +++ b/src/libfsm/charset.c @@ -41,8 +41,6 @@ fsm_intersect_charset(struct fsm *a, size_t n, const char *charset) return NULL; } - fsm_setoptions(b, a->opt); - if (!fsm_addstate(b, &state)) { goto error; } diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 897fe027a..f6012f277 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -33,15 +34,12 @@ fsm_clone(const struct fsm *fsm) size_t i; assert(fsm != NULL); - assert(fsm->opt != NULL); new = fsm_new_statealloc(fsm->alloc, fsm->statecount); if (new == NULL) { return NULL; } - fsm_setoptions(new, fsm->opt); - if (!fsm_addstate_bulk(new, fsm->statecount)) { fsm_free(new); return NULL; diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index a9902b6e4..b5b4f969f 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include diff --git a/src/libfsm/concat.c b/src/libfsm/concat.c index 980159f12..9f8780247 100644 --- a/src/libfsm/concat.c +++ b/src/libfsm/concat.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "internal.h" @@ -38,11 +37,6 @@ fsm_concat(struct fsm *a, struct fsm *b, return NULL; } - if (a->opt != b->opt) { - errno = EINVAL; - return NULL; - } - if (a->statecount == 0) { return b; } if (b->statecount == 0) { return a; } diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 79a7d7f01..fc15d4c26 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -74,8 +75,6 @@ fsm_consolidate(const struct fsm *src, goto cleanup; } - fsm_setoptions(dst, src->opt); - for (src_i = 0; src_i < mapping_count; src_i++) { #if LOG_MAPPING fprintf(stderr, "consolidate_mapping[%u]: %u\n", diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 731bc5aef..a32d15a9e 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -185,8 +185,6 @@ fsm_determinise(struct fsm *nfa) goto cleanup; } - fsm_setoptions(dfa, nfa->opt); - #if DUMP_MAPPING { fprintf(stderr, "#### fsm_determinise: mapping\n"); diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index b93f6f9f1..695251f76 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -5,6 +5,7 @@ */ #include +#include #include #include "endids_internal.h" diff --git a/src/libfsm/endids.h b/src/libfsm/endids.h index a5697f10b..a921f7a93 100644 --- a/src/libfsm/endids.h +++ b/src/libfsm/endids.h @@ -1,9 +1,6 @@ #ifndef ENDIDS_H #define ENDIDS_H -#include -#include - int fsm_endid_init(struct fsm *fsm); diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 8dc049eba..f7418947c 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -152,7 +153,7 @@ fsm_remove_epsilons(struct fsm *nfa) #if LOG_RESULT fprintf(stderr, "=== %s: about to update capture actions\n", __func__); - fsm_print(stderr, nfa, FSM_PRINT_FSM); + fsm_print(stderr, nfa, NULL, NULL, FSM_PRINT_FSM); #endif if (!remap_capture_actions(nfa, eclosures)) { @@ -160,7 +161,7 @@ fsm_remove_epsilons(struct fsm *nfa) } #if LOG_RESULT - fsm_print(stderr, nfa, FSM_PRINT_FSM); + fsm_print(stderr, nfa, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif diff --git a/src/libfsm/equal.c b/src/libfsm/equal.c index 21dd12e3a..02489b0b1 100644 --- a/src/libfsm/equal.c +++ b/src/libfsm/equal.c @@ -55,7 +55,7 @@ subtract(const struct fsm *a, const struct fsm *b) assert(a != NULL); assert(b != NULL); - assert(a->opt == b->opt); + assert(a->alloc == b->alloc); if (!ensure_dfa(a, &pa)) { return NULL; @@ -126,11 +126,6 @@ fsm_equal(const struct fsm *a, const struct fsm *b) return -1; } - if (a->opt != b->opt) { - errno = EINVAL; - return -1; - } - if (fsm_empty(a) && fsm_empty(b)) { return 1; } diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index e99e7ba96..ba2d2db26 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -6,12 +6,12 @@ #include #include +#include #include #include #include #include -#include #include #include @@ -65,7 +65,6 @@ fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc) } new->alloc = alloc; - new->opt = NULL; new->statealloc = statealloc; new->statecount = 0; new->endcount = 0; @@ -112,18 +111,6 @@ fsm_free(struct fsm *fsm) f_free(fsm->alloc, fsm); } -const struct fsm_options * -fsm_getoptions(const struct fsm *fsm) -{ - return fsm->opt; -} - -void -fsm_setoptions(struct fsm *fsm, const struct fsm_options *opts) -{ - fsm->opt = opts; -} - void fsm_move(struct fsm *dst, struct fsm *src) { @@ -135,11 +122,6 @@ fsm_move(struct fsm *dst, struct fsm *src) return; /* XXX */ } - if (dst->opt != src->opt) { - errno = EINVAL; - return; /* XXX */ - } - free_contents(dst); dst->states = src->states; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 3ab8545ef..f45e070ff 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -11,7 +11,6 @@ #include #include -#include #include @@ -76,7 +75,6 @@ struct fsm { struct fsm_capture_info *capture_info; struct endid_info *endid_info; - const struct fsm_options *opt; }; struct fsm * diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 28dfb9cc9..8f73c6e13 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -61,8 +61,6 @@ fsm_print_vmops_main fsm_clone fsm_free fsm_new -fsm_setoptions -fsm_getoptions fsm_move fsm_merge fsm_addstate diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index b7e8a5993..f9ec44593 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -204,11 +205,6 @@ fsm_mergeab(struct fsm *a, struct fsm *b, return NULL; } - if (a->opt != b->opt) { - errno = EINVAL; - return NULL; - } - /* * We merge b into a. */ @@ -232,11 +228,6 @@ fsm_merge(struct fsm *a, struct fsm *b, assert(b != NULL); assert(combine_info != NULL); - if (a->opt != b->opt) { - errno = EINVAL; - return NULL; - } - /* * We merge the smaller FSM into the larger FSM. * The number of allocate states is considered first, because realloc diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 576dfeeee..bc03168c3 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -144,10 +144,10 @@ fsm_minimise(struct fsm *fsm) fprintf(stderr, "%s: expected minimal DFA with %zu states, got %zu\n", __func__, exp_count, got_count); fprintf(stderr, "== expected:\n"); - fsm_print(stderr, oracle, FSM_PRINT_FSM); + fsm_print(stderr, oracle, NULL, NULL, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print(stderr, dst, FSM_PRINT_FSM); + fsm_print(stderr, dst, NULL, NULL, FSM_PRINT_FSM); assert(!"non-minimal result"); } diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index d2cfa404d..ec6d0d83c 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index 9a882a24f..2596c3cff 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -158,7 +158,7 @@ struct fsm * fsm_parse(FILE *f, - const struct fsm_alloc *alloc, const struct fsm_options *opt); + const struct fsm_alloc *alloc); @}; @@ -414,7 +414,7 @@ %trailer% @{ - struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { + struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -423,7 +423,6 @@ struct lx *lx; assert(f != NULL); - assert(opt != NULL); lex_state = &lex_state_s; @@ -456,8 +455,6 @@ return NULL; } - fsm_setoptions(new, opt); - ADVANCE_LEXER; switch (CURRENT_TERMINAL) { case TOK_UNKNOWN: diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 026e621ac..e4ac8a31b 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -1351,10 +1351,10 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 482 "src/libfsm/parser.act" +#line 479 "src/libfsm/parser.act" - struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { + struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -1363,7 +1363,6 @@ ZL1:; struct lx *lx; assert(f != NULL); - assert(opt != NULL); lex_state = &lex_state_s; @@ -1396,8 +1395,6 @@ ZL1:; return NULL; } - fsm_setoptions(new, opt); - ADVANCE_LEXER; switch (CURRENT_TERMINAL) { case TOK_UNKNOWN: @@ -1419,6 +1416,6 @@ ZL1:; return new; } -#line 1423 "src/libfsm/parser.c" +#line 1420 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index e7ca6f330..edeebb112 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -18,7 +18,7 @@ struct fsm * fsm_parse(FILE *f, - const struct fsm_alloc *alloc, const struct fsm_options *opt); + const struct fsm_alloc *alloc); #line 24 "src/libfsm/parser.h" @@ -27,7 +27,7 @@ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 483 "src/libfsm/parser.act" +#line 480 "src/libfsm/parser.act" #line 33 "src/libfsm/parser.h" diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 427ef5021..0cc4cd340 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -11,6 +11,7 @@ #include #include +#include #include #include "print.h" @@ -20,76 +21,97 @@ #include "print/ir.h" int -print_hook_args(FILE *f, const struct fsm_options *opt, +print_hook_args(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, int (*default_args)(FILE *f, const struct fsm_options *opt, - void *lang_opaque), + void *lang_opaque, void *hook_opaque), void *lang_opaque) { assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (default_args == NULL) { assert(lang_opaque == NULL); } - if (opt->hooks.accept != NULL) { - return opt->hooks.args(f, opt, lang_opaque); + if (hooks->accept != NULL) { + return hooks->args(f, opt, + lang_opaque, hooks->hook_opaque); } else if (default_args != NULL) { - return default_args(f, opt, lang_opaque); + return default_args(f, opt, + lang_opaque, hooks->hook_opaque); } return 0; } int -print_hook_accept(FILE *f, const struct fsm_options *opt, +print_hook_accept(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const fsm_end_id_t *ids, size_t count, int (*default_accept)(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque), + void *lang_opaque, void *hook_opaque), void *lang_opaque) { assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (default_accept == NULL) { assert(lang_opaque == NULL); } - if (opt->hooks.accept != NULL) { - return opt->hooks.accept(f, opt, ids, count, lang_opaque); + if (hooks->accept != NULL) { + return hooks->accept(f, opt, ids, count, + lang_opaque, hooks->hook_opaque); } else if (default_accept != NULL) { - return default_accept(f, opt, ids, count, lang_opaque); + return default_accept(f, opt, ids, count, + lang_opaque, hooks->hook_opaque); } return 0; } int -print_hook_reject(FILE *f, const struct fsm_options *opt, +print_hook_reject(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, int (*default_reject)(FILE *f, const struct fsm_options *opt, - void *lang_opaque), + void *lang_opaque, void *hook_opaque), void *lang_opaque) { assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (default_reject == NULL) { assert(lang_opaque == NULL); } - if (opt->hooks.reject != NULL) { - return opt->hooks.reject(f, opt, lang_opaque); + if (hooks->reject != NULL) { + return hooks->reject(f, opt, + lang_opaque, hooks->hook_opaque); } else if (default_reject != NULL) { - return default_reject(f, opt, lang_opaque); + return default_reject(f, opt, + lang_opaque, hooks->hook_opaque); } return 0; } int -fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) +fsm_print(FILE *f, const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum fsm_print_lang lang) { + static const struct fsm_options opt_zero; + static const struct fsm_hooks hooks_zero; + struct dfavm_assembler_ir a; struct ir *ir; int r; @@ -106,8 +128,14 @@ fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) vm_print_f *print_vm = NULL; assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + + if (opt == NULL) { + opt = &opt_zero; + } + + if (hooks == NULL) { + hooks = &hooks_zero; + } if (lang == FSM_PRINT_NONE) { return 0; @@ -145,17 +173,17 @@ fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) ir = NULL; if (print_fsm != NULL) { - r = print_fsm(f, fsm->opt, fsm); + r = print_fsm(f, opt, hooks, fsm); goto done; } - ir = make_ir(fsm); + ir = make_ir(fsm, opt); if (ir == NULL) { return -1; } if (print_ir != NULL) { - r = print_ir(f, fsm->opt, ir); + r = print_ir(f, opt, hooks, ir); goto done; } @@ -168,7 +196,7 @@ fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) } if (print_vm != NULL) { - r = print_vm(f, fsm->opt, a.linked); + r = print_vm(f, opt, hooks, a.linked); } dfavm_opasm_finalize_op(&a); diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 8b6a9555a..993eb4440 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -8,31 +8,51 @@ #define FSM_INTERNAL_PRINT_H struct fsm; +struct fsm_options; +struct fsm_hooks; struct ir; struct dfavm_op_ir; int -print_hook_args(FILE *f, const struct fsm_options *opt, +print_hook_args(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, int (*default_args)(FILE *f, const struct fsm_options *opt, - void *lang_opaque), + void *lang_opaque, void *hook_opaque), void *lang_opaque); int -print_hook_accept(FILE *f, const struct fsm_options *opt, +print_hook_accept(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const fsm_end_id_t *ids, size_t count, int (*default_accept)(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque), + void *lang_opaque, void *hook_opaque), void *lang_opaque); int -print_hook_reject(FILE *f, const struct fsm_options *opt, - int (*default_reject)(FILE *f, const struct fsm_options *opt, void *lang_opaque), +print_hook_reject(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + int (*default_reject)(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque), void *lang_opaque); -typedef int fsm_print_f(FILE *f, const struct fsm_options *opt, const struct fsm *fsm); -typedef int ir_print_f(FILE *f, const struct fsm_options *opt, const struct ir *ir); -typedef int vm_print_f(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops); +typedef int fsm_print_f(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm); + +typedef int ir_print_f(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir); + +typedef int vm_print_f(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops); vm_print_f fsm_print_amd64_att; vm_print_f fsm_print_amd64_go; diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index e7232da42..d7cb70f1f 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -43,7 +43,10 @@ rangeclass(unsigned char x, unsigned char y) } int -fsm_print_api(FILE *f, const struct fsm *fsm_orig) +fsm_print_api(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm_orig) { struct fsm *fsm; fsm_state_t start, end; @@ -52,7 +55,8 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) assert(f != NULL); assert(fsm_orig != NULL); - assert(fsm_orig->opt != NULL); + assert(opt != NULL); + assert(hooks != NULL); if (!fsm_has(fsm_orig, fsm_isend)) { errno = EINVAL; @@ -75,13 +79,13 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) /* TODO: leaf callback for opaques */ - if (fsm->opt->fragment) { + if (opt->fragment) { fprintf(f, "\t"); } fprintf(f, "/* Generated by libfsm */\n"); fprintf(f, "\n"); - if (!fsm->opt->fragment) { + if (!opt->fragment) { fprintf(f, "#include LF_HEADER\n"); fprintf(f, "\n"); @@ -95,7 +99,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "int\n"); fprintf(f, "%sfsm(struct fsm *fsm, fsm_state_t *x, fsm_state_t *y)\n", - fsm->opt->prefix != NULL ? fsm->opt->prefix : ""); + opt->prefix != NULL ? opt->prefix : ""); fprintf(f, "{\n"); } @@ -208,7 +212,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) } else if (lo == hi - 1) { fprintf(f, "\tif (!fsm_addedge_literal(fsm, s[%u], s[%u], ", from, to); - c_escputcharlit(f, fsm->opt, lo); + c_escputcharlit(f, opt, lo); fprintf(f, ")) { return 0; }\n"); } else { fprintf(f, "\tfor (i = 0x%02x; i <= 0x%02x; i++) {", @@ -230,7 +234,7 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\n"); - if (!fsm->opt->fragment) { + if (!opt->fragment) { fprintf(f, "\n"); fprintf(f, "\treturn 1;\n"); diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index 54684fb93..de98f95eb 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -51,13 +51,14 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; switch (opt->ambig) { case AMBIG_NONE: @@ -108,13 +109,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "return 0;"); @@ -144,15 +146,17 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, default_reject, NULL); + return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, + return print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, NULL); @@ -179,7 +183,9 @@ print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) /* TODO: eventually to be non-static */ static int -fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, +fsm_print_awkfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, struct dfavm_op_ir *ops, const char *cp, const char *prefix) { @@ -187,6 +193,7 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(cp != NULL); assert(prefix != NULL); @@ -236,7 +243,7 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } fprintf(f, ";"); @@ -244,7 +251,7 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, case VM_OP_FETCH: { fprintf(f, "if (s == \"\") "); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.fetch.end_bits)) { return -1; } fprintf(f, "\n"); @@ -279,13 +286,17 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_awk(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { const char *prefix; const char *cp; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -293,14 +304,14 @@ fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->hooks.cp != NULL) { - cp = opt->hooks.cp; + if (hooks->cp != NULL) { + cp = hooks->cp; } else { cp = "c"; /* XXX */ } if (opt->fragment) { - if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix)) { + if (-1 == fsm_print_awkfrag(f, opt, hooks, ops, cp, prefix)) { return -1; } } else { @@ -322,7 +333,7 @@ fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) fprintf(f, ", l, c) {\n"); - if (-1 == fsm_print_awkfrag(f, opt, ops, cp, prefix)) { + if (-1 == fsm_print_awkfrag(f, opt, hooks, ops, cp, prefix)) { return -1; } diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index c22435baf..24a04e843 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -116,13 +116,14 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; if (-1 == print_ids(f, opt->ambig, ids, count)) { return -1; @@ -133,13 +134,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "return 0;"); @@ -206,7 +208,9 @@ print_groups(FILE *f, const struct fsm_options *opt, } static int -print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, +print_case(FILE *f, const struct ir *ir, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const char *cp, struct ir_state *cs) { @@ -219,7 +223,7 @@ print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); - if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -248,7 +252,7 @@ print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_groups(f, opt, ir_indexof(ir, cs), cs->u.partial.groups, cs->u.partial.n); fprintf(f, "\t\t\tdefault: "); - if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -279,7 +283,7 @@ print_case(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_ranges(f, opt, cs->u.error.error.ranges, cs->u.error.error.n); fprintf(f, " "); - if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -335,7 +339,10 @@ print_stateenum(FILE *f, size_t n) } static int -print_endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) +print_endstates(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir) { unsigned i; @@ -359,7 +366,7 @@ print_endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) fprintf(f, "\tcase S%u: ", i); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, ir->states[i].endids.ids, ir->states[i].endids.count, default_accept, NULL)) @@ -372,7 +379,7 @@ print_endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) /* unexpected EOT */ fprintf(f, "\tdefault: "); - if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -382,7 +389,9 @@ print_endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) } int -fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, +fsm_print_cfrag(FILE *f, const struct ir *ir, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const char *cp) { unsigned i; @@ -407,7 +416,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } fprintf(f, "\n"); - if (-1 == print_case(f, ir, opt, cp, &ir->states[i])) { + if (-1 == print_case(f, ir, opt, hooks, cp, &ir->states[i])) { return -1; } @@ -425,16 +434,19 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } static int -fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) +fsm_print_c_body(FILE *f, const struct ir *ir, + const struct fsm_options *opt, + const struct fsm_hooks *hooks) { const char *cp; assert(f != NULL); assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); - if (opt->hooks.cp != NULL) { - cp = opt->hooks.cp; + if (hooks->cp != NULL) { + cp = hooks->cp; } else { switch (opt->io) { case FSM_IO_GETC: cp = "c"; break; @@ -465,7 +477,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) break; } - if (-1 == fsm_print_cfrag(f, ir, opt, cp)) { + if (-1 == fsm_print_cfrag(f, ir, opt, hooks, cp)) { return -1; } @@ -473,7 +485,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "\n"); /* end states */ - if (-1 == print_endstates(f, opt, ir)) { + if (-1 == print_endstates(f, opt, hooks, ir)) { return -1; } @@ -481,13 +493,16 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) } int -fsm_print_c(FILE *f, const struct fsm_options *opt, +fsm_print_c(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir *ir) { const char *prefix; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(ir != NULL); if (opt->prefix != NULL) { @@ -497,7 +512,7 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, } if (opt->fragment) { - if (-1 == fsm_print_c_body(f, ir, opt)) { + if (-1 == fsm_print_c_body(f, ir, opt, hooks)) { return -1; } } else { @@ -544,11 +559,11 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, abort(); } - if (opt->hooks.args != NULL) { + if (hooks->args != NULL) { fprintf(stdout, ",\n"); fprintf(stdout, "\t"); - if (-1 == print_hook_args(f, opt, NULL, NULL)) { + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; } } @@ -582,7 +597,7 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, if (ir->n == 0) { fprintf(f, "\treturn 0; /* no matches */\n"); } else { - if (-1 == fsm_print_c_body(f, ir, opt)) { + if (-1 == fsm_print_c_body(f, ir, opt, hooks)) { return -1; } } diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 797bee340..aeb79b254 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -28,7 +28,7 @@ static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { fsm_state_t s; @@ -36,6 +36,8 @@ default_accept(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + (void) hook_opaque; + s = * (fsm_state_t *) lang_opaque; fprintf(f, "label = <"); @@ -63,7 +65,7 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { fsm_state_t s; @@ -71,6 +73,8 @@ default_reject(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + (void) hook_opaque; + s = * (fsm_state_t *) lang_opaque; if (!opt->anonymous_states) { @@ -81,7 +85,10 @@ default_reject(FILE *f, const struct fsm_options *opt, } static int -print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, +print_state(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm, const char *prefix, fsm_state_t s) { struct fsm_edge e; @@ -91,6 +98,7 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(fsm != NULL); assert(prefix != NULL); assert(s < fsm->statecount); @@ -201,7 +209,9 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, } static int -print_dotfrag(FILE *f, const struct fsm_options *opt, +print_dotfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct fsm *fsm, const char *prefix) { fsm_state_t s; @@ -216,7 +226,7 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, if (!opt->anonymous_states) { fprintf(f, "\t%sS%-2u [ ", prefix, s); - if (-1 == print_hook_reject(f, opt, default_reject, &s)) { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, &s)) { return -1; } @@ -226,9 +236,10 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, fsm_end_id_t *ids; size_t count; - ids = NULL; count = fsm_endid_count(fsm, s); - if (count > 0) { + if (count == 0) { + ids = NULL; + } else { int res; ids = f_malloc(fsm->alloc, count * sizeof *ids); @@ -247,7 +258,7 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, fprintf(f, ", "); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, ids, count, default_accept, &s)) { @@ -261,7 +272,7 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, /* TODO: show example here, unless !opt->comments */ - if (-1 == print_state(f, opt, fsm, prefix, s)) { + if (-1 == print_state(f, opt, hooks, fsm, prefix, s)) { return -1; } } @@ -270,13 +281,17 @@ print_dotfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_dot(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) +fsm_print_dot(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm) { const char *prefix; assert(f != NULL); assert(fsm != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -285,7 +300,7 @@ fsm_print_dot(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) } if (opt->fragment) { - if (-1 == print_dotfrag(f, opt, fsm, prefix)) { + if (-1 == print_dotfrag(f, opt, hooks, fsm, prefix)) { return -1; } } else { @@ -314,7 +329,7 @@ fsm_print_dot(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) fprintf(f, "\n"); } - if (-1 == print_dotfrag(f, opt, fsm, prefix)) { + if (-1 == print_dotfrag(f, opt, hooks, fsm, prefix)) { return -1; } diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index a029a82c2..cd42e01a3 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -27,13 +27,15 @@ static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); + (void) hook_opaque; + if (count == 0) { return 0; } @@ -117,7 +119,9 @@ findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) } static int -print_state_comments(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, +print_state_comments(FILE *f, + const struct fsm_options *opt, + const struct fsm *fsm, fsm_state_t dst) { fsm_state_t start; @@ -172,14 +176,15 @@ print_char_range(FILE *f, const struct fsm_options *opt, char lower, char upper) } static int -print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, +print_state(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct fsm *fsm, fsm_state_t s) { assert(f != NULL); assert(opt != NULL); if (!fsm_isend(fsm, s)) { - if (-1 == print_hook_reject(f, opt, NULL, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL)) { return -1; } } @@ -281,12 +286,17 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, } int -fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) +fsm_print_fsm(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm) { fsm_state_t s, start; size_t end; assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); assert(fsm != NULL); if (!opt->anonymous_states) { @@ -304,7 +314,7 @@ fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) } for (s = 0; s < fsm->statecount; s++) { - if (-1 == print_state(f, opt, fsm, s)) { + if (-1 == print_state(f, opt, hooks, fsm, s)) { return -1; } } @@ -335,7 +345,9 @@ fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) end--; count = fsm_endid_count(fsm, s); - if (count > 0) { + if (count == 0) { + ids = NULL; + } else { int res; ids = f_malloc(fsm->alloc, count * sizeof *ids); @@ -349,7 +361,7 @@ fsm_print_fsm(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) fprintf(f, "%u", s); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, ids, count, default_accept, NULL)) diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 442d20466..5a0ac9668 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -85,14 +85,15 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { - assert(f != NULL); - assert(opt != NULL); - assert(lang_opaque == NULL); + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "return true"); @@ -105,13 +106,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { - assert(f != NULL); - assert(opt != NULL); - assert(lang_opaque == NULL); + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "{\n\t\treturn false\n\t}\n"); @@ -145,18 +147,20 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, default_reject, NULL); + return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: fprintf(f, "{\n"); fprintf(f, "\t\t"); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, NULL)) @@ -196,7 +200,10 @@ print_fetch(FILE *f, const struct fsm_options *opt) /* TODO: eventually to be non-static */ static int -fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +fsm_print_gofrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, const char *cp) { struct dfavm_op_ir *op; @@ -265,12 +272,12 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - print_end(f, op, opt, op->u.stop.end_bits); + print_end(f, op, opt, hooks, op->u.stop.end_bits); break; case VM_OP_FETCH: print_fetch(f, opt); - print_end(f, op, opt, op->u.fetch.end_bits); + print_end(f, op, opt, hooks, op->u.fetch.end_bits); break; case VM_OP_BRANCH: @@ -290,7 +297,10 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops } int -fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_go(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { const char *prefix; const char *package_prefix; @@ -300,6 +310,7 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -314,7 +325,7 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) } if (opt->fragment) { - if (-1 == fsm_print_gofrag(f, opt, ops, cp)) { + if (-1 == fsm_print_gofrag(f, opt, hooks, ops, cp)) { return -1; } } else { @@ -337,10 +348,10 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) return -1; } - if (opt->hooks.args != NULL) { + if (hooks->args != NULL) { fprintf(stdout, ", "); - if (-1 == print_hook_args(f, opt, NULL, NULL)) { + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; } } @@ -369,7 +380,7 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) fprintf(f, " {\n"); - if (-1 == fsm_print_gofrag(f, opt, ops, cp)) { + if (-1 == fsm_print_gofrag(f, opt, hooks, ops, cp)) { return -1; } diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 3703afcba..083604ff9 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -335,7 +335,6 @@ make_state(const struct fsm *fsm, fsm_state_t state, } mode; assert(fsm != NULL); - assert(fsm->opt != NULL); assert(state < fsm->statecount); /* TODO: IR_TABLE */ @@ -433,13 +432,13 @@ make_state(const struct fsm *fsm, fsm_state_t state, } struct ir * -make_ir(const struct fsm *fsm) +make_ir(const struct fsm *fsm, const struct fsm_options *opt) { fsm_state_t start, i; struct ir *ir; assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); if (!fsm_getstart(fsm, &start)) { goto empty; @@ -498,7 +497,7 @@ make_ir(const struct fsm *fsm) goto error; } - if (!fsm->opt->comments) { + if (!opt->comments) { ir->states[i].example = NULL; } else { char *p; diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index 05ad758e7..02ab9a731 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -18,6 +18,8 @@ * we're done with graph algorithmics. */ +struct fsm_options; + enum ir_strategy { IR_NONE = 1 << 0, IR_SAME = 1 << 1, @@ -105,7 +107,7 @@ struct ir { /* TODO: can pass in mask of allowed strategies */ struct ir * -make_ir(const struct fsm *fsm); +make_ir(const struct fsm *fsm, const struct fsm_options *opt); void free_ir(const struct fsm *fsm, struct ir *ir); diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index db0f559ee..186365203 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -175,7 +175,9 @@ print_grouplinks(FILE *f, unsigned self, } static int -print_state(FILE *f, const struct fsm_options *opt, +print_state(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir *ir, const struct ir_state *cs) { assert(f != NULL); @@ -216,10 +218,10 @@ print_state(FILE *f, const struct fsm_options *opt, /* TODO: leaf callback for dot output */ /* showing hook in addition to existing content */ - if (cs->isend && opt->hooks.accept != NULL) { + if (cs->isend && hooks->accept != NULL) { fprintf(f, "\t\t \n"); fprintf(f, "\t\t\t\n"); @@ -213,7 +220,7 @@ fsm_print_nodes(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) case VM_OP_FETCH: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -311,12 +318,15 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir /* TODO: eventually to be non-static */ static int -fsm_print_vmdotfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_vmdotfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { assert(f != NULL); assert(opt != NULL); - if (-1 == fsm_print_nodes(f, opt, ops)) { + if (-1 == fsm_print_nodes(f, opt, hooks, ops)) { return -1; } fprintf(f, "\n"); @@ -327,13 +337,17 @@ fsm_print_vmdotfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir * } int -fsm_print_vmdot(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_vmdot(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->fragment) { - if (-1 == fsm_print_vmdotfrag(f, opt, ops)) { + if (-1 == fsm_print_vmdotfrag(f, opt, hooks, ops)) { return -1; } } else { @@ -351,7 +365,7 @@ fsm_print_vmdot(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) fprintf(f, "\tstart [ shape = none, label = \"\" ];\n"); fprintf(f, "\tstart -> S0:i0:w [ style = bold ];\n"); - if (-1 == fsm_print_vmdotfrag(f, opt, ops)) { + if (-1 == fsm_print_vmdotfrag(f, opt, hooks, ops)) { return -1; } diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 4520dc702..7eff8d4e9 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -53,7 +53,7 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { const char *prefix; @@ -61,6 +61,8 @@ default_accept(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + (void) hook_opaque; + prefix = lang_opaque; // TODO: print ids @@ -74,7 +76,7 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { const char *prefix; @@ -82,6 +84,8 @@ default_reject(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + (void) hook_opaque; + prefix = lang_opaque; fprintf(f, "%sactionRET, 0", prefix); @@ -119,13 +123,15 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const char *prefix, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - if (-1 == print_hook_reject(f, opt, default_reject, + if (-1 == print_hook_reject(f, opt, hooks, default_reject, (void *) prefix)) { return -1; @@ -133,7 +139,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, break; case VM_END_SUCC: - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, (void *) prefix)) @@ -178,7 +184,10 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) /* TODO: eventually to be non-static */ static int -fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +fsm_print_vmopsfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, const char *prefix) { struct dfavm_op_ir *op; @@ -197,7 +206,7 @@ fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir * if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, prefix, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, prefix, op->u.stop.end_bits)) { return -1; } break; @@ -206,7 +215,7 @@ fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir * if (-1 == print_fetch(f, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, prefix, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, prefix, op->u.fetch.end_bits)) { return -1; } break; @@ -230,13 +239,17 @@ fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir * } int -fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +fsm_print_vmops(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, enum vmops_dialect dialect) { const char *prefix; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -246,7 +259,7 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, if (opt->fragment) { if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix)) { + if (-1 == fsm_print_vmopsfrag(f, opt, hooks, ops, prefix)) { return -1; } } @@ -258,7 +271,7 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, fprintf(f, "#include \"%svmops.h\"\n", prefix); fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, opt, ops, prefix)) { + if (-1 == fsm_print_vmopsfrag(f, opt, hooks, ops, prefix)) { return -1; } fprintf(f, "\t};\n"); @@ -413,20 +426,29 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, } int -fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_vmops_c(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, ops, VMOPS_C); + return fsm_print_vmops(f, opt, hooks, ops, VMOPS_C); } int -fsm_print_vmops_h(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_vmops_h(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, ops, VMOPS_H); + return fsm_print_vmops(f, opt, hooks, ops, VMOPS_H); } int -fsm_print_vmops_main(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_vmops_main(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, ops, VMOPS_MAIN); + return fsm_print_vmops(f, opt, hooks, ops, VMOPS_MAIN); } diff --git a/src/libfsm/reverse.c b/src/libfsm/reverse.c index b8e6d65f5..c7093738b 100644 --- a/src/libfsm/reverse.c +++ b/src/libfsm/reverse.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -29,7 +28,6 @@ fsm_reverse(struct fsm *fsm) struct state_set **epsilons; assert(fsm != NULL); - assert(fsm->opt != NULL); /* * Reversing an FSM means to reverse the language the FSM matches. @@ -59,8 +57,6 @@ fsm_reverse(struct fsm *fsm) return 0; } - fsm_setoptions(new, fsm->opt); - if (!fsm_addstate(new, &new->start)) { fsm_free(new); return 0; diff --git a/src/libfsm/state.c b/src/libfsm/state.c index 063cc78ab..47b7b5395 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/src/libfsm/trim.c b/src/libfsm/trim.c index 8f7a1da37..4f45607bd 100644 --- a/src/libfsm/trim.c +++ b/src/libfsm/trim.c @@ -6,6 +6,7 @@ #include #include +#include #include #include diff --git a/src/libfsm/union.c b/src/libfsm/union.c index efc57ca65..a3b4b230c 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -14,7 +15,6 @@ #include #include #include -#include #include "internal.h" @@ -43,11 +43,6 @@ fsm_union(struct fsm *a, struct fsm *b, return NULL; } - if (a->opt != b->opt) { - errno = EINVAL; - return NULL; - } - if (a->statecount == 0) { fsm_free(a); return b; } if (b->statecount == 0) { fsm_free(b); return a; } diff --git a/src/libfsm/vm.c b/src/libfsm/vm.c index ac896d096..c9205815f 100644 --- a/src/libfsm/vm.c +++ b/src/libfsm/vm.c @@ -12,6 +12,7 @@ #include #include +#include #include #include "internal.h" @@ -84,28 +85,33 @@ cmp_name(int cmp) } struct fsm_dfavm * -fsm_vm_compile_with_options(const struct fsm *fsm, struct fsm_vm_compile_opts opts) +fsm_vm_compile_with_options(const struct fsm *fsm, + const struct fsm_options *opt, + struct fsm_vm_compile_opts vmopts) { static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct ir *ir; struct fsm_dfavm *vm; - ir = make_ir(fsm); + assert(fsm != NULL); + assert(opt != NULL); + + ir = make_ir(fsm, opt); if (ir == NULL) { return NULL; } a = zero; - if (!dfavm_compile_ir(&a, ir, opts)) { + if (!dfavm_compile_ir(&a, ir, vmopts)) { free_ir(fsm, ir); return NULL; } free_ir(fsm, ir); - vm = dfavm_compile_vm(&a, opts); + vm = dfavm_compile_vm(&a, vmopts); if (vm == NULL) { return NULL; } @@ -120,8 +126,19 @@ fsm_vm_compile(const struct fsm *fsm) { // static const struct fsm_vm_compile_opts defaults = { FSM_VM_COMPILE_DEFAULT_FLAGS | FSM_VM_COMPILE_PRINT_IR_PREOPT | FSM_VM_COMPILE_PRINT_IR, NULL }; static const struct fsm_vm_compile_opts defaults = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - - return fsm_vm_compile_with_options(fsm, defaults); + static const struct fsm_options opt_default; + + assert(fsm != NULL); + + /* + * The only fsm_option make_ir() uses is .comments, + * for whether to generate examples. + * + * The caller won't use that, because the only user-facing + * vm interfaces are to interpret for execution. So we use + * default options here. + */ + return fsm_vm_compile_with_options(fsm, &opt_default, defaults); } void diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index 42db6b389..6d6ab022a 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -9,10 +9,10 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -567,8 +567,6 @@ fsm_walk2(const struct fsm *a, const struct fsm *b, goto error; } - fsm_setoptions(data.new, a->opt); - data.states = tuple_set_create(data.new->alloc, cmp_walk2_tuple); if (data.states == NULL) { goto error; @@ -603,8 +601,6 @@ fsm_walk2(const struct fsm *a, const struct fsm *b, return NULL; } - fsm_setoptions(new, a->opt); - return new; error: diff --git a/src/libre/ac.h b/src/libre/ac.h index edabca27c..c8a4f195f 100644 --- a/src/libre/ac.h +++ b/src/libre/ac.h @@ -11,7 +11,6 @@ struct fsm; struct fsm_state; -struct fsm_options; struct trie_graph; diff --git a/src/libre/ast.h b/src/libre/ast.h index d443db9af..9a2c9a877 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -312,10 +312,8 @@ ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const /* XXX: exposed for sake of re(1) printing an ast; * it's not part of the API proper */ -struct fsm_options; struct ast * re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, int *unsatisfiable); const char * diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index 0628bd127..013db708c 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -12,8 +12,6 @@ */ struct ast; -struct re_err; -struct fsm_options; enum ast_analysis_res { AST_ANALYSIS_OK, diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 32dcd8bdf..0ea0b08d8 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -193,14 +193,13 @@ fsm_unionxy(struct fsm *a, struct fsm *b, fsm_state_t x, fsm_state_t y) static struct fsm * expr_compile(struct ast_expr *e, enum re_flags flags, const struct fsm_alloc *alloc, - const struct fsm_options *opt, struct re_err *err) { struct ast ast; ast.expr = e; - return ast_compile(&ast, flags, alloc, opt, err); + return ast_compile(&ast, flags, alloc, err); } static int @@ -885,13 +884,13 @@ comp_iter(struct comp_env *env, re_flags &= ~(unsigned)RE_REVERSE; a = expr_compile(n->u.subtract.a, re_flags, - env->fsm->alloc, fsm_getoptions(env->fsm), env->err); + env->fsm->alloc, env->err); if (a == NULL) { return 0; } b = expr_compile(n->u.subtract.b, re_flags, - env->fsm->alloc, fsm_getoptions(env->fsm), env->err); + env->fsm->alloc, env->err); if (b == NULL) { fsm_free(a); return 0; @@ -962,7 +961,6 @@ struct fsm * ast_compile(const struct ast *ast, enum re_flags re_flags, const struct fsm_alloc *alloc, - const struct fsm_options *opt, struct re_err *err) { fsm_state_t x, y; @@ -975,8 +973,6 @@ ast_compile(const struct ast *ast, return NULL; } - fsm_setoptions(fsm, opt); - if (!fsm_addstate(fsm, &x)) { goto error; } diff --git a/src/libre/ast_compile.h b/src/libre/ast_compile.h index 86a7bb882..4fed57af4 100644 --- a/src/libre/ast_compile.h +++ b/src/libre/ast_compile.h @@ -13,14 +13,12 @@ struct fsm; struct fsm_alloc; -struct fsm_options; struct re_err; struct fsm * ast_compile(const struct ast *ast, enum re_flags re_flags, const struct fsm_alloc *alloc, - const struct fsm_options *opt, struct re_err *err); #endif diff --git a/src/libre/ast_new_from_fsm.c b/src/libre/ast_new_from_fsm.c index 64322bc02..bafd998c6 100644 --- a/src/libre/ast_new_from_fsm.c +++ b/src/libre/ast_new_from_fsm.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/libre/ast_rewrite.c b/src/libre/ast_rewrite.c index 6f13447af..592dd58d5 100644 --- a/src/libre/ast_rewrite.c +++ b/src/libre/ast_rewrite.c @@ -147,7 +147,7 @@ compile_subexpr(struct ast_expr *e, const struct fsm_alloc *alloc, return 0; } - fsm = ast_compile(&ast, flags | RE_ANCHORED, alloc, NULL, NULL); + fsm = ast_compile(&ast, flags | RE_ANCHORED, alloc, NULL); if (fsm == NULL) { return 0; } diff --git a/src/libre/dialect/comp.h b/src/libre/dialect/comp.h index d223ab90e..4671bd921 100644 --- a/src/libre/dialect/comp.h +++ b/src/libre/dialect/comp.h @@ -15,7 +15,6 @@ typedef struct ast * re_dialect_parse_fun(re_getchar_fun *getchar, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end); diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 746220d4c..c8f021380 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 22 "src/libre/parser.act" +#line 275 "src/libre/parser.act" #include @@ -304,7 +304,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -322,7 +322,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -348,7 +348,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -358,7 +358,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -378,7 +378,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -397,7 +397,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { @@ -418,7 +418,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -428,7 +428,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -440,7 +440,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 808 "src/libre/parser.act" +#line 809 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -449,7 +449,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -478,7 +478,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 704 "src/libre/parser.act" +#line 708 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -490,7 +490,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -523,7 +523,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -537,7 +537,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -558,7 +558,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -575,7 +575,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -601,7 +601,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 753 "src/libre/parser.act" +#line 757 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -626,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1052 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" static int @@ -646,7 +646,6 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { @@ -783,6 +782,6 @@ ZL0:; return NULL; } -#line 787 "src/libre/dialect/glob/parser.c" +#line 786 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index c0ce31ed6..ec618caca 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 281 "src/libre/parser.act" +#line 292 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1208 "src/libre/parser.act" +#line 1209 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index 6543095e3..64bfe1078 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 22 "src/libre/parser.act" +#line 275 "src/libre/parser.act" #include @@ -304,7 +304,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -322,7 +322,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -348,7 +348,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -358,7 +358,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -378,7 +378,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -397,7 +397,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { @@ -418,7 +418,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -428,7 +428,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -440,7 +440,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 808 "src/libre/parser.act" +#line 809 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -449,7 +449,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -478,7 +478,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 704 "src/libre/parser.act" +#line 708 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -490,7 +490,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -523,7 +523,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -537,7 +537,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -558,7 +558,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -575,7 +575,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -601,7 +601,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 753 "src/libre/parser.act" +#line 757 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -626,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1052 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" static int @@ -646,7 +646,6 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { @@ -783,6 +782,6 @@ ZL0:; return NULL; } -#line 787 "src/libre/dialect/like/parser.c" +#line 786 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index 8563a04a6..f6c87ad7b 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 281 "src/libre/parser.act" +#line 292 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1208 "src/libre/parser.act" +#line 1209 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index f52793a0c..44547716b 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 22 "src/libre/parser.act" +#line 275 "src/libre/parser.act" #include @@ -304,7 +304,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -322,7 +322,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -349,7 +349,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -363,7 +363,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -384,7 +384,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -401,7 +401,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* END OF INLINE: 117 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -427,7 +427,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 753 "src/libre/parser.act" +#line 757 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -467,7 +467,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -490,7 +490,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -506,7 +506,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 704 "src/libre/parser.act" +#line 708 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -518,7 +518,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -539,7 +539,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1052 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" static int @@ -559,7 +559,6 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { @@ -696,6 +695,6 @@ ZL0:; return NULL; } -#line 700 "src/libre/dialect/literal/parser.c" +#line 699 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index 672d23e8e..be58db4ea 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 281 "src/libre/parser.act" +#line 292 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1208 "src/libre/parser.act" +#line 1209 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 05a3ff0e7..809383bf8 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 22 "src/libre/parser.act" +#line 275 "src/libre/parser.act" #include @@ -326,7 +326,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -349,7 +349,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 386 "src/libre/parser.act" +#line 391 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -385,7 +385,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 527 "src/libre/parser.act" +#line 535 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -443,7 +443,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 484 "src/libre/parser.act" +#line 492 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -499,7 +499,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 141 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -531,7 +531,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI216 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -545,7 +545,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); @@ -555,7 +555,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -586,7 +586,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 627 "src/libre/parser.act" +#line 636 "src/libre/parser.act" unsigned long u; char *e; @@ -618,7 +618,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI219 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -636,7 +636,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); @@ -646,7 +646,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -696,7 +696,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -709,7 +709,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /* UNREACHED */ + /*UNREACHED*/ case (ERROR_TERMINAL): return; default: @@ -736,7 +736,7 @@ p_154(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI155 = '-'; ZI156 = lex_state->lx.start; @@ -760,7 +760,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 718 "src/libre/parser.act" +#line 722 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; @@ -795,7 +795,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -815,7 +815,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -849,7 +849,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -875,7 +875,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 386 "src/libre/parser.act" +#line 391 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -914,7 +914,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 527 "src/libre/parser.act" +#line 535 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -975,7 +975,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 484 "src/libre/parser.act" +#line 492 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1031,7 +1031,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 109 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1064,7 +1064,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1096,7 +1096,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 386 "src/libre/parser.act" +#line 391 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1141,7 +1141,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 527 "src/libre/parser.act" +#line 535 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1208,7 +1208,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 647 "src/libre/parser.act" +#line 648 "src/libre/parser.act" ZI243 = DIALECT_CLASS(lex_state->buf.a); if (ZI243 == NULL) { @@ -1241,7 +1241,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 484 "src/libre/parser.act" +#line 492 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1325,7 +1325,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 647 "src/libre/parser.act" +#line 648 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1349,7 +1349,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 844 "src/libre/parser.act" +#line 845 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); @@ -1389,7 +1389,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 318 "src/libre/parser.act" +#line 319 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI167 = lex_state->lx.end; @@ -1403,7 +1403,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1430,7 +1430,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 334 "src/libre/parser.act" +#line 335 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI186 = lex_state->lx.end; @@ -1444,7 +1444,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1457,7 +1457,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 885 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1471,7 +1471,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -1493,7 +1493,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 326 "src/libre/parser.act" +#line 327 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI178 = lex_state->lx.end; @@ -1507,7 +1507,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1520,7 +1520,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 966 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1577,7 +1577,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 342 "src/libre/parser.act" +#line 343 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI193 = lex_state->lx.end; @@ -1591,7 +1591,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1604,7 +1604,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 966 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1647,7 +1647,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 885 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1661,7 +1661,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -1693,7 +1693,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 350 "src/libre/parser.act" +#line 351 "src/libre/parser.act" ZI200 = ']'; ZI201 = lex_state->lx.start; @@ -1709,7 +1709,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 767 "src/libre/parser.act" +#line 768 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -1728,7 +1728,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 360 "src/libre/parser.act" +#line 361 "src/libre/parser.act" ZIcrange = '-'; ZI203 = lex_state->lx.start; @@ -1744,7 +1744,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { @@ -1756,7 +1756,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; @@ -1767,7 +1767,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 767 "src/libre/parser.act" +#line 768 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -1785,7 +1785,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 725 "src/libre/parser.act" +#line 729 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -1821,7 +1821,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -1842,7 +1842,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { @@ -1865,7 +1865,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); @@ -1875,7 +1875,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI183 = '-'; ZI184 = lex_state->lx.start; @@ -1896,7 +1896,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -1935,7 +1935,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 182 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; @@ -1977,7 +1977,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZIc = '-'; ZI149 = lex_state->lx.start; @@ -1993,7 +1993,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -2050,7 +2050,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2088,7 +2088,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2109,7 +2109,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2129,7 +2129,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 711 "src/libre/parser.act" +#line 715 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -2141,7 +2141,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2170,7 +2170,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { @@ -2194,7 +2194,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); @@ -2204,7 +2204,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI196 = '-'; ZI197 = lex_state->lx.start; @@ -2225,7 +2225,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); @@ -2235,7 +2235,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -2294,7 +2294,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -2308,7 +2308,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -2334,7 +2334,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 753 "src/libre/parser.act" +#line 757 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -2422,7 +2422,7 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; @@ -2441,7 +2441,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -2453,7 +2453,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 711 "src/libre/parser.act" +#line 715 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -2485,7 +2485,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 370 "src/libre/parser.act" +#line 371 "src/libre/parser.act" ZI263 = lex_state->lx.start; ZI264 = lex_state->lx.end; @@ -2501,7 +2501,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 627 "src/libre/parser.act" +#line 636 "src/libre/parser.act" unsigned long u; char *e; @@ -2541,7 +2541,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 816 "src/libre/parser.act" +#line 817 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); @@ -2555,7 +2555,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 812 "src/libre/parser.act" +#line 813 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); @@ -2569,7 +2569,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 808 "src/libre/parser.act" +#line 809 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -2582,7 +2582,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 820 "src/libre/parser.act" +#line 821 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -2599,7 +2599,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 697 "src/libre/parser.act" +#line 701 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; @@ -2611,7 +2611,7 @@ ZL1:; /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 820 "src/libre/parser.act" +#line 821 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -2640,7 +2640,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -2650,7 +2650,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -2667,7 +2667,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 942 "src/libre/parser.act" +#line 943 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -2687,7 +2687,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -2701,7 +2701,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { @@ -2725,7 +2725,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 935 "src/libre/parser.act" +#line 936 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { @@ -2765,7 +2765,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 704 "src/libre/parser.act" +#line 708 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -2777,7 +2777,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -2806,7 +2806,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI243)); if ((ZInode) == NULL) { @@ -2826,7 +2826,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 844 "src/libre/parser.act" +#line 845 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI243); @@ -2842,7 +2842,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 772 "src/libre/parser.act" +#line 773 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); @@ -2852,7 +2852,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -2908,7 +2908,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2942,7 +2942,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI247)); if ((ZInode) == NULL) { @@ -2962,7 +2962,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI247); @@ -2978,7 +2978,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 772 "src/libre/parser.act" +#line 773 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); @@ -2988,7 +2988,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -3035,7 +3035,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1052 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" static int @@ -3055,7 +3055,6 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { @@ -3192,6 +3191,6 @@ ZL0:; return NULL; } -#line 3196 "src/libre/dialect/native/parser.c" +#line 3195 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 8b1c4d866..e19648892 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 281 "src/libre/parser.act" +#line 292 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1208 "src/libre/parser.act" +#line 1209 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 2400a0812..78c11d795 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 22 "src/libre/parser.act" +#line 275 "src/libre/parser.act" #include @@ -325,7 +325,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 665 "src/libre/parser.act" +#line 666 "src/libre/parser.act" ZIc = RE_EXTENDED; @@ -335,7 +335,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 800 "src/libre/parser.act" +#line 801 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); @@ -356,7 +356,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 661 "src/libre/parser.act" +#line 662 "src/libre/parser.act" ZIc = RE_ICASE; @@ -366,7 +366,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 800 "src/libre/parser.act" +#line 801 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); @@ -381,7 +381,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 669 "src/libre/parser.act" +#line 670 "src/libre/parser.act" ZIc = RE_SINGLE; @@ -391,7 +391,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 800 "src/libre/parser.act" +#line 801 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); @@ -406,7 +406,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 739 "src/libre/parser.act" +#line 743 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; @@ -451,7 +451,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -474,7 +474,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 442 "src/libre/parser.act" +#line 448 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -522,7 +522,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -538,7 +538,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 386 "src/libre/parser.act" +#line 391 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -574,7 +574,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 527 "src/libre/parser.act" +#line 535 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -632,7 +632,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 484 "src/libre/parser.act" +#line 492 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -685,7 +685,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 426 "src/libre/parser.act" +#line 429 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -707,7 +707,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -726,7 +726,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 155 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -762,7 +762,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -775,7 +775,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /* UNREACHED */ + /*UNREACHED*/ case (ERROR_TERMINAL): return; default: @@ -822,7 +822,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -851,7 +851,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -874,7 +874,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI290)); if ((ZInode) == NULL) { @@ -894,7 +894,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 844 "src/libre/parser.act" +#line 845 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI290); @@ -910,7 +910,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 772 "src/libre/parser.act" +#line 773 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI291)); mark(&act_state->rangeend, &(ZIend)); @@ -920,7 +920,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -980,7 +980,7 @@ p_168(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI169 = '-'; ZI170 = lex_state->lx.start; @@ -1004,7 +1004,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 718 "src/libre/parser.act" +#line 722 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; @@ -1043,7 +1043,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1069,7 +1069,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CONTROL */ { -#line 442 "src/libre/parser.act" +#line 448 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1124,7 +1124,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 386 "src/libre/parser.act" +#line 391 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1163,7 +1163,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 527 "src/libre/parser.act" +#line 535 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1224,7 +1224,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: NOESC */ { -#line 412 "src/libre/parser.act" +#line 417 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1251,7 +1251,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 484 "src/libre/parser.act" +#line 492 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1307,7 +1307,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 426 "src/libre/parser.act" +#line 429 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1329,7 +1329,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -1348,7 +1348,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 111 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1381,7 +1381,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1413,7 +1413,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CONTROL */ { -#line 442 "src/libre/parser.act" +#line 448 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1461,7 +1461,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -1486,7 +1486,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 386 "src/libre/parser.act" +#line 391 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1531,7 +1531,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 527 "src/libre/parser.act" +#line 535 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1598,7 +1598,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 647 "src/libre/parser.act" +#line 648 "src/libre/parser.act" ZI290 = DIALECT_CLASS(lex_state->buf.a); if (ZI290 == NULL) { @@ -1631,7 +1631,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NOESC */ { -#line 412 "src/libre/parser.act" +#line 417 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1651,7 +1651,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1671,7 +1671,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 484 "src/libre/parser.act" +#line 492 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1733,7 +1733,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 426 "src/libre/parser.act" +#line 429 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1755,7 +1755,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -1801,7 +1801,7 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 683 "src/libre/parser.act" +#line 687 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; @@ -1835,7 +1835,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 647 "src/libre/parser.act" +#line 648 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1859,7 +1859,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 844 "src/libre/parser.act" +#line 845 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); @@ -1899,7 +1899,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 318 "src/libre/parser.act" +#line 319 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI181 = lex_state->lx.end; @@ -1913,7 +1913,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1940,7 +1940,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 334 "src/libre/parser.act" +#line 335 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI200 = lex_state->lx.end; @@ -1954,7 +1954,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1967,7 +1967,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 885 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1981,7 +1981,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -2003,7 +2003,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 326 "src/libre/parser.act" +#line 327 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI192 = lex_state->lx.end; @@ -2017,7 +2017,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2030,7 +2030,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 966 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -2087,7 +2087,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 342 "src/libre/parser.act" +#line 343 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI207 = lex_state->lx.end; @@ -2101,7 +2101,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2114,7 +2114,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 966 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -2157,7 +2157,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 885 "src/libre/parser.act" +#line 886 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -2171,7 +2171,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -2203,7 +2203,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 350 "src/libre/parser.act" +#line 351 "src/libre/parser.act" ZI214 = ']'; ZI215 = lex_state->lx.start; @@ -2219,7 +2219,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 767 "src/libre/parser.act" +#line 768 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -2238,7 +2238,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 360 "src/libre/parser.act" +#line 361 "src/libre/parser.act" ZIcrange = '-'; ZI217 = lex_state->lx.start; @@ -2254,7 +2254,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { @@ -2266,7 +2266,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; @@ -2277,7 +2277,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 767 "src/libre/parser.act" +#line 768 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -2295,7 +2295,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 725 "src/libre/parser.act" +#line 729 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -2338,7 +2338,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZIc = '-'; ZI163 = lex_state->lx.start; @@ -2354,7 +2354,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -2401,7 +2401,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI314)); if ((ZInode) == NULL) { @@ -2421,7 +2421,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI314); @@ -2437,7 +2437,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 772 "src/libre/parser.act" +#line 773 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI315)); mark(&act_state->rangeend, &(ZIend)); @@ -2447,7 +2447,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -2522,7 +2522,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* BEGINNING OF ACTION: count-one */ { -#line 820 "src/libre/parser.act" +#line 821 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -2531,7 +2531,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2578,7 +2578,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI256 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2592,7 +2592,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); @@ -2602,7 +2602,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2656,7 +2656,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2677,7 +2677,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 711 "src/libre/parser.act" +#line 715 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -2689,7 +2689,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2722,7 +2722,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2736,7 +2736,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); @@ -2746,7 +2746,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ { -#line 804 "src/libre/parser.act" +#line 805 "src/libre/parser.act" (ZIn) = AST_COUNT_UNBOUNDED; @@ -2755,7 +2755,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2783,7 +2783,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: COUNT */ { -#line 627 "src/libre/parser.act" +#line 636 "src/libre/parser.act" unsigned long u; char *e; @@ -2811,7 +2811,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI259 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2829,7 +2829,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); @@ -2839,7 +2839,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2885,7 +2885,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -2906,7 +2906,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { @@ -2929,7 +2929,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); @@ -2939,7 +2939,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI197 = '-'; ZI198 = lex_state->lx.start; @@ -2960,7 +2960,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -2999,7 +2999,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 196 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; @@ -3044,7 +3044,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 796 "src/libre/parser.act" +#line 797 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; @@ -3053,7 +3053,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 796 "src/libre/parser.act" +#line 797 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; @@ -3110,7 +3110,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 926 "src/libre/parser.act" +#line 931 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3124,7 +3124,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -3144,7 +3144,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 918 "src/libre/parser.act" +#line 919 "src/libre/parser.act" (ZIflags) = *flags; @@ -3153,7 +3153,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 926 "src/libre/parser.act" +#line 931 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3172,7 +3172,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 922 "src/libre/parser.act" +#line 923 "src/libre/parser.act" *flags = (ZIflags); @@ -3197,7 +3197,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 746 "src/libre/parser.act" +#line 750 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; @@ -3209,7 +3209,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -3250,7 +3250,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3275,7 +3275,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -3292,7 +3292,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 760 "src/libre/parser.act" +#line 764 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -3384,7 +3384,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 647 "src/libre/parser.act" +#line 648 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3408,7 +3408,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { @@ -3439,7 +3439,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { @@ -3463,7 +3463,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); @@ -3473,7 +3473,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI210 = '-'; ZI211 = lex_state->lx.start; @@ -3494,7 +3494,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); @@ -3504,7 +3504,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -3566,7 +3566,7 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; @@ -3585,7 +3585,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -3597,7 +3597,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 711 "src/libre/parser.act" +#line 715 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -3629,7 +3629,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 370 "src/libre/parser.act" +#line 371 "src/libre/parser.act" ZI318 = lex_state->lx.start; ZI319 = lex_state->lx.end; @@ -3645,7 +3645,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 627 "src/libre/parser.act" +#line 636 "src/libre/parser.act" unsigned long u; char *e; @@ -3685,7 +3685,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 816 "src/libre/parser.act" +#line 817 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); @@ -3699,7 +3699,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 812 "src/libre/parser.act" +#line 813 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); @@ -3713,7 +3713,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 808 "src/libre/parser.act" +#line 809 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -3732,7 +3732,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 697 "src/libre/parser.act" +#line 701 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; @@ -3744,7 +3744,7 @@ ZL1:; /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 820 "src/libre/parser.act" +#line 821 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -3774,7 +3774,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -3788,7 +3788,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -3814,7 +3814,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 753 "src/libre/parser.act" +#line 757 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -3850,7 +3850,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -3860,7 +3860,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -3877,7 +3877,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 942 "src/libre/parser.act" +#line 943 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3894,7 +3894,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 949 "src/libre/parser.act" +#line 950 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3922,7 +3922,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 787 "src/libre/parser.act" +#line 789 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; @@ -3932,7 +3932,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { @@ -3944,7 +3944,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { @@ -3956,7 +3956,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 889 "src/libre/parser.act" +#line 890 "src/libre/parser.act" (ZIcr) = '\r'; @@ -3965,7 +3965,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { @@ -3977,7 +3977,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 893 "src/libre/parser.act" +#line 894 "src/libre/parser.act" (ZInl) = '\n'; @@ -3986,7 +3986,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { @@ -3998,7 +3998,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; @@ -4009,7 +4009,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; @@ -4020,7 +4020,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -4032,7 +4032,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; @@ -4043,7 +4043,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; @@ -4063,7 +4063,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 918 "src/libre/parser.act" +#line 919 "src/libre/parser.act" (ZIflags) = *flags; @@ -4072,7 +4072,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -4086,7 +4086,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 922 "src/libre/parser.act" +#line 923 "src/libre/parser.act" *flags = (ZIflags); @@ -4095,7 +4095,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { @@ -4119,7 +4119,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 935 "src/libre/parser.act" +#line 936 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { @@ -4178,7 +4178,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 704 "src/libre/parser.act" +#line 708 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -4190,7 +4190,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -4223,7 +4223,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -4244,7 +4244,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -4287,7 +4287,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -4299,7 +4299,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; @@ -4319,7 +4319,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1052 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" static int @@ -4339,7 +4339,6 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { @@ -4476,6 +4475,6 @@ ZL0:; return NULL; } -#line 4480 "src/libre/dialect/pcre/parser.c" +#line 4479 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index a0543aebe..c0cfbabe3 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 281 "src/libre/parser.act" +#line 292 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1208 "src/libre/parser.act" +#line 1209 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index 078b8f67d..d380c5b7d 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 22 "src/libre/parser.act" +#line 275 "src/libre/parser.act" #include @@ -311,7 +311,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: INVERT */ { -#line 302 "src/libre/parser.act" +#line 303 "src/libre/parser.act" ZI203 = '^'; @@ -337,7 +337,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZIc = '-'; ZI114 = lex_state->lx.start; @@ -353,7 +353,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -365,7 +365,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -401,7 +401,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -415,7 +415,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -441,7 +441,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 753 "src/libre/parser.act" +#line 757 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -484,7 +484,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; @@ -499,7 +499,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 690 "src/libre/parser.act" +#line 694 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; @@ -522,7 +522,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -560,7 +560,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1040 "src/libre/parser.act" +#line 1041 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -579,7 +579,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -606,7 +606,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -670,7 +670,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 318 "src/libre/parser.act" +#line 319 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -688,7 +688,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { @@ -713,7 +713,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 350 "src/libre/parser.act" +#line 351 "src/libre/parser.act" ZI154 = ']'; ZIclose__start = lex_state->lx.start; @@ -729,7 +729,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 767 "src/libre/parser.act" +#line 768 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); @@ -748,7 +748,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: INVERT */ { -#line 302 "src/libre/parser.act" +#line 303 "src/libre/parser.act" ZI158 = '^'; @@ -760,7 +760,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { @@ -789,7 +789,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 350 "src/libre/parser.act" +#line 351 "src/libre/parser.act" ZI163 = ']'; ZIclose__start = lex_state->lx.start; @@ -809,7 +809,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 767 "src/libre/parser.act" +#line 768 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); @@ -823,7 +823,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 725 "src/libre/parser.act" +#line 729 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -839,7 +839,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 162 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 959 "src/libre/parser.act" +#line 960 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { @@ -862,7 +862,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 725 "src/libre/parser.act" +#line 729 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -874,7 +874,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -917,7 +917,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -954,7 +954,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 867 "src/libre/parser.act" +#line 868 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -975,7 +975,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 711 "src/libre/parser.act" +#line 715 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -987,7 +987,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1019,7 +1019,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZIc = '-'; ZI117 = lex_state->lx.start; @@ -1035,7 +1035,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1047,7 +1047,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -1058,7 +1058,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 966 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1105,7 +1105,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 966 "src/libre/parser.act" +#line 995 "src/libre/parser.act" struct ast_expr *any; @@ -1167,7 +1167,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI205)); if ((ZInode) == NULL) { @@ -1192,7 +1192,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; (ZIa).u.literal.c = (unsigned char) (*ZI205); @@ -1202,7 +1202,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 308 "src/libre/parser.act" +#line 309 "src/libre/parser.act" ZI136 = '-'; ZI137 = lex_state->lx.start; @@ -1220,7 +1220,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1243,7 +1243,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 839 "src/libre/parser.act" +#line 840 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char) (ZIcz); @@ -1253,7 +1253,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 772 "src/libre/parser.act" +#line 773 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI206)); mark(&act_state->rangeend, &(ZIend)); @@ -1263,7 +1263,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1004 "src/libre/parser.act" +#line 1007 "src/libre/parser.act" unsigned char lower, upper; @@ -1321,7 +1321,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI176 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1335,7 +1335,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); @@ -1345,7 +1345,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1376,7 +1376,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 627 "src/libre/parser.act" +#line 636 "src/libre/parser.act" unsigned long u; char *e; @@ -1408,7 +1408,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 378 "src/libre/parser.act" +#line 379 "src/libre/parser.act" ZI179 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1426,7 +1426,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 777 "src/libre/parser.act" +#line 778 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); @@ -1436,7 +1436,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 824 "src/libre/parser.act" +#line 825 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1486,7 +1486,7 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1046 "src/libre/parser.act" +#line 1047 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; @@ -1505,7 +1505,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /* UNREACHED */ + /*UNREACHED*/ default: break; } @@ -1517,7 +1517,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 711 "src/libre/parser.act" +#line 715 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -1549,7 +1549,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 370 "src/libre/parser.act" +#line 371 "src/libre/parser.act" ZI209 = lex_state->lx.start; ZI210 = lex_state->lx.end; @@ -1565,7 +1565,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 627 "src/libre/parser.act" +#line 636 "src/libre/parser.act" unsigned long u; char *e; @@ -1605,7 +1605,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 816 "src/libre/parser.act" +#line 817 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); @@ -1619,7 +1619,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 812 "src/libre/parser.act" +#line 813 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); @@ -1633,7 +1633,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 808 "src/libre/parser.act" +#line 809 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -1646,7 +1646,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 820 "src/libre/parser.act" +#line 821 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -1663,7 +1663,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 697 "src/libre/parser.act" +#line 701 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; @@ -1675,7 +1675,7 @@ ZL1:; /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 820 "src/libre/parser.act" +#line 821 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -1704,7 +1704,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -1714,7 +1714,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -1734,7 +1734,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* BEGINNING OF EXTRACT: CHAR */ { -#line 575 "src/libre/parser.act" +#line 579 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1753,7 +1753,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 874 "src/libre/parser.act" +#line 875 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -1774,7 +1774,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 782 "src/libre/parser.act" +#line 784 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -1784,7 +1784,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -1796,7 +1796,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 808 "src/libre/parser.act" +#line 809 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -1805,7 +1805,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 897 "src/libre/parser.act" +#line 898 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1832,7 +1832,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 881 "src/libre/parser.act" +#line 882 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -1846,7 +1846,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 911 "src/libre/parser.act" +#line 912 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { @@ -1884,7 +1884,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 704 "src/libre/parser.act" +#line 708 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -1896,7 +1896,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -1932,7 +1932,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 647 "src/libre/parser.act" +#line 648 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1956,7 +1956,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1033 "src/libre/parser.act" +#line 1034 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { @@ -1986,7 +1986,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 860 "src/libre/parser.act" +#line 861 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2007,7 +2007,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 853 "src/libre/parser.act" +#line 854 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2032,7 +2032,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1052 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" static int @@ -2052,7 +2052,6 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { @@ -2189,6 +2188,6 @@ ZL0:; return NULL; } -#line 2193 "src/libre/dialect/sql/parser.c" +#line 2192 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index e9850510a..c5e885439 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 281 "src/libre/parser.act" +#line 292 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1208 "src/libre/parser.act" +#line 1209 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/parser.act b/src/libre/parser.act index d8d0e2ac4..984036cb8 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -1068,7 +1068,6 @@ struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, struct re_err *err, struct re_pos *end) { diff --git a/src/libre/re.c b/src/libre/re.c index 3e9ee5753..15af848b5 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -91,7 +91,6 @@ re_flags(const char *s, enum re_flags *f) struct ast * re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, int *unsatisfiable) { const struct dialect *m; @@ -109,7 +108,7 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = m->parse(getc, opaque, opt, flags, m->overlap, err, &end); + ast = m->parse(getc, opaque, flags, m->overlap, err, &end); if (ast == NULL) { return NULL; @@ -177,7 +176,6 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, struct fsm * re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, const struct fsm_alloc *alloc, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err) { struct ast *ast; @@ -195,7 +193,7 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = re_parse(dialect, getc, opaque, opt, flags, err, &unsatisfiable); + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); if (ast == NULL) { return NULL; } @@ -213,7 +211,7 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, ast->expr = ast_expr_tombstone; } - new = ast_compile(ast, flags, alloc, opt, err); + new = ast_compile(ast, flags, alloc, err); ast_free(ast); @@ -243,7 +241,6 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, */ int re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, enum re_literal_category *category, char **s, size_t *n) { @@ -264,7 +261,7 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = re_parse(dialect, getc, opaque, opt, flags, err, &unsatisfiable); + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); if (ast == NULL) { return -1; } diff --git a/src/libre/re_strings.c b/src/libre/re_strings.c index 34a9fa0fe..dc554c801 100644 --- a/src/libre/re_strings.c +++ b/src/libre/re_strings.c @@ -19,7 +19,7 @@ #include "ac.h" struct fsm * -re_strings(const struct fsm_alloc *alloc, const struct fsm_options *opt, +re_strings(const struct fsm_alloc *alloc, const char *a[], size_t n, enum re_strings_flags flags) { @@ -38,7 +38,7 @@ re_strings(const struct fsm_alloc *alloc, const struct fsm_options *opt, } } - fsm = re_strings_build(g, alloc, opt, flags); + fsm = re_strings_build(g, alloc, flags); re_strings_free(g); return fsm; @@ -84,7 +84,6 @@ re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endi struct fsm * re_strings_build(struct re_strings *g, const struct fsm_alloc *alloc, - const struct fsm_options *opt, enum re_strings_flags flags) { struct fsm *fsm; @@ -105,8 +104,6 @@ re_strings_build(struct re_strings *g, goto error; } - fsm_setoptions(fsm, opt); - have_end = 0; if ((flags & RE_STRINGS_AC_AUTOMATON) == 0 && (flags & RE_STRINGS_ANCHOR_RIGHT) == 0) { diff --git a/src/lx/ast.c b/src/lx/ast.c index 97977b803..58646ca68 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -123,8 +123,6 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, return NULL; } - fsm_setoptions(m->fsm, fsm->opt); - m->token = token; m->to = to; m->conflict = NULL; diff --git a/src/lx/main.c b/src/lx/main.c index 572014171..fb7a176f4 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -308,15 +308,9 @@ zone_equal(const struct ast_zone *a, const struct ast_zone *b) return -1; } - { - /* opt.carryopaque = carryopaque; */ - - if (!fsm_determinise(q)) { - fsm_free(q); - return -1; - } - - /* opt.carryopaque = NULL; */ + if (!fsm_determinise(q)) { + fsm_free(q); + return -1; } { @@ -383,8 +377,6 @@ zone_minimise(void *arg) return "fsm_new"; } - fsm_setoptions(z->fsm, &opt); - if (!fsm_addstate(z->fsm, &start)) { pthread_mutex_lock(&zmtx); zerror = errno; @@ -647,7 +639,7 @@ main(int argc, char *argv[]) fprintf(stderr, "-- parsing:"); } - ast = lx_parse(stdin, alloc, &opt); + ast = lx_parse(stdin, alloc); if (ast == NULL) { return EXIT_FAILURE; } @@ -693,12 +685,10 @@ main(int argc, char *argv[]) zn = 0; } - /* opt.carryopaque = carryopaque; */ cur_zone = ast->zl; if (run_threads(concurrency, zone_determinise)) { return EXIT_FAILURE; } - /* opt.carryopaque = NULL; */ if (print_progress) { fprintf(stderr, "\n"); @@ -955,7 +945,7 @@ main(int argc, char *argv[]) fprintf(stderr, "-- output:"); } - lx_print(stdout, ast, lang); + lx_print(stdout, ast, &opt, lang); if (print_progress) { fprintf(stderr, "\n"); diff --git a/src/lx/parser.act b/src/lx/parser.act index 04001783f..0db0c3bb5 100644 --- a/src/lx/parser.act +++ b/src/lx/parser.act @@ -38,7 +38,6 @@ #include #include #include - #include #include @@ -68,7 +67,6 @@ enum lx_token lex_tok; enum lx_token lex_tok_save; unsigned int zn; - const struct fsm_options *opt; const struct fsm_alloc *alloc; }; @@ -136,8 +134,7 @@ typedef struct act_state * act_state; typedef struct ast * ast; - struct ast *lx_parse(FILE *f, - const struct fsm_alloc *alloc, const struct fsm_options *opt); + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc); @}; @@ -317,8 +314,6 @@ @!; } - fsm_setoptions(@r, act_state->opt); - if (!fsm_addstate(@r, &start)) { perror("fsm_addstate"); @!; @@ -369,7 +364,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_LITERAL, act_agetc, @a, act_state->alloc, act_state->opt, 0, &err); + @r = re_comp(RE_LITERAL, act_agetc, @a, act_state->alloc, 0, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -384,7 +379,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->alloc, act_state->opt, @f, &err); + @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->alloc, @f, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -403,8 +398,6 @@ @!; } - fsm_setoptions(@r, act_state->opt); - if (!fsm_addstate(@r, &start)) { perror("fsm_addstate"); fsm_free(@r); @@ -824,7 +817,7 @@ %trailer% @{ - struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -856,7 +849,6 @@ /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - act_state->opt = opt; act_state->alloc = alloc; if (print_progress) { diff --git a/src/lx/parser.c b/src/lx/parser.c index b492f4cd5..f6f759693 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 129 "src/lx/parser.act" +#line 127 "src/lx/parser.act" #include @@ -24,7 +24,6 @@ #include #include #include - #include #include @@ -54,7 +53,6 @@ enum lx_token lex_tok; enum lx_token lex_tok_save; unsigned int zn; - const struct fsm_options *opt; const struct fsm_alloc *alloc; }; @@ -112,7 +110,7 @@ return a->len--, (unsigned char) *a->p++; } -#line 116 "src/lx/parser.c" +#line 114 "src/lx/parser.c" #ifndef ERROR_TERMINAL @@ -184,11 +182,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 816 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 192 "src/lx/parser.c" +#line 190 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -206,17 +204,17 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: IDENT */ { -#line 231 "src/lx/parser.act" +#line 228 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); -#line 214 "src/lx/parser.c" +#line 212 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-var */ { -#line 283 "src/lx/parser.act" +#line 280 "src/lx/parser.act" struct ast_zone *z; @@ -243,7 +241,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) goto ZL1; } -#line 247 "src/lx/parser.c" +#line 245 "src/lx/parser.c" } /* END OF ACTION: deref-var */ } @@ -254,18 +252,18 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: TOKEN */ { -#line 227 "src/lx/parser.act" +#line 224 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ -#line 263 "src/lx/parser.c" +#line 261 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 311 "src/lx/parser.act" +#line 308 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -279,8 +277,6 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) goto ZL1; } - fsm_setoptions((ZIr), act_state->opt); - if (!fsm_addstate((ZIr), &start)) { perror("fsm_addstate"); goto ZL1; @@ -324,7 +320,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) fsm_setstart((ZIr), start); -#line 328 "src/lx/parser.c" +#line 324 "src/lx/parser.c" } /* END OF ACTION: deref-token */ } @@ -378,7 +374,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-reverse */ { -#line 684 "src/lx/parser.act" +#line 677 "src/lx/parser.act" assert((ZI210) != NULL); @@ -387,7 +383,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 391 "src/lx/parser.c" +#line 387 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -402,7 +398,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI231) != NULL); @@ -422,7 +418,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 426 "src/lx/parser.c" +#line 422 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI232, &ZI233); @@ -453,7 +449,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complete */ { -#line 675 "src/lx/parser.act" +#line 668 "src/lx/parser.act" assert((ZI210) != NULL); @@ -462,7 +458,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 466 "src/lx/parser.c" +#line 462 "src/lx/parser.c" } /* END OF ACTION: op-complete */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -477,7 +473,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI239) != NULL); @@ -497,7 +493,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 501 "src/lx/parser.c" +#line 497 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI240, &ZI241); @@ -513,11 +509,11 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: IDENT */ { -#line 231 "src/lx/parser.act" +#line 228 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); -#line 521 "src/lx/parser.c" +#line 517 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; @@ -567,7 +563,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI248) != NULL); @@ -587,7 +583,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 591 "src/lx/parser.c" +#line 587 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI249, &ZI250); @@ -618,7 +614,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complement */ { -#line 666 "src/lx/parser.act" +#line 659 "src/lx/parser.act" assert((ZI210) != NULL); @@ -627,7 +623,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 631 "src/lx/parser.c" +#line 627 "src/lx/parser.c" } /* END OF ACTION: op-complement */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -642,7 +638,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI223) != NULL); @@ -662,7 +658,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 666 "src/lx/parser.c" +#line 662 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI224, &ZI225); @@ -689,18 +685,18 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: TOKEN */ { -#line 227 "src/lx/parser.act" +#line 224 "src/lx/parser.act" /* TODO: submatch addressing */ ZI253 = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ -#line 698 "src/lx/parser.c" +#line 694 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 311 "src/lx/parser.act" +#line 308 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -714,8 +710,6 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } - fsm_setoptions((ZI254), act_state->opt); - if (!fsm_addstate((ZI254), &start)) { perror("fsm_addstate"); goto ZL1; @@ -759,7 +753,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z fsm_setstart((ZI254), start); -#line 763 "src/lx/parser.c" +#line 757 "src/lx/parser.c" } /* END OF ACTION: deref-token */ p_197 (lex_state, act_state, ZI254, &ZI210); @@ -775,7 +769,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI259) != NULL); @@ -795,7 +789,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 799 "src/lx/parser.c" +#line 793 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI260, &ZI261); @@ -835,7 +829,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI268) != NULL); @@ -855,7 +849,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 859 "src/lx/parser.c" +#line 853 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI269, &ZI270); @@ -875,11 +869,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-thing */ { -#line 819 "src/lx/parser.act" +#line 812 "src/lx/parser.act" err_expected(lex_state, "mapping, binding or zone"); -#line 883 "src/lx/parser.c" +#line 877 "src/lx/parser.c" } /* END OF ACTION: err-expected-thing */ } @@ -905,11 +899,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-open */ { -#line 807 "src/lx/parser.act" +#line 800 "src/lx/parser.act" err_expected(lex_state, "'{'"); -#line 913 "src/lx/parser.c" +#line 907 "src/lx/parser.c" } /* END OF ACTION: err-expected-open */ } @@ -935,11 +929,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-close */ { -#line 811 "src/lx/parser.act" +#line 804 "src/lx/parser.act" err_expected(lex_state, "'}'"); -#line 943 "src/lx/parser.c" +#line 937 "src/lx/parser.c" } /* END OF ACTION: err-expected-close */ } @@ -961,14 +955,14 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: CHAR */ { -#line 222 "src/lx/parser.act" +#line 219 "src/lx/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 972 "src/lx/parser.c" +#line 966 "src/lx/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -978,7 +972,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: ESC */ { -#line 152 "src/lx/parser.act" +#line 149 "src/lx/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -996,7 +990,7 @@ ZL2_pattern_C_Cbody:; default: ZIc = '\0'; break; /* TODO: handle error */ } -#line 1000 "src/lx/parser.c" +#line 994 "src/lx/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1006,7 +1000,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: HEX */ { -#line 215 "src/lx/parser.act" +#line 212 "src/lx/parser.act" unsigned long u; char *e; @@ -1033,7 +1027,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1037 "src/lx/parser.c" +#line 1031 "src/lx/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1043,7 +1037,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: OCT */ { -#line 188 "src/lx/parser.act" +#line 185 "src/lx/parser.act" unsigned long u; char *e; @@ -1070,7 +1064,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1074 "src/lx/parser.c" +#line 1068 "src/lx/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1083,12 +1077,12 @@ ZL2_pattern_C_Cbody:; /* END OF INLINE: 84 */ /* BEGINNING OF ACTION: pattern-char */ { -#line 252 "src/lx/parser.act" +#line 249 "src/lx/parser.act" /* TODO */ *lex_state->p++ = (ZIc); -#line 1092 "src/lx/parser.c" +#line 1086 "src/lx/parser.c" } /* END OF ACTION: pattern-char */ /* BEGINNING OF INLINE: pattern::body */ @@ -1217,7 +1211,7 @@ ZL2_180:; } /* BEGINNING OF ACTION: op-alt */ { -#line 732 "src/lx/parser.act" +#line 725 "src/lx/parser.act" assert((ZI177) != NULL); assert((ZIb) != NULL); @@ -1228,7 +1222,7 @@ ZL2_180:; goto ZL1; } -#line 1232 "src/lx/parser.c" +#line 1226 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: 180 */ @@ -1297,7 +1291,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-reverse */ { -#line 684 "src/lx/parser.act" +#line 677 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1306,7 +1300,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1310 "src/lx/parser.c" +#line 1304 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ } @@ -1321,7 +1315,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complete */ { -#line 675 "src/lx/parser.act" +#line 668 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1330,7 +1324,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1334 "src/lx/parser.c" +#line 1328 "src/lx/parser.c" } /* END OF ACTION: op-complete */ } @@ -1345,7 +1339,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complement */ { -#line 666 "src/lx/parser.act" +#line 659 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1354,7 +1348,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1358 "src/lx/parser.c" +#line 1352 "src/lx/parser.c" } /* END OF ACTION: op-complement */ } @@ -1404,7 +1398,7 @@ ZL2_186:; } /* BEGINNING OF ACTION: op-intersect */ { -#line 721 "src/lx/parser.act" +#line 714 "src/lx/parser.act" assert((ZI183) != NULL); assert((ZIb) != NULL); @@ -1415,7 +1409,7 @@ ZL2_186:; goto ZL1; } -#line 1419 "src/lx/parser.c" +#line 1413 "src/lx/parser.c" } /* END OF ACTION: op-intersect */ /* BEGINNING OF INLINE: 186 */ @@ -1461,7 +1455,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ } /* BEGINNING OF ACTION: op-alt */ { -#line 732 "src/lx/parser.act" +#line 725 "src/lx/parser.act" assert((ZIold_Hexit) != NULL); assert((ZInew_Hexit) != NULL); @@ -1472,7 +1466,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ goto ZL1; } -#line 1476 "src/lx/parser.c" +#line 1470 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-to-mappings::list-of-zone-to-mappings-x */ @@ -1560,7 +1554,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-cross */ { -#line 582 "src/lx/parser.act" +#line 575 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1605,7 +1599,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1609 "src/lx/parser.c" +#line 1603 "src/lx/parser.c" } /* END OF ACTION: op-cross */ /* BEGINNING OF INLINE: 197 */ @@ -1619,7 +1613,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-qmark */ { -#line 627 "src/lx/parser.act" +#line 620 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1664,7 +1658,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1668 "src/lx/parser.c" +#line 1662 "src/lx/parser.c" } /* END OF ACTION: op-qmark */ /* BEGINNING OF INLINE: 197 */ @@ -1678,7 +1672,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-star */ { -#line 532 "src/lx/parser.act" +#line 525 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1728,7 +1722,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1732 "src/lx/parser.c" +#line 1726 "src/lx/parser.c" } /* END OF ACTION: op-star */ /* BEGINNING OF INLINE: 197 */ @@ -1801,7 +1795,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) } /* BEGINNING OF ACTION: op-subtract */ { -#line 710 "src/lx/parser.act" +#line 703 "src/lx/parser.act" assert((*ZI202) != NULL); assert((ZIb) != NULL); @@ -1812,7 +1806,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) goto ZL1; } -#line 1816 "src/lx/parser.c" +#line 1810 "src/lx/parser.c" } /* END OF ACTION: op-subtract */ } @@ -1876,7 +1870,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) } /* BEGINNING OF ACTION: op-concat */ { -#line 693 "src/lx/parser.act" +#line 686 "src/lx/parser.act" assert((*ZI206) != NULL); assert((ZIb) != NULL); @@ -1887,7 +1881,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) goto ZL1; } -#line 1891 "src/lx/parser.c" +#line 1885 "src/lx/parser.c" } /* END OF ACTION: op-concat */ } @@ -1926,13 +1920,13 @@ p_212(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI210, fsm *ZOq) } /* BEGINNING OF ACTION: op-product */ { -#line 705 "src/lx/parser.act" +#line 698 "src/lx/parser.act" fprintf(stderr, "unimplemented\n"); (ZIq) = NULL; goto ZL1; -#line 1936 "src/lx/parser.c" +#line 1930 "src/lx/parser.c" } /* END OF ACTION: op-product */ } @@ -1966,7 +1960,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* BEGINNING OF EXTRACT: RE */ { -#line 239 "src/lx/parser.act" +#line 236 "src/lx/parser.act" assert(lex_state->buf.a[0] == '/'); @@ -1980,13 +1974,13 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ZIf |= RE_ANCHORED; /* regexps in lx are implicitly anchored */ ZIf |= RE_SINGLE; /* we can't assume we're lexing text */ -#line 1984 "src/lx/parser.c" +#line 1978 "src/lx/parser.c" } /* END OF EXTRACT: RE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 266 "src/lx/parser.act" +#line 263 "src/lx/parser.act" size_t len; @@ -2011,19 +2005,19 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2015 "src/lx/parser.c" +#line 2009 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-regex */ { -#line 384 "src/lx/parser.act" +#line 379 "src/lx/parser.act" struct re_err err; assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->alloc, act_state->opt, (ZIf), &err); + (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->alloc, (ZIf), &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -2031,16 +2025,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2035 "src/lx/parser.c" +#line 2029 "src/lx/parser.c" } /* END OF ACTION: compile-regex */ /* BEGINNING OF ACTION: free-arr */ { -#line 773 "src/lx/parser.act" +#line 766 "src/lx/parser.act" free((ZIa)); -#line 2044 "src/lx/parser.c" +#line 2038 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2052,7 +2046,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 266 "src/lx/parser.act" +#line 263 "src/lx/parser.act" size_t len; @@ -2077,19 +2071,19 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2081 "src/lx/parser.c" +#line 2075 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-literal */ { -#line 369 "src/lx/parser.act" +#line 364 "src/lx/parser.act" struct re_err err; assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_LITERAL, act_agetc, (ZIa), act_state->alloc, act_state->opt, 0, &err); + (ZIr) = re_comp(RE_LITERAL, act_agetc, (ZIa), act_state->alloc, 0, &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -2097,16 +2091,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2101 "src/lx/parser.c" +#line 2095 "src/lx/parser.c" } /* END OF ACTION: compile-literal */ /* BEGINNING OF ACTION: free-arr */ { -#line 773 "src/lx/parser.act" +#line 766 "src/lx/parser.act" free((ZIa)); -#line 2110 "src/lx/parser.c" +#line 2104 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2170,7 +2164,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2190,12 +2184,12 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } } -#line 2194 "src/lx/parser.c" +#line 2188 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 460 "src/lx/parser.act" +#line 453 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2228,7 +2222,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho nmappings++; } -#line 2232 "src/lx/parser.c" +#line 2226 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-from-mappings::list-of-zone-from-mappings-x */ @@ -2262,16 +2256,16 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* BEGINNING OF ACTION: no-zone */ { -#line 522 "src/lx/parser.act" +#line 515 "src/lx/parser.act" (ZIparent) = NULL; -#line 2270 "src/lx/parser.c" +#line 2264 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: make-ast */ { -#line 431 "src/lx/parser.act" +#line 424 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2279,12 +2273,12 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2283 "src/lx/parser.c" +#line 2277 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: make-zone */ { -#line 439 "src/lx/parser.act" +#line 432 "src/lx/parser.act" assert((ZIa) != NULL); @@ -2302,28 +2296,28 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2306 "src/lx/parser.c" +#line 2300 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: no-exit */ { -#line 518 "src/lx/parser.act" +#line 511 "src/lx/parser.act" (ZIexit) = NULL; -#line 2315 "src/lx/parser.c" +#line 2309 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: set-globalzone */ { -#line 507 "src/lx/parser.act" +#line 500 "src/lx/parser.act" assert((ZIa) != NULL); assert((ZIz) != NULL); (ZIa)->global = (ZIz); -#line 2327 "src/lx/parser.c" +#line 2321 "src/lx/parser.c" } /* END OF ACTION: set-globalzone */ p_list_Hof_Hthings (lex_state, act_state, ZIa, ZIz, ZIexit); @@ -2347,11 +2341,11 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 815 "src/lx/parser.act" +#line 808 "src/lx/parser.act" err_expected(lex_state, "EOF"); -#line 2355 "src/lx/parser.c" +#line 2349 "src/lx/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2364,7 +2358,7 @@ ZL1:; { /* BEGINNING OF ACTION: make-ast */ { -#line 431 "src/lx/parser.act" +#line 424 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2372,17 +2366,17 @@ ZL1:; goto ZL4; } -#line 2376 "src/lx/parser.c" +#line 2370 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: err-syntax */ { -#line 783 "src/lx/parser.act" +#line 776 "src/lx/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 2386 "src/lx/parser.c" +#line 2380 "src/lx/parser.c" } /* END OF ACTION: err-syntax */ } @@ -2439,7 +2433,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ } /* BEGINNING OF ACTION: add-mapping */ { -#line 460 "src/lx/parser.act" +#line 453 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2472,12 +2466,12 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ nmappings++; } -#line 2476 "src/lx/parser.c" +#line 2470 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF ACTION: clone */ { -#line 763 "src/lx/parser.act" +#line 756 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2487,7 +2481,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ goto ZL1; } -#line 2491 "src/lx/parser.c" +#line 2485 "src/lx/parser.c" } /* END OF ACTION: clone */ } @@ -2523,11 +2517,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: err-expected-map */ { -#line 791 "src/lx/parser.act" +#line 784 "src/lx/parser.act" err_expected(lex_state, "'->'"); -#line 2531 "src/lx/parser.c" +#line 2525 "src/lx/parser.c" } /* END OF ACTION: err-expected-map */ } @@ -2538,12 +2532,12 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) case (TOK_TOKEN): /* BEGINNING OF EXTRACT: TOKEN */ { -#line 227 "src/lx/parser.act" +#line 224 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ -#line 2547 "src/lx/parser.c" +#line 2541 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ break; @@ -2557,11 +2551,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: no-token */ { -#line 514 "src/lx/parser.act" +#line 507 "src/lx/parser.act" (ZIt) = NULL; -#line 2565 "src/lx/parser.c" +#line 2559 "src/lx/parser.c" } /* END OF ACTION: no-token */ } @@ -2650,11 +2644,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-semi */ { -#line 799 "src/lx/parser.act" +#line 792 "src/lx/parser.act" err_expected(lex_state, "';'"); -#line 2658 "src/lx/parser.c" +#line 2652 "src/lx/parser.c" } /* END OF ACTION: err-expected-semi */ } @@ -2701,16 +2695,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 522 "src/lx/parser.act" +#line 515 "src/lx/parser.act" (ZIto) = NULL; -#line 2709 "src/lx/parser.c" +#line 2703 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 460 "src/lx/parser.act" +#line 453 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2743,7 +2737,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2747 "src/lx/parser.c" +#line 2741 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -2755,7 +2749,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: make-zone */ { -#line 439 "src/lx/parser.act" +#line 432 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -2773,12 +2767,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 2777 "src/lx/parser.c" +#line 2771 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 460 "src/lx/parser.act" +#line 453 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2811,7 +2805,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2815 "src/lx/parser.c" +#line 2809 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ p_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Hof_Hzone_Hfrom_Hmappings_Hx (lex_state, act_state, *ZIa, *ZIz, ZIchild, *ZIexit); @@ -2835,11 +2829,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-to */ { -#line 803 "src/lx/parser.act" +#line 796 "src/lx/parser.act" err_expected(lex_state, "'..'"); -#line 2843 "src/lx/parser.c" +#line 2837 "src/lx/parser.c" } /* END OF ACTION: err-expected-to */ } @@ -2864,25 +2858,25 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 522 "src/lx/parser.act" +#line 515 "src/lx/parser.act" (ZIx) = NULL; -#line 2872 "src/lx/parser.c" +#line 2866 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: no-token */ { -#line 514 "src/lx/parser.act" +#line 507 "src/lx/parser.act" (ZIy) = NULL; -#line 2881 "src/lx/parser.c" +#line 2875 "src/lx/parser.c" } /* END OF ACTION: no-token */ /* BEGINNING OF ACTION: regex-any */ { -#line 400 "src/lx/parser.act" +#line 395 "src/lx/parser.act" fsm_state_t start, end; @@ -2892,8 +2886,6 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL5; } - fsm_setoptions((ZIw), act_state->opt); - if (!fsm_addstate((ZIw), &start)) { perror("fsm_addstate"); fsm_free((ZIw)); @@ -2915,12 +2907,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit fsm_setstart((ZIw), start); fsm_setend((ZIw), end, 1); -#line 2919 "src/lx/parser.c" +#line 2911 "src/lx/parser.c" } /* END OF ACTION: regex-any */ /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZIw) != NULL); @@ -2940,12 +2932,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 2944 "src/lx/parser.c" +#line 2936 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 460 "src/lx/parser.act" +#line 453 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2978,7 +2970,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2982 "src/lx/parser.c" +#line 2974 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -3005,11 +2997,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 816 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3013 "src/lx/parser.c" +#line 3005 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3025,16 +3017,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: no-exit */ { -#line 518 "src/lx/parser.act" +#line 511 "src/lx/parser.act" (ZIr2) = NULL; -#line 3033 "src/lx/parser.c" +#line 3025 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: make-zone */ { -#line 439 "src/lx/parser.act" +#line 432 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -3052,12 +3044,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3056 "src/lx/parser.c" +#line 3048 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 460 "src/lx/parser.act" +#line 453 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -3090,7 +3082,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 3094 "src/lx/parser.c" +#line 3086 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: 131 */ @@ -3109,11 +3101,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 816 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3117 "src/lx/parser.c" +#line 3109 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3157,11 +3149,11 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-bind */ { -#line 795 "src/lx/parser.act" +#line 788 "src/lx/parser.act" err_expected(lex_state, "'='"); -#line 3165 "src/lx/parser.c" +#line 3157 "src/lx/parser.c" } /* END OF ACTION: err-expected-bind */ } @@ -3176,7 +3168,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: add-binding */ { -#line 492 "src/lx/parser.act" +#line 485 "src/lx/parser.act" struct var *v; @@ -3193,7 +3185,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3197 "src/lx/parser.c" +#line 3189 "src/lx/parser.c" } /* END OF ACTION: add-binding */ } @@ -3219,7 +3211,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: deref-var */ { -#line 283 "src/lx/parser.act" +#line 280 "src/lx/parser.act" struct ast_zone *z; @@ -3246,7 +3238,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3250 "src/lx/parser.c" +#line 3242 "src/lx/parser.c" } /* END OF ACTION: deref-var */ p_197 (lex_state, act_state, ZI273, &ZI210); @@ -3262,7 +3254,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: subtract-exit */ { -#line 743 "src/lx/parser.act" +#line 736 "src/lx/parser.act" assert((ZI278) != NULL); @@ -3282,7 +3274,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 3286 "src/lx/parser.c" +#line 3278 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, ZIa, ZIz, ZIexit, &ZI279, &ZI280); @@ -3305,10 +3297,10 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 888 "src/lx/parser.act" +#line 880 "src/lx/parser.act" - struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc, const struct fsm_options *opt) { + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -3340,7 +3332,6 @@ ZL1:; /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - act_state->opt = opt; act_state->alloc = alloc; if (print_progress) { @@ -3369,6 +3360,6 @@ ZL1:; return ast; } -#line 3373 "src/lx/parser.c" +#line 3364 "src/lx/parser.c" /* END OF FILE */ diff --git a/src/lx/parser.h b/src/lx/parser.h index 53085c140..fdaff9879 100644 --- a/src/lx/parser.h +++ b/src/lx/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 142 "src/lx/parser.act" +#line 139 "src/lx/parser.act" #include @@ -20,19 +20,18 @@ typedef struct act_state * act_state; typedef struct ast * ast; - struct ast *lx_parse(FILE *f, - const struct fsm_alloc *alloc, const struct fsm_options *opt); + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc); -#line 27 "src/lx/parser.h" +#line 26 "src/lx/parser.h" /* BEGINNING OF FUNCTION DECLARATIONS */ extern void p_lx(lex_state, act_state, ast *); /* BEGINNING OF TRAILER */ -#line 890 "src/lx/parser.act" +#line 882 "src/lx/parser.act" -#line 37 "src/lx/parser.h" +#line 36 "src/lx/parser.h" /* END OF FILE */ diff --git a/src/lx/print.c b/src/lx/print.c index 6c65a52c0..de71a54ef 100644 --- a/src/lx/print.c +++ b/src/lx/print.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -14,7 +15,8 @@ #include "print.h" int -lx_print(FILE *f, const struct ast *ast, enum lx_print_lang lang) +lx_print(FILE *f, const struct ast *ast, const struct fsm_options *opt, + enum lx_print_lang lang) { lx_print_f *print; @@ -37,7 +39,7 @@ lx_print(FILE *f, const struct ast *ast, enum lx_print_lang lang) } // TODO: return int - print(f, ast); + print(f, ast, opt); return 0; } diff --git a/src/lx/print.h b/src/lx/print.h index 7452e86d7..771028e14 100644 --- a/src/lx/print.h +++ b/src/lx/print.h @@ -10,6 +10,7 @@ #include struct ast; +struct fsm_options; /* TODO: combine all variables here into an lx_outoptions struct */ struct prefix { @@ -55,9 +56,8 @@ extern enum api_tokbuf api_tokbuf; extern enum api_getc api_getc; extern enum api_exclude api_exclude; -extern struct fsm_options opt; - -typedef void (lx_print_f)(FILE *f, const struct ast *ast); +typedef void (lx_print_f)(FILE *f, + const struct ast *ast, const struct fsm_options *opt); lx_print_f lx_print_c; lx_print_f lx_print_dot; @@ -66,7 +66,9 @@ lx_print_f lx_print_h; lx_print_f lx_print_zdot; int -lx_print(FILE *f, const struct ast *ast, enum lx_print_lang lang); +lx_print(FILE *f, const struct ast *ast, + const struct fsm_options *opt, + enum lx_print_lang lang); #endif diff --git a/src/lx/print/c.c b/src/lx/print/c.c index bc5e5e91a..58f3279ce 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -27,7 +27,9 @@ /* XXX: abstraction */ int -fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, +fsm_print_cfrag(FILE *f, const struct ir *ir, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const char *cp); static int @@ -138,20 +140,19 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, static int accept_c(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { const struct ast *ast; const struct ast_mapping *m; - ast = opt->hooks.hook_opaque; - assert(f != NULL); assert(opt != NULL); - assert(opt->hooks.hook_opaque != NULL); assert(ids != NULL); assert(count > 0); assert(lang_opaque == NULL); + assert(hook_opaque != NULL); + ast = hook_opaque; m = ast_getendmappingbyendid(ids[0]); /* XXX: don't need this if complete */ @@ -173,14 +174,15 @@ accept_c(FILE *f, const struct fsm_options *opt, static int reject_c(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); - assert(opt->hooks.hook_opaque != NULL); assert(lang_opaque == NULL); + assert(hook_opaque != NULL); - (void) opt->hooks.hook_opaque; + (void) lang_opaque; + (void) hook_opaque; /* XXX: don't need this if complete */ switch (opt->io) { @@ -338,7 +340,7 @@ print_lgetc(FILE *f) } static void -print_io(FILE *f) +print_io(FILE *f, const struct fsm_options *opt) { if (print_progress) { fprintf(stderr, " io"); @@ -356,7 +358,7 @@ print_io(FILE *f) fprintf(f, "\tassert(lx != NULL);\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tassert(lx->lgetc != NULL);\n"); fprintf(f, "\n"); @@ -404,7 +406,7 @@ print_io(FILE *f) fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); fprintf(f, "\t\tlx->end.col = 1;\n"); - if (opt.io == FSM_IO_STR) { /* ignore terminating '\0' */ + if (opt->io == FSM_IO_STR) { /* ignore terminating '\0' */ fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); fprintf(f, "\t\tlx->end.byte--;\n"); fprintf(f, "\t\tlx->end.col--;\n"); @@ -426,7 +428,7 @@ print_io(FILE *f) fprintf(f, "{\n"); fprintf(f, "\tassert(lx != NULL);\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tassert(lx->c == EOF);\n"); fprintf(f, "\n"); @@ -621,13 +623,15 @@ print_stateenum(FILE *f, const struct fsm *fsm) } static int -print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) +print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, + const struct fsm_options *opt, const char *cp) { assert(f != NULL); assert(z != NULL); assert(z->fsm != NULL); assert(fsm_all(z->fsm, fsm_isdfa)); assert(ast != NULL); + assert(cp != NULL); /* TODO: prerequisite that the FSM is a DFA */ @@ -658,7 +662,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\n"); } - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\twhile (c = lx_getc(lx), c != EOF) {\n"); break; @@ -687,34 +691,25 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } { - const struct fsm_options *tmp; - static const struct fsm_options defaults; - struct fsm_options o = defaults; + static const struct fsm_hooks defaults; + struct fsm_hooks hooks = defaults; struct ir *ir; - tmp = z->fsm->opt; - - o.comments = z->fsm->opt->comments; - o.case_ranges = z->fsm->opt->case_ranges; - o.hooks.accept = accept_c; - o.hooks.reject = reject_c; - o.hooks.hook_opaque = (void *) ast; + assert(cp != NULL); - z->fsm->opt = &o; + hooks.accept = accept_c; + hooks.reject = reject_c; + hooks.hook_opaque = (void *) ast; - assert(opt.hooks.cp != NULL); - - ir = make_ir(z->fsm); + ir = make_ir(z->fsm, opt); if (ir == NULL) { /* TODO */ } /* XXX: abstraction */ - (void) fsm_print_cfrag(f, ir, &o, opt.hooks.cp); + (void) fsm_print_cfrag(f, ir, opt, &hooks, cp); free_ir(z->fsm, ir); - - z->fsm->opt = tmp; } if (~api_exclude & API_BUF) { @@ -765,7 +760,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\t\tdefault:\n"); fprintf(f, "\t\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.hooks.cp); + fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cp); fprintf(f, "\t\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t\t}\n"); fprintf(f, "\t\t\t}\n"); @@ -776,7 +771,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } else { fprintf(f, "\n"); fprintf(f, "\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.hooks.cp); + fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cp); fprintf(f, "\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t}\n"); fprintf(f, "\t\t}\n"); @@ -790,7 +785,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) { fsm_state_t i; - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tlx->lgetc = NULL;\n"); fprintf(f, "\n"); @@ -888,7 +883,7 @@ print_name(FILE *f, const struct ast *ast) } static int -print_example(FILE *f, const struct ast *ast) +print_example(FILE *f, const struct ast *ast, const struct fsm_options *opt) { struct ast_token *t; struct ast_zone *z; @@ -935,7 +930,7 @@ print_example(FILE *f, const struct ast *ast) fprintf(f, "\t\tcase %s", prefix.tok); esctok(f, t->s); fprintf(f, ": return \""); - escputs(f, z->fsm->opt, c_escputc_str, buf); + escputs(f, opt, c_escputc_str, buf); fprintf(f, "%s", n >= (int) sizeof buf - 1 ? "..." : ""); fprintf(f, "\";\n"); } @@ -960,18 +955,20 @@ print_example(FILE *f, const struct ast *ast) } void -lx_print_c(FILE *f, const struct ast *ast) +lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) { + const char *cp; const struct ast_zone *z; unsigned int zn; assert(f != NULL); assert(ast != NULL); + assert(opt != NULL); - switch (opt.io) { - case FSM_IO_GETC: opt.hooks.cp = "c"; break; - case FSM_IO_STR: opt.hooks.cp = "c"; break; - case FSM_IO_PAIR: opt.hooks.cp = "c"; break; + switch (opt->io) { + case FSM_IO_GETC: cp = "c"; break; + case FSM_IO_STR: cp = "c"; break; + case FSM_IO_PAIR: cp = "c"; break; } for (z = ast->zl; z != NULL; z = z->next) { @@ -1008,7 +1005,7 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\n"); - print_io(f); + print_io(f, opt); print_lgetc(f); print_buf(f); @@ -1025,7 +1022,7 @@ lx_print_c(FILE *f, const struct ast *ast) zn++; } - if (-1 == print_zone(f, ast, z)) { + if (-1 == print_zone(f, ast, z, opt, cp)) { return; /* XXX: handle error */ } } @@ -1035,7 +1032,7 @@ lx_print_c(FILE *f, const struct ast *ast) } if (~api_exclude & API_EXAMPLE) { - if (-1 == print_example(f, ast)) { + if (-1 == print_example(f, ast, opt)) { return; } } @@ -1051,7 +1048,7 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\t*lx = lx_default;\n"); fprintf(f, "\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tlx->c = EOF;\n"); break; @@ -1085,7 +1082,7 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\tassert(lx->z != NULL);\n"); fprintf(f, "\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tif (lx->lgetc == NULL) {\n"); break; @@ -1123,7 +1120,7 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\n"); - if (opt.io == FSM_IO_STR) { + if (opt->io == FSM_IO_STR) { fprintf(f, "void\n"); fprintf(f, "%sinput_str(struct %slx *lx, const char *p)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index d67a5e5ca..bf1a72063 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -67,7 +67,7 @@ mapping(FILE *f, const struct ast_mapping *m, const struct ast *ast) static int accept_dot(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { const struct ast_mapping *m; const struct ast *ast; @@ -75,12 +75,12 @@ accept_dot(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(opt->hooks.hook_opaque != NULL); assert(ids != NULL); assert(count > 0); assert(lang_opaque != NULL); + assert(hook_opaque != NULL); - ast = opt->hooks.hook_opaque; + ast = hook_opaque; s = * (fsm_state_t *) lang_opaque; m = ast_getendmappingbyendid(ids[0]); @@ -164,7 +164,8 @@ singlestate(FILE *f, const struct fsm *fsm, const struct ast *ast, } static void -print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) +print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, + const struct fsm_options *opt) { fsm_state_t start, i; @@ -172,6 +173,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) assert(ast != NULL); assert(z != NULL); assert(z->fsm != NULL); + assert(opt != NULL); (void) fsm_getstart(z->fsm, &start); @@ -188,28 +190,20 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\t\n"); { - const struct fsm_options *tmp; - static const struct fsm_options defaults; - struct fsm_options opt = defaults; - char p[128]; + static const struct fsm_hooks defaults; + struct fsm_hooks hooks = defaults; + struct fsm_options o = *opt; - tmp = z->fsm->opt; + char p[128]; (void) sprintf(p, "z%u", zindexof(ast, z)); - opt.anonymous_states = anonymous_states; - opt.fragment = 1; /* XXX */ - opt.consolidate_edges = z->fsm->opt->consolidate_edges; - opt.comments = z->fsm->opt->comments; - opt.prefix = p; - opt.hooks.accept = accept_dot; - opt.hooks.hook_opaque = (void *) ast; + o.prefix = p; - z->fsm->opt = &opt; + hooks.accept = accept_dot; + hooks.hook_opaque = (void *) ast; - fsm_print(f, z->fsm, FSM_PRINT_DOT); - - z->fsm->opt = tmp; + fsm_print(f, z->fsm, &o, &hooks, FSM_PRINT_DOT); } for (i = 0; i < z->fsm->statecount; i++) { @@ -224,13 +218,14 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } void -lx_print_dot(FILE *f, const struct ast *ast) +lx_print_dot(FILE *f, const struct ast *ast, const struct fsm_options *opt) { const struct ast_zone *z; unsigned int zn; fsm_state_t start; assert(f != NULL); + assert(opt != NULL); (void) fsm_getstart(ast->global->fsm, &start); @@ -295,7 +290,7 @@ lx_print_dot(FILE *f, const struct ast *ast) zn++; } - print_zone(f, ast, z); + print_zone(f, ast, z, opt); } fprintf(f, "}\n"); diff --git a/src/lx/print/dump.c b/src/lx/print/dump.c index 9b1b06b25..162630f88 100644 --- a/src/lx/print/dump.c +++ b/src/lx/print/dump.c @@ -13,10 +13,14 @@ #include "lx/print.h" -static void -print_dump(FILE *f) +void +lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) { assert(f != NULL); + assert(ast != NULL); + assert(opt != NULL); + + (void) ast; if (api_getc == API_FDGETC) { fprintf(f, "#define _POSIX_SOURCE\n"); @@ -67,7 +71,7 @@ print_dump(FILE *f) fprintf(f, "\tenum lx_token t;\n"); fprintf(f, "\tstruct lx lx = { 0 };\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tint (*lgetc)(struct lx *lx);\n"); fprintf(f, "\tvoid *getc_opaque;\n"); @@ -113,7 +117,7 @@ print_dump(FILE *f) fprintf(f, "\n"); - if (opt.io == FSM_IO_GETC && (api_getc != API_AGETC && api_getc != API_SGETC)) { + if (opt->io == FSM_IO_GETC && (api_getc != API_AGETC && api_getc != API_SGETC)) { fprintf(f, "\tif (argc != 1) {\n"); fprintf(f, "\t\tfprintf(stderr, \"usage: dump\\n\");\n"); fprintf(f, "\t\treturn 1;\n"); @@ -172,7 +176,7 @@ print_dump(FILE *f) fprintf(f, "\tlx_init(&lx);\n"); fprintf(f, "\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tlx.lgetc = lgetc;\n"); fprintf(f, "\tlx.getc_opaque = getc_opaque;\n"); @@ -303,14 +307,3 @@ print_dump(FILE *f) fprintf(f, "}\n"); } -void -lx_print_dump(FILE *f, const struct ast *ast) -{ - assert(f != NULL); - assert(ast != NULL); - - (void) ast; - - print_dump(f); -} - diff --git a/src/lx/print/h.c b/src/lx/print/h.c index c41b89662..01a34c62e 100644 --- a/src/lx/print/h.c +++ b/src/lx/print/h.c @@ -41,10 +41,11 @@ print_tokens(FILE *f, const struct ast *ast) } void -lx_print_h(FILE *f, const struct ast *ast) +lx_print_h(FILE *f, const struct ast *ast, const struct fsm_options *opt) { assert(f != NULL); assert(ast != NULL); + assert(opt != NULL); fprintf(f, "/* Generated by lx */\n"); /* TODO: date, input etc */ fprintf(f, "\n"); @@ -73,7 +74,7 @@ lx_print_h(FILE *f, const struct ast *ast) fprintf(f, "struct %slx {\n", prefix.lx); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tint (*lgetc)(struct %slx *lx);\n", prefix.lx); fprintf(f, "\tvoid *getc_opaque;\n"); @@ -233,7 +234,7 @@ lx_print_h(FILE *f, const struct ast *ast) fprintf(f, "\n"); } - if (opt.io == FSM_IO_STR) { + if (opt->io == FSM_IO_STR) { fprintf(f, "void %sinput_str(struct %slx *lx, const char *p);\n", prefix.api, prefix.lx); fprintf(f, "\n"); } diff --git a/src/lx/print/zdot.c b/src/lx/print/zdot.c index 7fa9b8b07..3fcf72b56 100644 --- a/src/lx/print/zdot.c +++ b/src/lx/print/zdot.c @@ -74,13 +74,16 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } void -lx_print_zdot(FILE *f, const struct ast *ast) +lx_print_zdot(FILE *f, const struct ast *ast, const struct fsm_options *opt) { const struct ast_zone *z; unsigned int zn; assert(f != NULL); assert(ast != NULL); + assert(opt != NULL); + + (void) opt; fprintf(f, "digraph %sG {\n", prefix.api); fprintf(f, "\trankdir = TB;\n"); diff --git a/src/lx/tokens.c b/src/lx/tokens.c index 3736ddf51..ac2fd52a9 100644 --- a/src/lx/tokens.c +++ b/src/lx/tokens.c @@ -6,6 +6,7 @@ #include #include +#include #include diff --git a/src/re/main.c b/src/re/main.c index 16c152cc0..e196103d4 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -425,8 +425,8 @@ printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) static int accept_c(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { const struct match *m; unsigned n; @@ -434,9 +434,11 @@ accept_c(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque == NULL); + assert(hook_opaque == NULL); (void) opt; (void) lang_opaque; + (void) hook_opaque; n = 0; @@ -467,8 +469,8 @@ accept_c(FILE *f, const struct fsm_options *opt, static int accept_rust(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { const struct match *m; unsigned n; @@ -476,9 +478,11 @@ accept_rust(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque == NULL); + assert(hook_opaque == NULL); (void) opt; (void) lang_opaque; + (void) hook_opaque; n = 0; @@ -509,8 +513,8 @@ accept_rust(FILE *f, const struct fsm_options *opt, static int accept_llvm(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { const struct match *m; unsigned n; @@ -518,8 +522,10 @@ accept_llvm(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + assert(hook_opaque == NULL); (void) opt; + (void) hook_opaque; n = 0; @@ -552,8 +558,8 @@ accept_llvm(FILE *f, const struct fsm_options *opt, static int accept_dot(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { const struct match *m; fsm_state_t s; @@ -561,8 +567,10 @@ accept_dot(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + assert(hook_opaque == NULL); (void) opt; + (void) hook_opaque; s = * (fsm_state_t *) lang_opaque; @@ -610,17 +618,19 @@ accept_dot(FILE *f, const struct fsm_options *opt, static int accept_json(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { const struct match *m; size_t i; assert(opt != NULL); assert(lang_opaque == NULL); + assert(hook_opaque == NULL); (void) opt; (void) lang_opaque; + (void) hook_opaque; fprintf(f, "[ "); @@ -683,10 +693,12 @@ int main(int argc, char *argv[]) { static const struct fsm_options zero_options; + static const struct fsm_hooks zero_hooks; /* TODO: use alloc hooks for -Q accounting */ struct fsm_alloc *alloc = NULL; struct fsm_options opt; + struct fsm_hooks hooks; struct fsm *fsm; struct fsm *(*join)(struct fsm *, struct fsm *, @@ -711,6 +723,7 @@ main(int argc, char *argv[]) atexit(do_fsm_cleanup); opt = zero_options; + hooks = zero_hooks; /* note these defaults are the opposite than for fsm(1) */ opt.anonymous_states = 1; @@ -871,7 +884,7 @@ main(int argc, char *argv[]) literal_r = re_is_literal(dialect, fsm_fgetc, f, - &opt, flags, &err, + flags, &err, &literal_category, &literal_s, &literal_n); fclose(f); @@ -882,7 +895,7 @@ main(int argc, char *argv[]) literal_r = re_is_literal(dialect, fsm_sgetc, &s, - &opt, flags, &err, + flags, &err, &literal_category, &literal_s, &literal_n); } @@ -945,11 +958,11 @@ main(int argc, char *argv[]) f = xopen(argv[0]); if (!fsmfiles) { - ast = re_parse(dialect, fsm_fgetc, f, &opt, flags, &err, NULL); + ast = re_parse(dialect, fsm_fgetc, f, flags, &err, NULL); } else { struct fsm *fsm; - fsm = fsm_parse(f, alloc, &opt); + fsm = fsm_parse(f, alloc); if (fsm == NULL) { perror("fsm_parse"); return EXIT_FAILURE; @@ -971,7 +984,7 @@ main(int argc, char *argv[]) s = argv[0]; - ast = re_parse(dialect, fsm_sgetc, &s, &opt, flags, &err, NULL); + ast = re_parse(dialect, fsm_sgetc, &s, flags, &err, NULL); } if (ast == NULL) { @@ -1001,8 +1014,6 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - fsm_setoptions(fsm, &opt); - fsm_to_cleanup = fsm; { @@ -1027,9 +1038,9 @@ main(int argc, char *argv[]) f = xopen(argv[i]); if (!fsmfiles) { - new = re_comp(dialect, fsm_fgetc, f, alloc, &opt, flags, &err); + new = re_comp(dialect, fsm_fgetc, f, alloc, flags, &err); } else { - new = fsm_parse(f, alloc, &opt); + new = fsm_parse(f, alloc); if (new == NULL) { perror("fsm_parse"); return EXIT_FAILURE; @@ -1042,7 +1053,7 @@ main(int argc, char *argv[]) s = argv[i]; - new = re_comp(dialect, fsm_sgetc, &s, alloc, &opt, flags, &err); + new = re_comp(dialect, fsm_sgetc, &s, alloc, flags, &err); } if (new == NULL) { @@ -1259,32 +1270,32 @@ main(int argc, char *argv[]) switch (fsm_lang) { case FSM_PRINT_C: case FSM_PRINT_VMC: - opt.hooks.accept = accept_c; + hooks.accept = accept_c; break; case FSM_PRINT_RUST: - opt.hooks.accept = accept_rust; + hooks.accept = accept_rust; break; case FSM_PRINT_LLVM: - opt.hooks.accept = accept_llvm; + hooks.accept = accept_llvm; break; case FSM_PRINT_DOT: case FSM_PRINT_VMDOT: - opt.hooks.accept = patterns ? accept_dot : NULL; + hooks.accept = patterns ? accept_dot : NULL; break; case FSM_PRINT_JSON: - opt.hooks.accept = patterns ? accept_json : NULL; + hooks.accept = patterns ? accept_json : NULL; break; default: - opt.hooks.accept = NULL; + hooks.accept = NULL; break; } - if (-1 == fsm_print(stdout, fsm, fsm_lang)) { + if (-1 == fsm_print(stdout, fsm, &opt, &hooks, fsm_lang)) { if (errno == ENOTSUP) { fprintf(stderr, "unsupported IO API\n"); } else { diff --git a/src/retest/cvtpcre.c b/src/retest/cvtpcre.c index 3eaa57ad1..d35b8459d 100644 --- a/src/retest/cvtpcre.c +++ b/src/retest/cvtpcre.c @@ -430,13 +430,10 @@ state_name(enum state st) int main(int argc, char **argv) { - static const struct fsm_options zero_options; - struct str l; enum state state; size_t count, nparsed, linenum, regexp_line; int re_ok = 0; - struct fsm_options opt; char **skip; size_t nskip; @@ -455,12 +452,6 @@ main(int argc, char **argv) l = init_str; state = ST_DEFAULT; - opt = zero_options; - opt.comments = 0; - opt.anonymous_states = 1; - opt.consolidate_edges = 1; - opt.io = FSM_IO_GETC; - count = 0; nparsed = 0; linenum = 0; @@ -620,7 +611,7 @@ main(int argc, char **argv) comp_err = err_zero; - fsm = re_comp(RE_PCRE, fsm_sgetc, &re, NULL, &opt, mods, &comp_err); + fsm = re_comp(RE_PCRE, fsm_sgetc, &re, NULL, mods, &comp_err); re_ok = (fsm != NULL); if (re_ok) { fsm_free(fsm); diff --git a/src/retest/main.c b/src/retest/main.c index c3dd4e74c..2183688ba 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -978,7 +978,7 @@ process_test_file(const char *filename, } re_str = regexp; - fsm = re_comp(dialect, fsm_sgetc, &re_str, alloc, &opt, flags, &err); + fsm = re_comp(dialect, fsm_sgetc, &re_str, alloc, flags, &err); if (fsm == NULL) { fprintf(stderr, "line %d: error with %s regexp /%s/%s: %s\n", linenum, dialect_name, regexp, flagdesc, re_strerror(err.e)); @@ -1062,7 +1062,7 @@ process_test_file(const char *filename, fprintf(stderr, "REGEXP matching for /%s/%s\n", regexp, flagdesc); #endif /* DEBUG_TEST_REGEXP */ - ret = fsm_runner_initialize(fsm, &runner, impl, vm_opts); + ret = fsm_runner_initialize(fsm, &opt, &runner, impl, vm_opts); fsm_free(fsm); diff --git a/src/retest/reperf.c b/src/retest/reperf.c index bcdbc5559..65fd9072a 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -688,7 +688,7 @@ perf_case_run(struct perf_case *c, enum halt halt, re = c->regexp.data; - fsm = re_comp(c->dialect, fsm_sgetc, &re, alloc, &opt, flags, &comp_err); + fsm = re_comp(c->dialect, fsm_sgetc, &re, alloc, flags, &comp_err); if (fsm == NULL) { return ERROR_PARSING_REGEXP; } @@ -728,7 +728,7 @@ perf_case_run(struct perf_case *c, enum halt halt, goto done; } - ret = fsm_runner_initialize(fsm, &runner, c->impl, vm_opts); + ret = fsm_runner_initialize(fsm, &opt, &runner, c->impl, vm_opts); if (ret != ERROR_NONE) { fsm_free(fsm); return ret; diff --git a/src/retest/runner.c b/src/retest/runner.c index 95f89aa91..ea5a54372 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -61,7 +61,10 @@ xmkstemps(char *s) } static int -print(const struct fsm *fsm, enum implementation impl, +print(const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum implementation impl, char *tmp_src) { int fd_src; @@ -76,7 +79,7 @@ print(const struct fsm *fsm, enum implementation impl, } /* the vmc codegen can emit memcmp() or strncmp() calls */ - if (impl == IMPL_VMC && (fsm_getoptions(fsm)->io == FSM_IO_PAIR || fsm_getoptions(fsm)->io == FSM_IO_STR)) { + if (impl == IMPL_VMC && (opt->io == FSM_IO_PAIR || opt->io == FSM_IO_STR)) { fprintf(f, "#include \n\n"); } @@ -84,18 +87,18 @@ print(const struct fsm *fsm, enum implementation impl, int e; switch (impl) { - case IMPL_C: e = fsm_print(f, fsm, FSM_PRINT_C); break; - case IMPL_RUST: e = fsm_print(f, fsm, FSM_PRINT_RUST); break; - case IMPL_LLVM: e = fsm_print(f, fsm, FSM_PRINT_LLVM); break; - case IMPL_VMC: e = fsm_print(f, fsm, FSM_PRINT_VMC); break; - case IMPL_GOASM: e = fsm_print(f, fsm, FSM_PRINT_AMD64_GO); break; - case IMPL_VMASM: e = fsm_print(f, fsm, FSM_PRINT_AMD64_ATT); break; - case IMPL_GO: e = fsm_print(f, fsm, FSM_PRINT_GO); break; + case IMPL_C: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_C); break; + case IMPL_RUST: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_RUST); break; + case IMPL_LLVM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_LLVM); break; + case IMPL_VMC: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMC); break; + case IMPL_GOASM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_AMD64_GO); break; + case IMPL_VMASM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_AMD64_ATT); break; + case IMPL_GO: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_GO); break; case IMPL_VMOPS: - e = fsm_print(f, fsm, FSM_PRINT_VMOPS_H) - | fsm_print(f, fsm, FSM_PRINT_VMOPS_C) - | fsm_print(f, fsm, FSM_PRINT_VMOPS_MAIN); + e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMOPS_H) + | fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMOPS_C) + | fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMOPS_MAIN); break; case IMPL_INTERPRET: @@ -285,7 +288,9 @@ compile(enum implementation impl, } static enum error_type -runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation impl) +runner_init_compiled(struct fsm *fsm, + const struct fsm_options *opt, + struct fsm_runner *r, enum implementation impl) { void *h; @@ -316,7 +321,8 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation abort(); } - if (!print(fsm, r->impl, tmp_src)) { + /* we don't override the print hooks for retest */ + if (!print(fsm, opt, NULL, r->impl, tmp_src)) { return ERROR_FILE_IO; } @@ -399,7 +405,8 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation } enum error_type -fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts) +fsm_runner_initialize(struct fsm *fsm, const struct fsm_options *opt, + struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts) { static const struct fsm_runner zero; struct fsm_dfavm *vm; @@ -418,10 +425,10 @@ fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation case IMPL_VMOPS: case IMPL_GO: case IMPL_GOASM: - return runner_init_compiled(fsm, r, impl); + return runner_init_compiled(fsm, opt, r, impl); case IMPL_INTERPRET: - vm = fsm_vm_compile_with_options(fsm, vm_opts); + vm = fsm_vm_compile_with_options(fsm, opt, vm_opts); if (vm == NULL) { fsm_free(fsm); return ERROR_COMPILING_BYTECODE; diff --git a/src/retest/runner.h b/src/retest/runner.h index 3bd1531ab..3d53bc511 100644 --- a/src/retest/runner.h +++ b/src/retest/runner.h @@ -6,6 +6,7 @@ #include struct fsm; +struct fsm_options; struct fsm_dfavm; enum error_type { @@ -73,7 +74,8 @@ struct fsm_runner { }; enum error_type -fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts); +fsm_runner_initialize(struct fsm *fsm, const struct fsm_options *opt, + struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts); void fsm_runner_finalize(struct fsm_runner *r); diff --git a/tests/aho_corasick/actest.c b/tests/aho_corasick/actest.c index b453060d0..40dcbdf4f 100644 --- a/tests/aho_corasick/actest.c +++ b/tests/aho_corasick/actest.c @@ -164,7 +164,7 @@ int main(int argc, char *argv[]) { flags |= RE_STRINGS_ANCHOR_RIGHT; } - fsm = re_strings(NULL, &opt, (const char **)words.list, words.len, flags); + fsm = re_strings(NULL, (const char **)words.list, words.len, flags); wordlist_finalize(&words); if (fsm == NULL) { @@ -173,7 +173,7 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } - fsm_print(stdout, fsm, lang); + fsm_print(stdout, fsm, &opt, NULL, lang); fsm_free(fsm); diff --git a/tests/capture/captest.c b/tests/capture/captest.c index ad2d0c21f..abfb125e0 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -80,7 +80,7 @@ captest_run_single(const struct captest_single_fsm_test_info *info) } #if CAPTEST_RUN_SINGLE_LOG - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "fsm", fsm); #endif diff --git a/tests/capture/capture3.c b/tests/capture/capture3.c index 919716e41..6dde011d6 100644 --- a/tests/capture/capture3.c +++ b/tests/capture/capture3.c @@ -83,15 +83,15 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "\n=== f_ab...\n"); - fsm_print(stderr, f_ab, FSM_PRINT_FSM); + fsm_print(stderr, f_ab, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_ab", f_ab); fprintf(stderr, "\n=== f_cde...\n"); - fsm_print(stderr, f_cde, FSM_PRINT_FSM); + fsm_print(stderr, f_cde, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_cde", f_cde); fprintf(stderr, "\n=== f_fghi...\n"); - fsm_print(stderr, f_fghi, FSM_PRINT_FSM); + fsm_print(stderr, f_fghi, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_fghi", f_fghi); #endif @@ -111,14 +111,14 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "=== unioned f_ab with f_cde... (CB ab: %u, cde: %u)\n", bases[0].capture, bases[1].capture); - fsm_print(stderr, f_all, FSM_PRINT_FSM); + fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_all", f_all); #endif #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "=== unioned f_all with f_fghi... (CB fghi: %u), %u captures\n", bases[2].capture, fsm_countcaptures(f_all)); - fsm_print(stderr, f_all, FSM_PRINT_FSM); + fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_all #2", f_all); #endif @@ -129,7 +129,7 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, f_all, FSM_PRINT_FSM); + fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### f_all", f_all); #endif @@ -164,7 +164,7 @@ det_and_min(const char *tag, struct fsm *fsm) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, tag, fsm); #endif diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index 5ee847770..ae27c416c 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -90,11 +90,11 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== abc \n"); - fsm_print(stderr, f_abc, FSM_PRINT_FSM); + fsm_print(stderr, f_abc, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "abc", f_abc); fprintf(stderr, "==================== ab*c \n"); - fsm_print(stderr, f_ab_c, FSM_PRINT_FSM); + fsm_print(stderr, f_ab_c, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "ab*c", f_ab_c); #endif @@ -110,7 +110,7 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== post-union \n"); - fsm_print(stderr, f_all, FSM_PRINT_FSM); + fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "capture_actions", f_all); fprintf(stderr, "====================\n"); #endif @@ -121,7 +121,7 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== post-det \n"); - fsm_print(stderr, f_all, FSM_PRINT_FSM); + fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "capture_actions", f_all); fprintf(stderr, "====================\n"); #endif @@ -144,7 +144,7 @@ det_and_min(const char *tag, struct fsm *fsm) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, tag, fsm); #endif diff --git a/tests/capture/capture5.c b/tests/capture/capture5.c index f103b851e..cbcb3ee9d 100644 --- a/tests/capture/capture5.c +++ b/tests/capture/capture5.c @@ -77,7 +77,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== built\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "built", fsm); #endif @@ -88,7 +88,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "after det", fsm); #endif @@ -99,7 +99,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after min\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "after min", fsm); #endif return fsm; diff --git a/tests/capture/capture_concat1.c b/tests/capture/capture_concat1.c index 084b666ea..23db17a25 100644 --- a/tests/capture/capture_concat1.c +++ b/tests/capture/capture_concat1.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after concat: cb_ab %u, cb_cde %u\n", *cb_a, *cb_b); - fsm_print(stderr, abcde, FSM_PRINT_FSM); + fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### after concat", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, abcde, FSM_PRINT_FSM); + fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/capture/capture_concat2.c b/tests/capture/capture_concat2.c index 4f8743244..b8e65025e 100644 --- a/tests/capture/capture_concat2.c +++ b/tests/capture/capture_concat2.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after concat: cb_abc %u, cb_de %u\n", *cb_a, *cb_b); - fsm_print(stderr, abcde, FSM_PRINT_FSM); + fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### after concat", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, abcde, FSM_PRINT_FSM); + fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/capture/capture_union1.c b/tests/capture/capture_union1.c index 704544997..8fe6fce58 100644 --- a/tests/capture/capture_union1.c +++ b/tests/capture/capture_union1.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after union: cb_ab %u, cb_cde %u\n", *cb_a, *cb_b); - fsm_print(stderr, abcde, FSM_PRINT_FSM); + fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); fsm_capture_dump(stderr, "#### after union", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, abcde, FSM_PRINT_FSM); + fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index 34e12f116..1f5e6f9b6 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -26,7 +26,7 @@ int main(void) size_t nend; size_t nstates, state_ind; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_determinise(fsm); assert(ret == 1); diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index 09ca8e8a3..86a66063c 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -28,7 +28,7 @@ int main(void) size_t i; fsm_state_t end_state; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_determinise(fsm); assert(ret == 1); diff --git a/tests/endids/endids10_minimise_partial_overlap.c b/tests/endids/endids10_minimise_partial_overlap.c index 3d7d9e78b..5a72d05d9 100644 --- a/tests/endids/endids10_minimise_partial_overlap.c +++ b/tests/endids/endids10_minimise_partial_overlap.c @@ -33,14 +33,14 @@ int main(void) const char *regex_ab_star_c = "^ab*c$"; const char *regex_abc = "^abc$"; - struct fsm *fsm_ab_star_c = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_ab_star_c, NULL, NULL, 0, NULL); + struct fsm *fsm_ab_star_c = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_ab_star_c, NULL, 0, NULL); assert(fsm_ab_star_c != NULL); if (!fsm_setendid(fsm_ab_star_c, ENDID_AB_STAR_C)) { assert(!"setendid"); } if (!fsm_determinise(fsm_ab_star_c)) { assert(!"determinise"); } if (!fsm_minimise(fsm_ab_star_c)) { assert(!"minimise"); } - struct fsm *fsm_abc = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_abc, NULL, NULL, 0, NULL); + struct fsm *fsm_abc = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_abc, NULL, 0, NULL); assert(fsm_abc != NULL); if (!fsm_setendid(fsm_abc, ENDID_ABC)) { assert(!"setendid"); } diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 5b34da511..472807eed 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -25,7 +25,7 @@ int main(void) int ret; size_t nstates, state_ind; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index a8327398a..a2feafb18 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -27,7 +27,7 @@ int main(void) int ret; size_t nstates, state_ind; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index 72588015f..0d6182437 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -26,8 +26,8 @@ int main(void) fsm_end_id_t all_endids[2]; unsigned nstates, state_ind; - s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); + s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index e152987c4..a573f1921 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -149,7 +149,7 @@ int main(void) strncpy(&buf[0], patterns[i], sizeof buf); bufp = &buf[0]; - new = re_comp(RE_NATIVE, fsm_sgetc, &bufp, NULL, NULL, 0, NULL); + new = re_comp(RE_NATIVE, fsm_sgetc, &bufp, NULL, 0, NULL); assert(new != NULL); ret = fsm_determinise(new); diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index a9e095193..a4530c93b 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -27,8 +27,8 @@ int main(void) fsm_end_id_t all_endids[2]; unsigned nstates, state_ind; - s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); + s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index 923e7503d..50030ea0d 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -27,8 +27,8 @@ int main(void) fsm_end_id_t all_endids[2]; unsigned nstates, state_ind; - s1 = "(abc|def)"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); + s1 = "(abc|def)"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index 37c7baced..e237c2881 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -60,14 +60,14 @@ int main(void) size_t nstates, ninfo, state_ind, info_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index 0637b2700..3021be830 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -72,14 +72,14 @@ int main(void) size_t nstates, ninfo, state_ind, info_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index 975c5c2c5..939c4b306 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -76,14 +76,14 @@ int main(void) size_t nstates, ninfo, state_ind, info_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index 16db9ac8e..12b36fae0 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -27,8 +27,8 @@ int main(void) fsm_end_id_t all_ids[2]; unsigned nstates, state_ind; - s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, NULL, 0, NULL); - s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, NULL, 0, NULL); + s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); + s2 = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s2, NULL, 0, NULL); assert(s1 != NULL); assert(s2 != NULL); diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index 0a0d5b66e..519399973 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -37,14 +37,14 @@ int main(void) size_t nstates, ninfo, state_ind, nend; int ret; - s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "abc"; fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm, (fsm_end_id_t) 1); assert(ret == 1); { struct fsm *fsm2; - s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, NULL, 0, NULL); + s = "def"; fsm2 = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); ret = fsm_setendid(fsm2, (fsm_end_id_t) 2); assert(ret == 1); diff --git a/tests/gen/gen3.c b/tests/gen/gen3.c index 56b158f41..ff9c9bd6f 100644 --- a/tests/gen/gen3.c +++ b/tests/gen/gen3.c @@ -61,7 +61,7 @@ build(void) } if (getenv("PRINT")) { - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); } return fsm; diff --git a/tests/minimise/minimise_test_case_list.c b/tests/minimise/minimise_test_case_list.c index c3e82a236..c66f02df2 100644 --- a/tests/minimise/minimise_test_case_list.c +++ b/tests/minimise/minimise_test_case_list.c @@ -81,7 +81,7 @@ check_minimisation(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, RE_MULTI, &err); assert(fsm != NULL); if (!fsm_determinise(fsm)) { return 0; @@ -110,10 +110,10 @@ check_minimisation(const char *pattern) __func__, pattern, expected_state_count, state_count_min); fprintf(stderr, "== expected:\n"); - fsm_print(stderr, oracle_min, FSM_PRINT_FSM); + fsm_print(stderr, oracle_min, &opt, NULL, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_print(stderr, fsm, &opt, NULL, FSM_PRINT_FSM); } fsm_free(oracle_min); diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c index d72a209f8..4f371f797 100644 --- a/tests/re_strings/testutil.c +++ b/tests/re_strings/testutil.c @@ -32,7 +32,7 @@ run_test(const char **strings) const int flags = 0; /* not anchored */ - struct fsm *fsm = re_strings_build(s, NULL, NULL, flags); + struct fsm *fsm = re_strings_build(s, NULL, flags); assert(fsm != NULL); /* Each literal string input should match, and the set of From 9483c505d28bc9ffe586657f8fd419104f165f5c Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 22 Jul 2024 09:44:17 +0100 Subject: [PATCH 209/373] fsm_dump(), a convenience to replace the original fsm_print() for debugging. Now fsm_print() has a bunch of arguments, it's going to be annoying for debugging during development. fsm_dump() returns the same interface, but with some options set for compact output. The output looks like this: ``` # src/re/main.c:1081 new 0; 1; 0 -> 1 "a" .. "c", "x"; # e.g. "a" start: 0; end: 1; ``` --- include/fsm/print.h | 5 +++++ src/libfsm/determinise.c | 2 +- src/libfsm/epsilons.c | 4 ++-- src/libfsm/libfsm.syms | 1 + src/libfsm/minimise.c | 4 ++-- src/libfsm/print.c | 23 +++++++++++++++++++++++ src/retest/main.c | 4 ++-- src/retest/reperf.c | 4 ++-- tests/capture/captest.c | 2 +- tests/capture/capture3.c | 14 +++++++------- tests/capture/capture4.c | 10 +++++----- tests/capture/capture5.c | 6 +++--- tests/capture/capture_concat1.c | 4 ++-- tests/capture/capture_concat2.c | 4 ++-- tests/capture/capture_union1.c | 4 ++-- tests/gen/gen3.c | 3 ++- tests/minimise/minimise_test_case_list.c | 10 ++-------- 17 files changed, 64 insertions(+), 40 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index ee889d211..4a47a7fed 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -10,6 +10,11 @@ struct fsm; struct fsm_options; +/* a convenience for debugging */ +void fsm_dump(FILE *f, const struct fsm *fsm, + const char *file, unsigned line, const char *name); +#define fsm_dump(f, fsm) (fsm_dump)((f), (fsm), __FILE__, __LINE__, #fsm); + enum fsm_print_lang { FSM_PRINT_NONE, /* No output */ diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index a32d15a9e..805443e37 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -46,7 +46,7 @@ fsm_determinise(struct fsm *nfa) #if LOG_DETERMINISE_CAPTURES || LOG_INPUT fprintf(stderr, "# post_remove_epsilons, pre_determinise\n"); - fsm_print(stderr, nfa, FSM_PRINT_FSM); + fsm_dump(stderr, nfa); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index f7418947c..51dd3e138 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -153,7 +153,7 @@ fsm_remove_epsilons(struct fsm *nfa) #if LOG_RESULT fprintf(stderr, "=== %s: about to update capture actions\n", __func__); - fsm_print(stderr, nfa, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, nfa); #endif if (!remap_capture_actions(nfa, eclosures)) { @@ -161,7 +161,7 @@ fsm_remove_epsilons(struct fsm *nfa) } #if LOG_RESULT - fsm_print(stderr, nfa, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, nfa); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 8f73c6e13..56734e7f2 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -35,6 +35,7 @@ fsm_generate_cb_printf_escaped fsm_parse # +fsm_dump fsm_print fsm_print_amd64_att fsm_print_amd64_go diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index bc03168c3..a8d53c57e 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -144,10 +144,10 @@ fsm_minimise(struct fsm *fsm) fprintf(stderr, "%s: expected minimal DFA with %zu states, got %zu\n", __func__, exp_count, got_count); fprintf(stderr, "== expected:\n"); - fsm_print(stderr, oracle, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, oracle); fprintf(stderr, "== got:\n"); - fsm_print(stderr, dst, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, dst); assert(!"non-minimal result"); } diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 0cc4cd340..5597a556c 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -214,3 +214,26 @@ fsm_print(FILE *f, const struct fsm *fsm, return r; } +void +(fsm_dump)(FILE *f, const struct fsm *fsm, + const char *file, unsigned line, const char *name) +{ + assert(f != NULL); + assert(file != NULL); + assert(name != NULL); + + fprintf(f, "# %s:%d %s\n", file, line, name); + + if (fsm == NULL) { + fprintf(f, "# (null)\n"); + return; + } + + fsm_print(f, fsm, & (const struct fsm_options) { + .consolidate_edges = 1, + .comments = 1, + .group_edges = 1 + }, NULL, FSM_PRINT_FSM); + +} + diff --git a/src/retest/main.c b/src/retest/main.c index 2183688ba..163e93bb9 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -1049,11 +1049,11 @@ process_test_file(const char *filename, #if DEBUG_VM_FSM fprintf(stderr, "FSM:\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fprintf(stderr, "---\n"); { FILE *f = fopen("dump.fsm", "w"); - fsm_print(f, fsm, FSM_PRINT_FSM); + fsm_dump(f, fsm); fclose(f); } #endif /* DEBUG_VM_FSM */ diff --git a/src/retest/reperf.c b/src/retest/reperf.c index 65fd9072a..43c294821 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -736,11 +736,11 @@ perf_case_run(struct perf_case *c, enum halt halt, #if DEBUG_VM_FSM fprintf(stderr, "FSM:\n"); - fsm_print(stderr, fsm, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fprintf(stderr, "---\n"); { FILE *f = fopen("dump.fsm", "w"); - fsm_print(f, fsm, FSM_PRINT_FSM); + fsm_dump(f, fsm); fclose(f); } #endif /* DEBUG_VM_FSM */ diff --git a/tests/capture/captest.c b/tests/capture/captest.c index abfb125e0..4e6e47beb 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -80,7 +80,7 @@ captest_run_single(const struct captest_single_fsm_test_info *info) } #if CAPTEST_RUN_SINGLE_LOG - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fsm_capture_dump(stderr, "fsm", fsm); #endif diff --git a/tests/capture/capture3.c b/tests/capture/capture3.c index 6dde011d6..40a103425 100644 --- a/tests/capture/capture3.c +++ b/tests/capture/capture3.c @@ -83,15 +83,15 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "\n=== f_ab...\n"); - fsm_print(stderr, f_ab, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_ab); fsm_capture_dump(stderr, "#### f_ab", f_ab); fprintf(stderr, "\n=== f_cde...\n"); - fsm_print(stderr, f_cde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_cde); fsm_capture_dump(stderr, "#### f_cde", f_cde); fprintf(stderr, "\n=== f_fghi...\n"); - fsm_print(stderr, f_fghi, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_fghi); fsm_capture_dump(stderr, "#### f_fghi", f_fghi); #endif @@ -111,14 +111,14 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "=== unioned f_ab with f_cde... (CB ab: %u, cde: %u)\n", bases[0].capture, bases[1].capture); - fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_all); fsm_capture_dump(stderr, "#### f_all", f_all); #endif #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "=== unioned f_all with f_fghi... (CB fghi: %u), %u captures\n", bases[2].capture, fsm_countcaptures(f_all)); - fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_all); fsm_capture_dump(stderr, "#### f_all #2", f_all); #endif @@ -129,7 +129,7 @@ int main(void) { #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_all); fsm_capture_dump(stderr, "#### f_all", f_all); #endif @@ -164,7 +164,7 @@ det_and_min(const char *tag, struct fsm *fsm) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fsm_capture_dump(stderr, tag, fsm); #endif diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c index ae27c416c..aa2a8ec3b 100644 --- a/tests/capture/capture4.c +++ b/tests/capture/capture4.c @@ -90,11 +90,11 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== abc \n"); - fsm_print(stderr, f_abc, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_abc); fsm_capture_dump(stderr, "abc", f_abc); fprintf(stderr, "==================== ab*c \n"); - fsm_print(stderr, f_ab_c, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_ab_c); fsm_capture_dump(stderr, "ab*c", f_ab_c); #endif @@ -110,7 +110,7 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== post-union \n"); - fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_all); fsm_capture_dump(stderr, "capture_actions", f_all); fprintf(stderr, "====================\n"); #endif @@ -121,7 +121,7 @@ build_and_combine(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==================== post-det \n"); - fsm_print(stderr, f_all, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, f_all); fsm_capture_dump(stderr, "capture_actions", f_all); fprintf(stderr, "====================\n"); #endif @@ -144,7 +144,7 @@ det_and_min(const char *tag, struct fsm *fsm) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fsm_capture_dump(stderr, tag, fsm); #endif diff --git a/tests/capture/capture5.c b/tests/capture/capture5.c index cbcb3ee9d..e09a20f27 100644 --- a/tests/capture/capture5.c +++ b/tests/capture/capture5.c @@ -77,7 +77,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== built\n"); - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fsm_capture_dump(stderr, "built", fsm); #endif @@ -88,7 +88,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after det\n"); - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fsm_capture_dump(stderr, "after det", fsm); #endif @@ -99,7 +99,7 @@ build(void) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after min\n"); - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); fsm_capture_dump(stderr, "after min", fsm); #endif return fsm; diff --git a/tests/capture/capture_concat1.c b/tests/capture/capture_concat1.c index 23db17a25..73a2ade57 100644 --- a/tests/capture/capture_concat1.c +++ b/tests/capture/capture_concat1.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after concat: cb_ab %u, cb_cde %u\n", *cb_a, *cb_b); - fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, abcde); fsm_capture_dump(stderr, "#### after concat", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, abcde); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/capture/capture_concat2.c b/tests/capture/capture_concat2.c index b8e65025e..e28ad8dca 100644 --- a/tests/capture/capture_concat2.c +++ b/tests/capture/capture_concat2.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after concat: cb_abc %u, cb_de %u\n", *cb_a, *cb_b); - fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, abcde); fsm_capture_dump(stderr, "#### after concat", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, abcde); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/capture/capture_union1.c b/tests/capture/capture_union1.c index 8fe6fce58..720a7848c 100644 --- a/tests/capture/capture_union1.c +++ b/tests/capture/capture_union1.c @@ -74,7 +74,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after union: cb_ab %u, cb_cde %u\n", *cb_a, *cb_b); - fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, abcde); fsm_capture_dump(stderr, "#### after union", abcde); @@ -87,7 +87,7 @@ build(unsigned *cb_a, unsigned *cb_b) #if LOG_INTERMEDIATE_FSMS fprintf(stderr, "==== after determinise\n"); - fsm_print(stderr, abcde, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, abcde); assert(fsm_countcaptures(abcde) == cc_abcde); diff --git a/tests/gen/gen3.c b/tests/gen/gen3.c index ff9c9bd6f..f24217622 100644 --- a/tests/gen/gen3.c +++ b/tests/gen/gen3.c @@ -11,6 +11,7 @@ #include #include +#include #include #include "gtest.h" @@ -61,7 +62,7 @@ build(void) } if (getenv("PRINT")) { - fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); } return fsm; diff --git a/tests/minimise/minimise_test_case_list.c b/tests/minimise/minimise_test_case_list.c index c66f02df2..06ad598bf 100644 --- a/tests/minimise/minimise_test_case_list.c +++ b/tests/minimise/minimise_test_case_list.c @@ -62,12 +62,6 @@ scanner_next(void *opaque) return (int) c; } -static const struct fsm_options opt = { - .consolidate_edges = 1, - .comments = 1, - .group_edges = 1, -}; - static int check_minimisation(const char *pattern) { @@ -110,10 +104,10 @@ check_minimisation(const char *pattern) __func__, pattern, expected_state_count, state_count_min); fprintf(stderr, "== expected:\n"); - fsm_print(stderr, oracle_min, &opt, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, oracle_min); fprintf(stderr, "== got:\n"); - fsm_print(stderr, fsm, &opt, NULL, FSM_PRINT_FSM); + fsm_dump(stderr, fsm); } fsm_free(oracle_min); From 2b9b1545858cc204cd8bb1204071908b585aa64a Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 26 Jul 2024 15:27:50 +0100 Subject: [PATCH 210/373] Port CLI tools to use opt.ambig and the conflict callback. There are a couple of awkward spots here (in particular calling fsm_endid_get() seems cumbersome for a user), but overall I think this came out really nicely. It does simplify a lot of the caller-side bookkeeping around tracking conflicts. And that's reflected in the diff removing program-specific datastructures. --- include/fsm/print.h | 18 +- src/fsm/main.c | 1 + src/libfsm/print.c | 179 ++++++++++++++++++- src/libfsm/print.h | 7 + src/libfsm/print/ir.c | 122 ++++++++----- src/libfsm/print/ir.h | 4 + src/lx/ast.c | 34 ---- src/lx/ast.h | 9 - src/lx/main.c | 161 ++++++++++------- src/lx/print/dot.c | 48 ++--- src/re/main.c | 405 +++++++++++++++--------------------------- 11 files changed, 534 insertions(+), 454 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index 4a47a7fed..9add9ba4d 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -53,23 +53,29 @@ enum fsm_print_lang { * Placement in the output stream depends on the format. * This replaces an entire "return xyz;" statement for C-like formats, * but appends extra information for others. + * + * ids[] is sorted and does not have duplicates. + * + * example may be NULL. */ struct fsm_hooks { /* character pointer, for C code fragment output. NULL for the default. */ const char *cp; int (*args)(FILE *, const struct fsm_options *opt, - void *leaf_opaque, void *hook_opaque); + void *lang_opaque, void *hook_opaque); - /* - * ids[] is sorted and does not have duplicates. - */ int (*accept)(FILE *, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *leaf_opaque, void *hook_opaque); + void *lang_opaque, void *hook_opaque); int (*reject)(FILE *, const struct fsm_options *opt, - void *leaf_opaque, void *hook_opaque); + void *lang_opaque, void *hook_opaque); + + int (*conflict)(FILE *, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + const char *example, + void *hook_opaque); void *hook_opaque; }; diff --git a/src/fsm/main.c b/src/fsm/main.c index 9665c9a2c..1d60ce5c7 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -386,6 +386,7 @@ main(int argc, char *argv[]) atexit(do_fsm_cleanup); opt = zero_options; + opt.ambig = AMBIG_MULTIPLE; opt.comments = 1; opt.io = FSM_IO_GETC; diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 5597a556c..6ff34e4ae 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,10 @@ print_hook_accept(FILE *f, assert(opt != NULL); assert(hooks != NULL); + if (opt->ambig == AMBIG_ERROR) { + assert(count <= 1); + } + if (default_accept == NULL) { assert(lang_opaque == NULL); } @@ -103,6 +108,105 @@ print_hook_reject(FILE *f, return 0; } +int +print_hook_conflict(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const fsm_end_id_t *ids, size_t count, + const char *example) +{ + assert(opt != NULL); + assert(hooks != NULL); + + /* f may be NULL for FSM_PRINT_NONE */ + + if (hooks->conflict != NULL) { + return hooks->conflict(f, opt, ids, count, + example, + hooks->hook_opaque); + } + + return 0; +} + +static int +print_conflicts(FILE *f, const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks) +{ + fsm_state_t s; + int conflicts; + + assert(fsm != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + conflicts = 0; + + for (s = 0; s < fsm->statecount; s++) { + char *example; + fsm_end_id_t *ids; + size_t count; + int res; + + if (!fsm_isend(fsm, s)) { + continue; + } + + count = fsm_endid_count(fsm, s); + if (count <= 1) { + continue; + } + + /* + * It's unfortunate that we call fsm_example() and allocate ids[] here + * as well as during make_ir(), but I can't find a way to do them in + * just one place. That's because we handle conflicts for output formats + * where there isn't an ir, and because the ir only exists for DFA, + * we can't move all output formats to use the ir. + */ + + if (!make_example(fsm, s, &example)) { + goto error; + } + + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + goto error; + } + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); + + // TODO: de-duplicate by ids[], so we don't call the conflict hook an unneccessary number of times + + /* + * The conflict hook is called here (rather in the caller), + * because it can be called multiple times, for different states. + */ + if (-1 == print_hook_conflict(f, opt, hooks, + ids, count, + example)) + { + goto error; + } + + if (example != NULL) { + f_free(fsm->alloc, example); + } + + f_free(fsm->alloc, ids); + + conflicts++; + } + + return conflicts; + +error: + + return -1; +} + int fsm_print(FILE *f, const struct fsm *fsm, const struct fsm_options *opt, @@ -127,7 +231,9 @@ fsm_print(FILE *f, const struct fsm *fsm, ir_print_f *print_ir = NULL; vm_print_f *print_vm = NULL; - assert(f != NULL); + if (lang != FSM_PRINT_NONE) { + assert(f != NULL); + } if (opt == NULL) { opt = &opt_zero; @@ -137,11 +243,40 @@ fsm_print(FILE *f, const struct fsm *fsm, hooks = &hooks_zero; } - if (lang == FSM_PRINT_NONE) { - return 0; + ir = NULL; + + /* + * fsm_print() can print an NFA or a DFA. The ir is DFA-only. + * So we can't use the ir for ambiguity checks, because it + * can't be constructed when we have an NFA. + * + * That also means we can't move all output routines to ir-only, + * we need to keep struct fsm around. So we're walking the fsm + * struct to check for ambiguities. + * + * This could be done conditionally, only when print_fsm != NULL + * and then we'd also check for ambiguities for ir and vm output + * at the point where the accept hook is called. But there's no + * reason to do that, it's simpler to detect all the conflicts + * ahead of time regardless. + */ + if (opt->ambig == AMBIG_ERROR) { + int count; + + count = print_conflicts(f, fsm, opt, hooks); + if (count == -1) { + goto error; + } + + if (count > 0) { + goto conflict; + } } switch (lang) { + case FSM_PRINT_NONE: + return 0; + case FSM_PRINT_AMD64_ATT: print_vm = fsm_print_amd64_att; break; case FSM_PRINT_AMD64_GO: print_vm = fsm_print_amd64_go; break; case FSM_PRINT_AMD64_NASM: print_vm = fsm_print_amd64_nasm; break; @@ -170,18 +305,32 @@ fsm_print(FILE *f, const struct fsm *fsm, return -1; } - ir = NULL; + r = 0; if (print_fsm != NULL) { r = print_fsm(f, opt, hooks, fsm); goto done; } + /* DFA only after this point */ + ir = make_ir(fsm, opt); if (ir == NULL) { return -1; } + if (opt->ambig == AMBIG_ERROR) { + size_t i; + + for (i = 0; i < ir->n; i++) { + if (!ir->states[i].isend) { + continue; + } + + assert(ir->states[i].endids.count <= 1); + } + } + if (print_ir != NULL) { r = print_ir(f, opt, hooks, ir); goto done; @@ -211,7 +360,27 @@ fsm_print(FILE *f, const struct fsm *fsm, return -1; } - return r; + if (r < 0) { + goto error; + } + + return 0; + +conflict: + + /* + * Finding an ambiguity conflict is not an error; + * we successfully identified AMBIG_ERROR. + * + * Success is >= 0 for a print function, we're + * distinguishing this from done: just for the caller. + */ + + return 1; + +error: + + return -1; } void diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 993eb4440..c45fcec53 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -39,6 +39,13 @@ print_hook_reject(FILE *f, void *lang_opaque, void *hook_opaque), void *lang_opaque); +int +print_hook_conflict(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const fsm_end_id_t *ids, size_t count, + const char *example); + typedef int fsm_print_f(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 083604ff9..457716dcc 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -431,6 +431,80 @@ make_state(const struct fsm *fsm, fsm_state_t state, return 0; } +int +make_example(const struct fsm *fsm, fsm_state_t s, char **example) +{ + fsm_state_t start; + size_t count; + char *p; + int n; + + assert(fsm != NULL); + assert(example != NULL); + + if (!fsm_getstart(fsm, &start)) { + *example = NULL; + return 1; + } + + count = fsm_countstates(fsm); + + /* + * Example lengths are approximately proportional to the number of states, + * and shorter where the graph branches often. + */ + p = f_malloc(fsm->alloc, count + 3 + 1); + if (p == NULL) { + return 0; + } + + /* + * fsm_example() doesn't distinguish an unreachable state from n = 0 + */ + if (s == start) { + p[0] = '\0'; + goto done; + } + + n = fsm_example(fsm, s, p, count + 1); + if (-1 == n) { + goto error; + } + + /* + * The goal state is always reachable for a DFA with + * no "stray" states, but the caller may not trim an FSM, + * and need not pass a DFA. + */ + if (n == 0) { + f_free(fsm->alloc, p); + p = NULL; + } else if ((size_t) n < count + 1) { + char *tmp; + + tmp = f_realloc(fsm->alloc, p, strlen(p) + 1); + if (tmp == NULL) { + goto error; + } + + p = tmp; + } else if ((size_t) n > count + 1) { + strcpy(p + count, "..."); + } + +done: + + *example = p; + + return 1; + +error: + + f_free(fsm->alloc, p); + + return 0; +} + struct ir * make_ir(const struct fsm *fsm, const struct fsm_options *opt) { @@ -500,54 +574,13 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) if (!opt->comments) { ir->states[i].example = NULL; } else { - char *p; - int n; - - if (i == start) { - ir->states[i].example = NULL; - continue; - } + char *example; - /* - * Example lengths are approximately proportional - * to the number of states in an fsm, and shorter where - * the graph branches often. - */ - p = f_malloc(fsm->alloc, ir->n + 3 + 1); - if (p == NULL) { + if (!make_example(fsm, i, &example)) { goto error_example; } - n = fsm_example(fsm, i, p, ir->n + 1); - if (-1 == n) { - f_free(fsm->alloc, p); - goto error_example; - } - - /* - * The goal state is always reachable for a DFA with - * no "stray" states, but the caller may not trim an FSM. - */ - if (n == 0) { - f_free(fsm->alloc, p); - p = NULL; - } else if ((size_t) n < ir->n + 1) { - char *tmp; - - n = strlen(p); - - tmp = f_realloc(fsm->alloc, p, n + 1); - if (tmp == NULL) { - f_free(fsm->alloc, p); - goto error_example; - } - - p = tmp; - } else if ((size_t) n > ir->n + 1) { - strcpy(p + ir->n, "..."); - } - - ir->states[i].example = p; + ir->states[i].example = example; } } @@ -577,7 +610,6 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) error_example: - ir->states[i].example = NULL; i++; error: diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index 02ab9a731..8be4dfc5e 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -105,6 +105,10 @@ struct ir { struct ir_state *states; /* array */ }; +/* caller frees */ +int +make_example(const struct fsm *fsm, fsm_state_t s, char **example); + /* TODO: can pass in mask of allowed strategies */ struct ir * make_ir(const struct fsm *fsm, const struct fsm_options *opt); diff --git a/src/lx/ast.c b/src/lx/ast.c index 58646ca68..a3776b644 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -125,7 +125,6 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, m->token = token; m->to = to; - m->conflict = NULL; m->next = z->ml; z->ml = m; @@ -141,39 +140,6 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, return m; } -/* TODO: merge with adt/set.c */ -struct mapping_set * -ast_addconflict(struct mapping_set **head, struct ast_mapping *m) -{ - struct mapping_set *new; - - assert(head != NULL); - assert(m != NULL); - - /* TODO: explain we find duplicate; return success */ - { - struct mapping_set *s; - - for (s = *head; s != NULL; s = s->next) { - if (s->m == m) { - return s; - } - } - } - - new = malloc(sizeof *new); - if (new == NULL) { - return NULL; - } - - new->m = m; - - new->next = *head; - *head = new; - - return new; -} - #define DEF_MAPPINGS 4 static fsm_end_id_t mapping_count = 0; static fsm_end_id_t mapping_ceil = DEF_MAPPINGS; diff --git a/src/lx/ast.h b/src/lx/ast.h index 097e43841..1f7fcf52f 100644 --- a/src/lx/ast.h +++ b/src/lx/ast.h @@ -14,7 +14,6 @@ struct ast_mapping { struct fsm *fsm; struct ast_token *token; struct ast_zone *to; - struct mapping_set *conflict; struct ast_mapping *next; }; @@ -34,11 +33,6 @@ struct ast_token { struct ast_token *next; }; -struct mapping_set { - struct ast_mapping *m; - struct mapping_set *next; -}; - struct ast { struct ast_zone *zl; struct ast_token *tl; @@ -59,9 +53,6 @@ struct ast_mapping * ast_addmapping(struct ast_zone *z, struct fsm *fsm, struct ast_token *token, struct ast_zone *to); -struct mapping_set * -ast_addconflict(struct mapping_set **head, struct ast_mapping *n); - int ast_setendmapping(struct fsm *fsm, struct ast_mapping *m); diff --git a/src/lx/main.c b/src/lx/main.c index fb7a176f4..f7a3fb5a5 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -328,12 +329,6 @@ zone_equal(const struct ast_zone *a, const struct ast_zone *b) fprintf(stderr, "zone_equal: asserting ast_getendmapping(q, state %d) != NULL: %p\n", i, (void *)m); } assert(m != NULL); - - if (m->conflict != NULL) { - /* TODO: free conflict set */ - fsm_free(q); - return 0; - } } } @@ -526,6 +521,73 @@ run_threads(int concurrency, void *(*fn)(void *)) return 0; } +static int +conflict(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, const char *example, + void *hook_opaque) +{ + const struct ast *ast; + size_t i; + + assert(opt != NULL); + assert(opt->ambig == AMBIG_ERROR); + assert(count > 1); + + (void) f; + (void) opt; + (void) hook_opaque; + + ast = hook_opaque; + + /* TODO: // delimeters depend on dialect */ + /* TODO: would deal with dialect: prefix here, too */ + + fprintf(stderr, "ambiguous mappings to "); + + for (i = 0; i < count; i++) { + const struct ast_mapping *m; + + /* + * When the example is '', we have two patterns which match the empty string. + * Here we defer to the error about the start state accepting, + * and it seems redundant to also show an error about both patterns + * matching the same input, even if there's a non-empty part. + */ + if (example != NULL && strlen(example) == 0) { + continue; + } + + m = ast_getendmappingbyendid(ids[i]); + + if (m->token != NULL) { + fprintf(stderr, "$%s", m->token->s); + } else if (m->to == NULL) { + fprintf(stderr, "skip"); + } + if (m->token != NULL && m->to != NULL) { + fprintf(stderr, "/"); + } + if (m->to == ast->global) { + fprintf(stderr, "global zone"); + } else if (m->to != NULL) { + fprintf(stderr, "z%p", (void *) m->to); /* TODO: zindexof(n->to) */ + } + + if (i + 1 < count) { + fprintf(stderr, ", "); + } + } + + if (example != NULL) { + /* TODO: escape hex etc */ + fprintf(stderr, "; for example on input '%s'", example); + } + + fprintf(stderr, "\n"); + + return 0; +} + int main(int argc, char *argv[]) { @@ -538,7 +600,6 @@ main(int argc, char *argv[]) print_progress = 0; concurrency = 1; - /* TODO: populate options */ opt.anonymous_states = 1; opt.consolidate_edges = 1; opt.comments = 1; @@ -798,7 +859,6 @@ main(int argc, char *argv[]) if (lang != LX_PRINT_H) { struct ast_zone *z; unsigned int zn; - fsm_state_t i; int e; if (print_progress) { @@ -829,6 +889,18 @@ main(int argc, char *argv[]) zn++; } + switch (lang) { + case LX_PRINT_NONE: + case LX_PRINT_C: + case LX_PRINT_H: + opt.ambig = AMBIG_ERROR; + break; + + default: + opt.ambig = AMBIG_MULTIPLE; + continue; + } + (void) fsm_getstart(z->fsm, &start); if (fsm_isend(z->fsm, start)) { @@ -836,68 +908,26 @@ main(int argc, char *argv[]) e = 1; } - /* pick up conflicts flagged by carryopaque() */ - for (i = 0; i < z->fsm->statecount; i++) { - struct ast_mapping *m; - - if (!fsm_isend(z->fsm, i)) { - continue; - } - - m = ast_getendmapping(z->fsm, i); - if (LOG()) { - fprintf(stderr, "main: m <- ast_getendmapping(dst_fsm, i]: %d) = %p // pick up conflicts\n", i, (void *)m); - } - assert(m != NULL); - - if (m->conflict != NULL) { - struct mapping_set *p; - char buf[50]; /* 50 looks reasonable for an on-screen limit */ - int n; + { + static const struct fsm_hooks zero_hooks; + struct fsm_hooks hooks = zero_hooks; + int r; - n = fsm_example(z->fsm, i, buf, sizeof buf); - if (-1 == n) { - perror("fsm_example"); - return EXIT_FAILURE; - } - - /* - * When n == 0, we have two patterns which match the empty string. - * Here we defer to the error about the start state accepting, - * and it seems redundant to also show an error about both patterns - * matching the same input, even if there's a non-empty part. - */ - if (n > 0) { - fprintf(stderr, "ambiguous mappings to "); - - for (p = m->conflict; p != NULL; p = p->next) { - if (p->m->token != NULL) { - fprintf(stderr, "$%s", p->m->token->s); - } else if (p->m->to == NULL) { - fprintf(stderr, "skip"); - } - if (p->m->token != NULL && p->m->to != NULL) { - fprintf(stderr, "/"); - } - if (p->m->to == ast->global) { - fprintf(stderr, "global zone"); - } else if (p->m->to != NULL) { - fprintf(stderr, "z%p", (void *) p->m->to); /* TODO: zindexof(n->to) */ - } + hooks.conflict = conflict; + hooks.hook_opaque = ast; - if (p->next != NULL) { - fprintf(stderr, ", "); - } - } + r = fsm_print(stdout, z->fsm, &opt, &hooks, FSM_PRINT_NONE); - /* TODO: escape hex etc */ - fprintf(stderr, "; for example on input '%s%s'\n", buf, - n >= (int) sizeof buf - 1 ? "..." : ""); + if (r < 0) { + exit(EXIT_FAILURE); + } - e = 1; - } + if (r == 1) { + /* conflict */ + exit(EXIT_FAILURE); } } + } if (print_progress) { @@ -932,9 +962,6 @@ main(int argc, char *argv[]) } if (m != NULL) { assert(m->fsm == NULL); - - /* TODO: free m->conflict, allocated in carryopaque */ - (void) m->conflict; } } } diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index bf1a72063..0369cfd33 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -83,30 +83,30 @@ accept_dot(FILE *f, const struct fsm_options *opt, ast = hook_opaque; s = * (fsm_state_t *) lang_opaque; - m = ast_getendmappingbyendid(ids[0]); - fprintf(f, "label = <"); if (!anonymous_states) { fprintf(f, "%u
", s); } - if (m->conflict != NULL) { - const struct mapping_set *p; + if (count == 1) { + m = ast_getendmappingbyendid(ids[0]); + mapping(f, m, ast); + } else { + size_t i; fprintf(f, ""); - for (p = m->conflict; p != NULL; p = p->next) { - mapping(f, p->m, ast); + for (i = 0; i < count; i++) { + m = ast_getendmappingbyendid(ids[i]); + mapping(f, m, ast); - if (p->next != NULL) { + if (i + 1 < count) { fprintf(f, "
"); } } fprintf(f, ""); - } else { - mapping(f, m, ast); } fprintf(f, ">"); @@ -136,30 +136,15 @@ singlestate(FILE *f, const struct fsm *fsm, const struct ast *ast, } assert(m != NULL); - if (m->conflict != NULL) { - const struct mapping_set *p; - - for (p = m->conflict; p != NULL; p = p->next) { - if (p->m->to != NULL) { - fsm_state_t start; - - (void) fsm_getstart(p->m->to->fsm, &start); - - fprintf(f, "\tz%uS%u -> z%uS%u [ color = red, style = dashed ];\n", - zindexof(ast, z), s, - zindexof(ast, p->m->to), start); - } - } - } else { - if (m->to != NULL) { - fsm_state_t start; + if (m->to != NULL) { + fsm_state_t start; - (void) fsm_getstart(m->to->fsm, &start); + (void) fsm_getstart(m->to->fsm, &start); - fprintf(f, "\tz%uS%u -> z%uS%u [ color = cornflowerblue, style = dashed ];\n", - zindexof(ast, z), s, - zindexof(ast, m->to), start); - } + fprintf(f, "\tz%uS%u -> z%uS%u [ color = %s, style = dashed ];\n", + zindexof(ast, z), s, + zindexof(ast, m->to), start, + fsm_endid_count(fsm, s) > 1 ? "red" : "cornflowerblue"); } } @@ -199,6 +184,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, (void) sprintf(p, "z%u", zindexof(ast, z)); o.prefix = p; + o.fragment = 1; hooks.accept = accept_dot; hooks.hook_opaque = (void *) ast; diff --git a/src/re/main.c b/src/re/main.c index e196103d4..86258f318 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -49,11 +49,8 @@ * TODO: flags; -r for RE_REVERSE, etc */ -struct match { - fsm_end_id_t id; - const char *s; - struct match *next; -}; +static char *const *matchv; +static size_t matchc; static void usage(void) @@ -275,152 +272,65 @@ xopen(const char *s) return f; } -struct matches_list { - struct matches_list *next; - struct match *head; -}; - -static struct matches_list *all_matches = NULL; - -static struct match * -find_match_with_id(fsm_end_id_t id) -{ - struct matches_list *res = all_matches; - while (res != NULL) { - struct match *m = res->head; - if (m->id == id) { - return m; - } - res = res->next; - } - return NULL; -} - -static struct match * -find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) -{ - fsm_end_id_t *ids, id; - size_t count; - int res; - - if (!fsm_isend(dfa, s)) { - return NULL; - } - - count = fsm_endid_count(dfa, s); - if (count == 0) { - return NULL; - } - - ids = xmalloc(count * sizeof *ids); - - res = fsm_endid_get(dfa, s, count, ids); - assert(res == 1); - - id = ids[0]; - - free(ids); - - return find_match_with_id(id); -} - -static struct match * -addmatch(struct match **head, fsm_end_id_t id, const char *s) +static void +printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) { - struct match *new; + char buf[256]; /* TODO */ + int n; - assert(head != NULL); - assert(s != NULL); + assert(f != NULL); + assert(fsm != NULL); - if ((1U << id) > INT_MAX) { - fprintf(stderr, "Too many patterns for int bitmap\n"); + n = fsm_example(fsm, state, buf, sizeof buf); + if (-1 == n) { + perror("fsm_example"); exit(EXIT_FAILURE); } - /* TODO: explain we find duplicate; return success */ - /* - * This is a purposefully shallow comparison (rather than calling strcmp) - * so that 're xyz xyz' will find the ambiguity. - */ - { - struct match *p; - - for (p = *head; p != NULL; p = p->next) { - if (p->s == s) { - return p; - } - } - } - - new = xmalloc(sizeof *new); - - new->id = id; - new->s = s; - - new->next = *head; - *head = new; - - return new; + /* TODO: escape hex etc */ + fprintf(f, "%s%s", buf, + n >= (int) sizeof buf - 1 ? "..." : ""); } -static void -add_matches_list(struct match *head) +static int +conflict(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, const char *example, + void *hook_opaque) { - struct matches_list *new; - - if (head == NULL) { - return; - } + size_t i; - new = xmalloc(sizeof *new); + assert(opt != NULL); + assert(opt->ambig == AMBIG_ERROR); + assert(count > 1); - new->next = all_matches; - new->head = head; - all_matches = new; -} + (void) f; + (void) opt; + (void) hook_opaque; -static void -free_all_matches(void) -{ - struct matches_list *clst; - struct matches_list *nlst; + /* TODO: // delimeters depend on dialect */ + /* TODO: would deal with dialect: prefix here, too */ - for (clst = all_matches; clst != NULL; clst = nlst) { - struct match *curr; - struct match *next; + fprintf(stderr, "ambiguous matches for "); - nlst = clst->next; + for (i = 0; i < count; i++) { + assert(ids[i] < matchc); - for (curr = clst->head; curr != NULL; curr = next) { - next = curr->next; + /* TODO: print nicely */ + fprintf(stderr, "/%s/", matchv[ids[i]]); - free(curr); + if (i + 1 < count) { + fprintf(stderr, ", "); } - - free(clst); } - all_matches = NULL; -} - -static void -printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) -{ - char buf[256]; /* TODO */ - int n; - - assert(f != NULL); - assert(fsm != NULL); - - n = fsm_example(fsm, state, buf, sizeof buf); - if (-1 == n) { - perror("fsm_example"); - exit(EXIT_FAILURE); + if (example != NULL) { + /* TODO: escape hex etc. or perhaps make_ir() did that for us */ + fprintf(stderr, "; for example on input '%s'", example); } - /* TODO: escape hex etc */ - fprintf(f, "%s%s", buf, - n >= (int) sizeof buf - 1 ? "..." : ""); + fprintf(stderr, "\n"); + + return 0; } static int @@ -428,7 +338,6 @@ accept_c(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const struct match *m; unsigned n; size_t i; @@ -443,9 +352,7 @@ accept_c(FILE *f, const struct fsm_options *opt, n = 0; for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - n |= 1U << m->id; - } + n |= 1U << ids[i]; } fprintf(f, "return %#x;", (unsigned) n); @@ -453,12 +360,12 @@ accept_c(FILE *f, const struct fsm_options *opt, fprintf(f, " /* "); for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + assert(ids[i] < matchc); - if (m->next != NULL || i < count - 1) { - fprintf(f, ", "); - } + fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + + if (i + 1 < count) { + fprintf(f, ", "); } } @@ -472,7 +379,6 @@ accept_rust(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const struct match *m; unsigned n; size_t i; @@ -487,9 +393,7 @@ accept_rust(FILE *f, const struct fsm_options *opt, n = 0; for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - n |= 1U << m->id; - } + n |= 1U << ids[i]; } fprintf(f, "return Some(%#x)", (unsigned) n); @@ -497,12 +401,12 @@ accept_rust(FILE *f, const struct fsm_options *opt, fprintf(f, " /* "); for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + assert(ids[i] < matchc); - if (m->next != NULL || i < count - 1) { - fprintf(f, ", "); - } + fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + + if (i + 1 < count) { + fprintf(f, ", "); } } @@ -516,7 +420,6 @@ accept_llvm(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const struct match *m; unsigned n; size_t i; @@ -530,9 +433,7 @@ accept_llvm(FILE *f, const struct fsm_options *opt, n = 0; for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - n |= 1U << m->id; - } + n |= 1U << ids[i]; } i = * (const size_t *) lang_opaque; @@ -542,12 +443,12 @@ accept_llvm(FILE *f, const struct fsm_options *opt, fprintf(f, " ; "); for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + assert(ids[i] < matchc); - if (m->next != NULL || i < count - 1) { - fprintf(f, ", "); - } + fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + + if (i + 1 < count) { + fprintf(f, ", "); } } @@ -561,7 +462,6 @@ accept_dot(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const struct match *m; fsm_state_t s; size_t i; @@ -585,12 +485,12 @@ accept_dot(FILE *f, const struct fsm_options *opt, } for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "#%u", m->id); + assert(ids[i] < matchc); - if (m->next != NULL || i < count - 1) { - fprintf(f, ","); - } + fprintf(f, "#%u", ids[i]); + + if (i + 1 < count) { + fprintf(f, ","); } } @@ -598,19 +498,19 @@ accept_dot(FILE *f, const struct fsm_options *opt, /* TODO: centralise to libfsm/print/dot.c */ if (opt->comments) { - fprintf(f, " # "); + fprintf(f, " /* "); for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + assert(ids[i] < matchc); - if (m->next != NULL || i < count - 1) { - fprintf(f, ", "); - } + fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + + if (i + 1 < count) { + fprintf(f, ", "); } } - fprintf(f, "\n"); + fprintf(f, " */"); } return 0; @@ -621,7 +521,6 @@ accept_json(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const struct match *m; size_t i; assert(opt != NULL); @@ -635,12 +534,12 @@ accept_json(FILE *f, const struct fsm_options *opt, fprintf(f, "[ "); for (i = 0; i < count; i++) { - for (m = find_match_with_id(ids[i]); m != NULL; m = m->next) { - fprintf(f, "%u", m->id); + assert(ids[i] < matchc); - if (m->next != NULL) { - fprintf(f, ", "); - } + fprintf(f, "%u", ids[i]); + + if (i + 1 < count) { + fprintf(f, ", "); } } @@ -658,8 +557,6 @@ do_fsm_cleanup(void) fsm_free(fsm_to_cleanup); fsm_to_cleanup = NULL; } - - free_all_matches(); } static void @@ -715,7 +612,6 @@ main(int argc, char *argv[]) int isliteral; int keep_nfa; int patterns; - int ambig; int makevm; struct fsm_dfavm *vm; @@ -729,6 +625,7 @@ main(int argc, char *argv[]) opt.anonymous_states = 1; opt.consolidate_edges = 1; + opt.ambig = AMBIG_ERROR; opt.comments = 1; opt.io = FSM_IO_GETC; @@ -740,7 +637,6 @@ main(int argc, char *argv[]) isliteral = 0; keep_nfa = 0; patterns = 0; - ambig = 0; makevm = 0; fsm_lang = FSM_PRINT_NONE; ast_lang = AST_PRINT_NONE; @@ -781,8 +677,8 @@ main(int argc, char *argv[]) case 'p': fsm_lang = FSM_PRINT_FSM; break; case 'q': query = comparison(optarg); break; case 'r': dialect = dialect_name(optarg); break; + case 'u': opt.ambig = AMBIG_MULTIPLE; break; - case 'u': ambig = 1; break; case 'f': fsmfiles = 1; break; case 'x': xfiles = 1; break; case 'y': yfiles = 1; break; @@ -855,7 +751,7 @@ main(int argc, char *argv[]) } if (keep_nfa) { - ambig = 1; + opt.ambig = AMBIG_MULTIPLE; } /* XXX: repetitive */ @@ -1079,26 +975,13 @@ main(int argc, char *argv[]) } } - { - struct match *matches = NULL; - - /* - * Attach this mapping to each end state for this regexp. - * XXX: we should share the same matches struct for all end states - * in the same regexp, and keep an argc-sized array of pointers to free(). - */ - if (!fsm_setendid(new, i)) { - perror("fsm_setendid"); - return EXIT_FAILURE; - } - - if (!addmatch(&matches, i, argv[i])) { - perror("addmatch"); - return EXIT_FAILURE; - } - - add_matches_list(matches); - + /* + * Attach this mapping to each end state for this regexp. + * We pick this up from matchv[] indexed by the end id. + */ + if (!fsm_setendid(new, i)) { + perror("fsm_setendid"); + return EXIT_FAILURE; } /* TODO: implement concatenating patterns for -s in conjunction with -z. @@ -1137,6 +1020,9 @@ main(int argc, char *argv[]) } } + matchv = argv; + matchc = i; + argc -= i; argv += i; } @@ -1150,9 +1036,11 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if (!ambig) { + hooks.conflict = conflict; + + if ((opt.ambig | AMBIG_SINGLE)) { struct fsm *dfa; - size_t s; + int r; dfa = fsm_clone(fsm); if (dfa == NULL) { @@ -1160,54 +1048,37 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - { - if (!fsm_determinise(dfa)) { - perror("fsm_determinise"); - return EXIT_FAILURE; - } + if (!fsm_determinise(dfa)) { + perror("fsm_determinise"); + return EXIT_FAILURE; } - for (s = 0; s < dfa->statecount; s++) { - const struct match *matches; - matches = find_first_match_for_end_state(dfa, s); - if (matches == NULL) { - continue; - } - - assert(matches != NULL); - if (matches->next != NULL) { - const struct match *m; - - /* TODO: // delimeters depend on dialect */ - /* TODO: would deal with dialect: prefix here, too */ - - fprintf(stderr, "ambiguous matches for "); - for (m = matches; m != NULL; m = m->next) { - /* TODO: print nicely */ - fprintf(stderr, "/%s/", m->s); - if (m->next != NULL) { - fprintf(stderr, ", "); - } - } + /* + * This is silly, we're doing this just to find conflicts, + * because otherwise we won't find them during print, + * because in re(1) we don't always print. + * + * I don't like that we have two places where we call fsm_print(). + */ + // XXX: stdout is a non-NULL placeholder, would prefer f=NULL because FSM_PRINT_NONE + r = fsm_print(stdout, dfa, &opt, &hooks, FSM_PRINT_NONE); - fprintf(stderr, "; for example on input '"); - printexample(stderr, dfa, s); - fprintf(stderr, "'\n"); + fsm_free(dfa); - /* TODO: consider different error codes */ - return EXIT_FAILURE; - } + if (r < 0) { + exit(EXIT_FAILURE); } - /* TODO: free opaques */ - - fsm_free(dfa); + if (r == 1) { + exit(EXIT_FAILURE); + } } if (!keep_nfa) { /* * Convert to a DFA, then minimise unless we need to keep the end * state information separated per regexp. */ +// TODO: redundant wrt above if (!fsm_determinise(fsm)) { perror("fsm_determinise"); return EXIT_FAILURE; @@ -1235,19 +1106,30 @@ main(int argc, char *argv[]) /* TODO: would deal with dialect: prefix here, too */ if (patterns) { - const struct match *m; - m = find_first_match_for_end_state(fsm, s); - - while (m != NULL) { - /* TODO: print nicely */ - printf("/%s/", m->s); - if (m->next != NULL) { - printf(", "); - } - m = m->next; + fsm_end_id_t *ids; + size_t count, i; + + count = fsm_endid_count(fsm, s); + if (count > 0) { + int res; + + ids = xmalloc(count * sizeof *ids); + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); + + for (i = 0; i < count; i++) { + /* TODO: print nicely */ + printf("/%s/", matchv[ids[i]]); + if (i + 1 < count) { + printf(", "); + } } - printf(": "); + free(ids); + } + + printf(": "); } printexample(stdout, fsm, s); @@ -1268,6 +1150,9 @@ main(int argc, char *argv[]) * patterns in comments for the whole FSM */ switch (fsm_lang) { + case FSM_PRINT_NONE: + break; + case FSM_PRINT_C: case FSM_PRINT_VMC: hooks.accept = accept_c; @@ -1291,7 +1176,6 @@ main(int argc, char *argv[]) break; default: - hooks.accept = NULL; break; } @@ -1360,12 +1244,19 @@ main(int argc, char *argv[]) } if (patterns) { - const struct match *m; - m = find_first_match_for_end_state(fsm, state); - while (m != NULL) { + fsm_end_id_t ids[1]; + size_t count, i; + int res; + + count = fsm_endid_count(fsm, state); + assert(count == 1); + + res = fsm_endid_get(fsm, state, count, ids); + assert(res == 1); + + for (i = 0; i < count; i++) { /* TODO: print nicely */ - printf("match: /%s/\n", m->s); - m = m->next; + printf("match: /%s/\n", matchv[ids[i]]); } } } From e03de6b29af0f39e4695519f066a3bedcb844e40 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 26 Jul 2024 17:19:27 +0100 Subject: [PATCH 211/373] Clarification. --- include/fsm/options.h | 21 ++++++++++++++++++++- include/fsm/print.h | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/fsm/options.h b/include/fsm/options.h index b5e81fec2..c0d688888 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -13,7 +13,25 @@ enum fsm_io { FSM_IO_PAIR }; -// TODO: comment +/* + * Ambiguity mode. + * + * The ambiguity mode controls how endids are emitted into generated code, + * and passed to the hooks.accept() callback for an accepting state. + * + * This also causes fsm_print() to resolve endid conflicts in various ways + * for accepting states that have multiple endids present: + * + * AMBIG_NONE - Do not emit endids. + * AMBIG_ERROR - Emit a single endid. + * hook.conflict() is called for accepting states + * which have multiple endids, and printing errors. + * AMBIG_EARLIEST - Emit the lowest numbered endid only. + * AMBIG_MULTIPLE - Emit all endids. + * + * What it means to emit multiple or single endids is particular to each + * output language; not every language supports both. + */ enum fsm_ambig { AMBIG_NONE = 0, /* default */ AMBIG_ERROR = 1 << 0, @@ -25,6 +43,7 @@ enum fsm_ambig { /* * Print options. + * * This is separate to because we also borrow it for libre. */ struct fsm_options { diff --git a/include/fsm/print.h b/include/fsm/print.h index 9add9ba4d..453d3d915 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -72,6 +72,7 @@ struct fsm_hooks { int (*reject)(FILE *, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque); + /* only called for AMBIG_ERROR; see opt.ambig */ int (*conflict)(FILE *, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, const char *example, From a564db1d2f86bc6d9b4b8cb781a758f30db5bfd2 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 26 Jul 2024 23:02:14 +0100 Subject: [PATCH 212/373] -s for overriding dialect by file extension. --- src/rx/main.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 9d43b9bcd..161669809 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -995,7 +995,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQquvx] [-C charset] [-k io] [-l ] [-p prefix] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); + printf("usage: %s: [-ciQqsuvx] [-C charset] [-k io] [-l ] [-p prefix] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); printf(" %s -h\n", name); } @@ -1006,6 +1006,7 @@ main(int argc, char *argv[]) bool strict = false; bool verbose = false; bool show_stats = false; + bool override_dialect = false; bool unanchored_literals = false; enum ambig ambig = AMBIG_ERROR; const char *lang = "c"; @@ -1048,7 +1049,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:ikF:l:n:p:r:R:Qquvx"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:ikF:l:n:p:r:sR:Qquvx"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -1122,6 +1123,11 @@ main(int argc, char *argv[]) default_dialect = dialect_name(optarg); break; + case 's': + /* override dialect by file extension */ + override_dialect = true; + break; + case 'Q': /* one-off stats on resource consumption */ show_stats = true; @@ -1228,12 +1234,14 @@ main(int argc, char *argv[]) char *s; enum re_dialect dialect = default_dialect; - const char *ext = strrchr(argv[arg], '.'); - if (ext != NULL) { - ext++; - fprintf(stderr, "overriding dialect by extension for %s: %s\n", - argv[arg], ext); - dialect = dialect_name(ext); + if (override_dialect) { + const char *ext = strrchr(argv[arg], '.'); + if (ext != NULL) { + ext++; + fprintf(stderr, "overriding dialect by extension for %s: %s\n", + argv[arg], ext); + dialect = dialect_name(ext); + } } f = xopen(argv[arg]); From 18b2d73c0971016e8db1c4d05938754a9da408bd Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 27 Jul 2024 01:40:56 +0100 Subject: [PATCH 213/373] -e and -E for prefixes. --- src/re/main.c | 2 +- src/rx/main.c | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/re/main.c b/src/re/main.c index ea2c68ae8..7df763a6a 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -63,7 +63,7 @@ usage(void) { fprintf(stderr, "usage: re [-r ] [-nbiusfyz] [-x] ... [ | -- ... ]\n"); fprintf(stderr, " re [-r ] [-nbiusfyz] {-q } ...\n"); - fprintf(stderr, " re -p [-r ] [-nbiusfyz] [-l ] [-acwX] [-k ] [-e ] ...\n"); + fprintf(stderr, " re -p [-r ] [-nbiusfyz] [-l ] [-acwX] [-k ] [-E ] [-e ] ...\n"); fprintf(stderr, " re -m [-r ] [-nbiusfyz] ...\n"); fprintf(stderr, " re -G [-r ] [-biu] \n"); fprintf(stderr, " re -h\n"); diff --git a/src/rx/main.c b/src/rx/main.c index 161669809..e3037995c 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -995,7 +995,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQqsuvx] [-C charset] [-k io] [-l ] [-p prefix] [-r dialect] [-R reject] [-d declined-file] input-file...\n", name); + printf("usage: %s: [-ciQqsuvx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); printf(" %s -h\n", name); } @@ -1049,7 +1049,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:ikF:l:n:p:r:sR:Qquvx"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:E:e:ikF:l:n:r:sR:Qquvx"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -1083,6 +1083,14 @@ main(int argc, char *argv[]) declined_file = optarg; break; + case 'E': + opt.package_prefix = optarg; + break; + + case 'e': + opt.prefix = optarg; + break; + case 'h': usage(name); exit(EXIT_SUCCESS); @@ -1107,10 +1115,6 @@ main(int argc, char *argv[]) general_limit = atoi(optarg); break; - case 'p': - opt.prefix = optarg; - break; - case 'R': reject = optarg; From 366fce67caff2308d07c12e311a2069271c9e1bd Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 27 Jul 2024 17:09:18 +0100 Subject: [PATCH 214/373] Move out retlist construction. --- src/libfsm/print/llvm.c | 147 +----------------------------------- src/libfsm/vm/Makefile | 1 + src/libfsm/vm/retlist.c | 161 ++++++++++++++++++++++++++++++++++++++++ src/libfsm/vm/retlist.h | 34 +++++++++ 4 files changed, 198 insertions(+), 145 deletions(-) create mode 100644 src/libfsm/vm/retlist.c create mode 100644 src/libfsm/vm/retlist.h diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index ac87090df..0992ce8ca 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -29,6 +29,7 @@ #include "libfsm/print.h" #include "libfsm/vm/vm.h" +#include "libfsm/vm/retlist.h" #define OPAQUE_POINTERS 1 @@ -44,16 +45,6 @@ static const char *ptr_void = "i8*"; static const struct dfavm_op_ir fail; // used as a unqiue address only -struct ret { - size_t count; - const fsm_end_id_t *ids; -}; - -struct ret_list { - size_t count; - struct ret *a; -}; - /* * If we had a stack, the current set of live values would be a frame. * We're a DFA, so we don't have a stack. But I still think of them as a frame. @@ -410,113 +401,6 @@ print_fetch(FILE *f, const struct fsm_options *opt, } } -static bool -append_ret(struct ret_list *list, - const fsm_end_id_t *ids, size_t count) -{ - const size_t low = 16; /* must be power of 2 */ - const size_t factor = 2; /* must be even */ - - assert(list != NULL); - - if (list->count == 0) { - list->a = malloc(low * sizeof *list->a); - if (list->a == NULL) { - return false; - } - } else if (list->count >= low && (list->count & (list->count - 1)) == 0) { - void *tmp; - size_t new = list->count * factor; - if (new < list->count) { - errno = E2BIG; - perror("realloc"); - exit(EXIT_FAILURE); - } - - tmp = realloc(list->a, new * sizeof *list->a); - if (tmp == NULL) { - return false; - } - - list->a = tmp; - } - - list->a[list->count].ids = ids; - list->a[list->count].count = count; - - list->count++; - - return true; -} - -static int -cmp_ret_by_endid(const void *pa, const void *pb) -{ - const struct ret *a = pa; - const struct ret *b = pb; - - if (a->count < b->count) { return -1; } - if (a->count > b->count) { return +1; } - - return memcmp(a->ids, b->ids, a->count * sizeof *a->ids); -} - -static struct ret * -find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, - int (*cmp)(const void *pa, const void *pb)) -{ - struct ret key; - - assert(op != NULL); - assert(cmp != NULL); - - key.count = op->endids.count; - key.ids = op->endids.ids; - - return bsearch(&key, list->a, list->count, sizeof *list->a, cmp); -} - -static bool -build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) -{ - const struct dfavm_op_ir *op; - - assert(list != NULL); - - for (op = a; op != NULL; op = op->next) { - switch (op->instr) { - case VM_OP_STOP: - if (op->u.stop.end_bits == VM_END_FAIL) { - /* %fail is special, don't add to retlist */ - continue; - } - - break; - - case VM_OP_FETCH: - if (op->u.fetch.end_bits == VM_END_FAIL) { - /* %fail is special, don't add to retlist */ - continue; - } - - break; - - case VM_OP_BRANCH: - continue; - - default: - assert(!"unreached"); - abort(); - } - - if (!append_ret(list, op->endids.ids, op->endids.count)) { - return false; - } - } - - return true; -} - /* TODO: eventually to be non-static */ static int fsm_print_llvmfrag(FILE *f, @@ -546,33 +430,8 @@ fsm_print_llvmfrag(FILE *f, } { - retlist.count = 0; build_retlist(&retlist, ops); - if (retlist.count > 0) { - size_t j = 0; - - /* sort for both dedup and bsearch */ - qsort(retlist.a, retlist.count, sizeof *retlist.a, cmp_ret_by_endid); - - /* deduplicate based on endids only. - * j is the start of a run; i increments until we find - * the start of the next run */ - for (size_t i = 1; i < retlist.count; i++) { - assert(i > j); - if (cmp_ret_by_endid(&retlist.a[j], &retlist.a[i]) == 0) { - continue; - } - - j++; - retlist.a[j] = retlist.a[i]; - } - - retlist.count = j + 1; - - assert(retlist.count > 0); - } - print_jump(f, ops); /* @@ -734,9 +593,7 @@ fsm_print_llvmfrag(FILE *f, } } - if (retlist.count > 0) { - free(retlist.a); - } + free_retlist(&retlist); return 0; } diff --git a/src/libfsm/vm/Makefile b/src/libfsm/vm/Makefile index fe54811b8..ce32cc585 100644 --- a/src/libfsm/vm/Makefile +++ b/src/libfsm/vm/Makefile @@ -4,6 +4,7 @@ SRC += src/libfsm/vm/ir.c SRC += src/libfsm/vm/vm.c SRC += src/libfsm/vm/v1.c SRC += src/libfsm/vm/v2.c +SRC += src/libfsm/vm/retlist.c .for src in ${SRC:Msrc/libfsm/vm/*.c} CFLAGS.${src} += -I src # XXX: for internal.h diff --git a/src/libfsm/vm/retlist.c b/src/libfsm/vm/retlist.c new file mode 100644 index 000000000..867c8b561 --- /dev/null +++ b/src/libfsm/vm/retlist.c @@ -0,0 +1,161 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "libfsm/vm/vm.h" +#include "libfsm/vm/retlist.h" + +static bool +append_ret(struct ret_list *list, + const fsm_end_id_t *ids, size_t count) +{ + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + + assert(list != NULL); + + // TODO: alloc callbacks + if (list->count == 0) { + list->a = malloc(low * sizeof *list->a); + if (list->a == NULL) { + return false; + } + } else if (list->count >= low && (list->count & (list->count - 1)) == 0) { + void *tmp; + size_t new = list->count * factor; + if (new < list->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + tmp = realloc(list->a, new * sizeof *list->a); + if (tmp == NULL) { + return false; + } + + list->a = tmp; + } + + list->a[list->count].ids = ids; + list->a[list->count].count = count; + + list->count++; + + return true; +} + +int +cmp_ret_by_endid(const void *pa, const void *pb) +{ + const struct ret *a = pa; + const struct ret *b = pb; + + if (a->count < b->count) { return -1; } + if (a->count > b->count) { return +1; } + + return memcmp(a->ids, b->ids, a->count * sizeof *a->ids); +} + +struct ret * +find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, + int (*cmp)(const void *pa, const void *pb)) +{ + struct ret key; + + assert(op != NULL); + assert(cmp != NULL); + + key.count = op->endids.count; + key.ids = op->endids.ids; + + return bsearch(&key, list->a, list->count, sizeof *list->a, cmp); +} + +bool +build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) +{ + const struct dfavm_op_ir *op; + + assert(list != NULL); + + list->count = 0; + + for (op = a; op != NULL; op = op->next) { + switch (op->instr) { + case VM_OP_STOP: + if (op->u.stop.end_bits == VM_END_FAIL) { + /* %fail is special, don't add to retlist */ + continue; + } + + break; + + case VM_OP_FETCH: + if (op->u.fetch.end_bits == VM_END_FAIL) { + /* %fail is special, don't add to retlist */ + continue; + } + + break; + + case VM_OP_BRANCH: + continue; + + default: + assert(!"unreached"); + abort(); + } + + if (!append_ret(list, op->endids.ids, op->endids.count)) { + return false; + } + } + + if (list->count > 0) { + size_t j = 0; + + /* sort for both dedup and bsearch */ + qsort(list->a, list->count, sizeof *list->a, cmp_ret_by_endid); + + /* deduplicate based on endids only. + * j is the start of a run; i increments until we find + * the start of the next run */ + for (size_t i = 1; i < list->count; i++) { + assert(i > j); + if (cmp_ret_by_endid(&list->a[j], &list->a[i]) == 0) { + continue; + } + + j++; + list->a[j] = list->a[i]; + } + + list->count = j + 1; + + assert(list->count > 0); + } + + return true; +} + +void +free_retlist(struct ret_list *list) +{ + if (list->count > 0) { + free(list->a); + } +} + diff --git a/src/libfsm/vm/retlist.h b/src/libfsm/vm/retlist.h new file mode 100644 index 000000000..168bd0f70 --- /dev/null +++ b/src/libfsm/vm/retlist.h @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#ifndef FSM_INTERNAL_RETLIST_H +#define FSM_INTERNAL_RETLIST_H + +struct ret { + size_t count; + const fsm_end_id_t *ids; +}; + +struct ret_list { + size_t count; + struct ret *a; +}; + +int +cmp_ret_by_endid(const void *pa, const void *pb); + +struct ret * +find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, + int (*cmp)(const void *pa, const void *pb)); + +bool +build_retlist(struct ret_list *list, const struct dfavm_op_ir *a); + +void +free_retlist(struct ret_list *list); + +#endif + From 684433c68bf227274facc3e55c165ee825102659 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 28 Jul 2024 10:31:54 +0100 Subject: [PATCH 215/373] Constuct retlist ahead of time for all vm-based formats. Now the VM IR ops point to retlist entries, rather than carrying their own endid sets. This means we can use the same indexing to de-dup them for every VM-based format. --- src/libfsm/print.c | 23 +++++++- src/libfsm/print.h | 2 + src/libfsm/print/awk.c | 11 +++- src/libfsm/print/go.c | 30 ++++++---- src/libfsm/print/llvm.c | 41 ++++++-------- src/libfsm/print/rust.c | 11 +++- src/libfsm/print/sh.c | 5 +- src/libfsm/print/vmasm.c | 14 +++-- src/libfsm/print/vmc.c | 11 +++- src/libfsm/print/vmdot.c | 11 +++- src/libfsm/print/vmops.c | 20 +++++-- src/libfsm/vm.c | 11 +++- src/libfsm/vm/ir.c | 119 +++++++++++++++++++++------------------ src/libfsm/vm/retlist.c | 57 ++++++------------- src/libfsm/vm/retlist.h | 10 ++-- src/libfsm/vm/vm.h | 15 ++--- 16 files changed, 219 insertions(+), 172 deletions(-) diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 6ff34e4ae..34c8863c5 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -18,6 +18,7 @@ #include "print.h" #include "internal.h" +#include "vm/retlist.h" #include "vm/vm.h" #include "print/ir.h" @@ -179,6 +180,7 @@ print_conflicts(FILE *f, const struct fsm *fsm, assert(res == 1); // TODO: de-duplicate by ids[], so we don't call the conflict hook an unneccessary number of times + // TODO: now i think this is the same as calling once per retlist entry /* * The conflict hook is called here (rather in the caller), @@ -336,20 +338,37 @@ fsm_print(FILE *f, const struct fsm *fsm, goto done; } + /* + * We're building the retlist here based on the ir. + * I think we could build the retlist earlier instead, + * and then point at the struct ret entries from the ir, + * and then dfavm_compile_ir() would pick those up from there. + * But for now this is good. + */ + struct ret_list retlist; + + if (!build_retlist(&retlist, ir)) { + free_ir(fsm, ir); + goto error; + } + a = zero; /* TODO: non-const a */ - if (!dfavm_compile_ir(&a, ir, vm_opts)) { + if (!dfavm_compile_ir(&a, ir, &retlist, vm_opts)) { + free_retlist(&retlist); free_ir(fsm, ir); return -1; } if (print_vm != NULL) { - r = print_vm(f, opt, hooks, a.linked); + r = print_vm(f, opt, hooks, &retlist, a.linked); } dfavm_opasm_finalize_op(&a); + free_retlist(&retlist); + done: if (ir != NULL) { diff --git a/src/libfsm/print.h b/src/libfsm/print.h index c45fcec53..8e52e8c41 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -12,6 +12,7 @@ struct fsm_options; struct fsm_hooks; struct ir; struct dfavm_op_ir; +struct ret_list; int print_hook_args(FILE *f, @@ -59,6 +60,7 @@ typedef int ir_print_f(FILE *f, typedef int vm_print_f(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops); vm_print_f fsm_print_amd64_att; diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index de98f95eb..410290319 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -26,6 +26,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" #define START UINT32_MAX @@ -157,7 +158,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_SUCC: return print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, default_accept, NULL); @@ -186,6 +187,7 @@ static int fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, const char *cp, const char *prefix) { @@ -194,6 +196,7 @@ fsm_print_awkfrag(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); assert(cp != NULL); assert(prefix != NULL); @@ -289,6 +292,7 @@ int fsm_print_awk(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { const char *prefix; @@ -297,6 +301,7 @@ fsm_print_awk(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -311,7 +316,7 @@ fsm_print_awk(FILE *f, } if (opt->fragment) { - if (-1 == fsm_print_awkfrag(f, opt, hooks, ops, cp, prefix)) { + if (-1 == fsm_print_awkfrag(f, opt, hooks, retlist, ops, cp, prefix)) { return -1; } } else { @@ -333,7 +338,7 @@ fsm_print_awk(FILE *f, fprintf(f, ", l, c) {\n"); - if (-1 == fsm_print_awkfrag(f, opt, hooks, ops, cp, prefix)) { + if (-1 == fsm_print_awkfrag(f, opt, hooks, retlist, ops, cp, prefix)) { return -1; } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 5a0ac9668..fecafddb1 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -25,6 +25,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" static const char * @@ -72,8 +73,10 @@ print_ids(FILE *f, break; case AMBIG_MULTIPLE: - assert(!"unimplemented"); - abort(); + assert(!"unreached"); +// TODO: fprintf(f, ", ret%u[0:%u];", TODO: retlist index, count); +// ids[] + break; default: assert(!"unreached"); @@ -161,9 +164,9 @@ print_end(FILE *f, const struct dfavm_op_ir *op, fprintf(f, "\t\t"); if (-1 == print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, - default_accept, - NULL)) + op->ret->ids, op->ret->count, + default_accept, NULL)) + // TODO: index for retlist { return -1; } @@ -203,6 +206,7 @@ static int fsm_print_gofrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, const char *cp) { @@ -210,6 +214,7 @@ fsm_print_gofrag(FILE *f, assert(f != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); /* TODO: we'll need to heed cp for e.g. lx's codegen */ @@ -300,6 +305,7 @@ int fsm_print_go(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { const char *prefix; @@ -311,6 +317,7 @@ fsm_print_go(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -325,7 +332,7 @@ fsm_print_go(FILE *f, } if (opt->fragment) { - if (-1 == fsm_print_gofrag(f, opt, hooks, ops, cp)) { + if (-1 == fsm_print_gofrag(f, opt, hooks, retlist, ops, cp)) { return -1; } } else { @@ -362,17 +369,16 @@ fsm_print_go(FILE *f, case AMBIG_NONE: fprintf(f, "bool"); break; - + case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(stdout, "(bool, uint)"); break; case AMBIG_MULTIPLE: - // TODO: fprintf(stdout, "(bool, uint[])"); - errno = ENOTSUP; - return -1; - + fprintf(stdout, "(bool, uint[])"); + break; + default: assert(!"unreached"); abort(); @@ -380,7 +386,7 @@ fsm_print_go(FILE *f, fprintf(f, " {\n"); - if (-1 == fsm_print_gofrag(f, opt, hooks, ops, cp)) { + if (-1 == fsm_print_gofrag(f, opt, hooks, retlist, ops, cp)) { return -1; } diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 0992ce8ca..575d76f96 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -28,8 +28,8 @@ #include "libfsm/internal.h" #include "libfsm/print.h" -#include "libfsm/vm/vm.h" #include "libfsm/vm/retlist.h" +#include "libfsm/vm/vm.h" #define OPAQUE_POINTERS 1 @@ -406,14 +406,15 @@ static int fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, const char *cp) { - struct ret_list retlist; struct dfavm_op_ir *op; assert(f != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); /* TODO: we'll need to heed cp for e.g. lx's codegen */ @@ -430,8 +431,6 @@ fsm_print_llvmfrag(FILE *f, } { - build_retlist(&retlist, ops); - print_jump(f, ops); /* @@ -475,11 +474,11 @@ fsm_print_llvmfrag(FILE *f, print_rettype(f, opt->ambig); fprintf(f, "\n"); - for (size_t i = 0; i < retlist.count; i++) { + for (size_t i = 0; i < retlist->count; i++) { fprintf(f, "\t "); if (-1 == print_hook_accept(f, opt, hooks, - retlist.a[i].ids, retlist.a[i].count, + retlist->a[i].ids, retlist->a[i].count, default_accept, &i)) { return -1; @@ -500,7 +499,7 @@ fsm_print_llvmfrag(FILE *f, print_label(f, false, "stop"); fprintf(f, "\n"); - for (size_t i = 0; i < retlist.count; i++) { + for (size_t i = 0; i < retlist->count; i++) { print_label(f, true, "ret%zu", i); fprintf(f, "\tbr "); print_label(f, false, "stop"); @@ -542,14 +541,12 @@ fsm_print_llvmfrag(FILE *f, if (op->u.stop.end_bits == VM_END_FAIL) { /* handled above */ } else { - assert(retlist.count > 0); - const struct ret *ret = find_ret(&retlist, op, cmp_ret_by_endid); + assert(retlist->count > 0); + const struct ret *ret = op->ret; assert(ret != NULL); - assert(ret >= retlist.a && ret <= (retlist.a + retlist.count)); - assert(ret->count == op->endids.count); - assert(0 == memcmp(ret->ids, op->endids.ids, ret->count)); + assert(ret >= retlist->a && ret <= (retlist->a + retlist->count)); fprintf(f, "\tbr "); - print_label(f, false, "ret%u", ret - retlist.a); + print_label(f, false, "ret%u", ret - retlist->a); fprintf(f, "\n"); } break; @@ -560,14 +557,12 @@ fsm_print_llvmfrag(FILE *f, if (op->u.fetch.end_bits == VM_END_FAIL) { /* handled in print_fetch() */ } else { - assert(retlist.count > 0); - const struct ret *ret = find_ret(&retlist, op, cmp_ret_by_endid); + assert(retlist->count > 0); + const struct ret *ret = op->ret; assert(ret != NULL); - assert(ret >= retlist.a && ret <= (retlist.a + retlist.count)); - assert(ret->count == op->endids.count); - assert(0 == memcmp(ret->ids, op->endids.ids, ret->count)); + assert(ret >= retlist->a && ret <= (retlist->a + retlist->count)); fprintf(f, "\tbr "); - print_label(f, false, "ret%u", ret - retlist.a); + print_label(f, false, "ret%u", ret - retlist->a); fprintf(f, "\n"); } break; @@ -593,8 +588,6 @@ fsm_print_llvmfrag(FILE *f, } } - free_retlist(&retlist); - return 0; } @@ -602,6 +595,7 @@ int fsm_print_llvm(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { const char *prefix; @@ -610,6 +604,7 @@ fsm_print_llvm(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -624,7 +619,7 @@ fsm_print_llvm(FILE *f, } if (opt->fragment) { - fsm_print_llvmfrag(f, opt, hooks, ops, cp); + fsm_print_llvmfrag(f, opt, hooks, retlist, ops, cp); return 0; } @@ -681,7 +676,7 @@ fsm_print_llvm(FILE *f, exit(EXIT_FAILURE); } - fsm_print_llvmfrag(f, opt, hooks, ops, cp); + fsm_print_llvmfrag(f, opt, hooks, retlist, ops, cp); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 5c35747d9..07b8f7c29 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -26,6 +26,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" #define START UINT32_MAX @@ -170,7 +171,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_SUCC: return print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, default_accept, NULL); @@ -205,6 +206,7 @@ static int fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, const char *cp) { @@ -213,6 +215,7 @@ fsm_print_rustfrag(FILE *f, assert(f != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); /* TODO: we'll need to heed cp for e.g. lx's codegen */ @@ -415,6 +418,7 @@ int fsm_print_rust(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { const char *prefix; @@ -423,6 +427,7 @@ fsm_print_rust(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -437,7 +442,7 @@ fsm_print_rust(FILE *f, } if (opt->fragment) { - fsm_print_rustfrag(f, opt, hooks, ops, cp); + fsm_print_rustfrag(f, opt, hooks, retlist, ops, cp); goto error; } @@ -471,7 +476,7 @@ fsm_print_rust(FILE *f, exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, opt, hooks, ops, cp); + fsm_print_rustfrag(f, opt, hooks, retlist, ops, cp); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 05692df59..5ebca2f85 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -26,6 +26,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" static const char * @@ -205,7 +206,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_SUCC: return print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, default_accept, NULL); @@ -233,6 +234,7 @@ int fsm_print_sh(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { struct dfavm_op_ir *op; @@ -240,6 +242,7 @@ fsm_print_sh(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->io != FSM_IO_STR) { errno = ENOTSUP; diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index c5ac87492..ecab3763a 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -24,6 +24,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" enum asm_dialect { @@ -50,7 +51,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_SUCC: if (-1 == print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, NULL, NULL)) { return -1; @@ -358,6 +359,7 @@ static int print_vmasm_encoding(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, enum asm_dialect dialect) { @@ -366,6 +368,7 @@ print_vmasm_encoding(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (dialect == AMD64_GO) { if (opt->io != FSM_IO_STR && opt->io != FSM_IO_PAIR) { @@ -392,26 +395,29 @@ int fsm_print_amd64_att(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, hooks, ops, AMD64_ATT); + return print_vmasm_encoding(f, opt, hooks, retlist, ops, AMD64_ATT); } int fsm_print_amd64_nasm(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, hooks, ops, AMD64_NASM); + return print_vmasm_encoding(f, opt, hooks, retlist, ops, AMD64_NASM); } int fsm_print_amd64_go(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, hooks, ops, AMD64_GO); + return print_vmasm_encoding(f, opt, hooks, retlist, ops, AMD64_GO); } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 206e333ae..ffe96b782 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -25,6 +25,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" static const char * @@ -176,7 +177,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_SUCC: return print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, default_accept, NULL); @@ -360,6 +361,7 @@ static int fsm_print_cfrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, const char *cp) { @@ -367,6 +369,7 @@ fsm_print_cfrag(FILE *f, assert(f != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); /* TODO: we'll need to heed cp for e.g. lx's codegen */ @@ -512,6 +515,7 @@ int fsm_print_vmc(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { const char *prefix; @@ -522,6 +526,7 @@ fsm_print_vmc(FILE *f, assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -530,7 +535,7 @@ fsm_print_vmc(FILE *f, } if (opt->fragment) { - if (-1 == fsm_print_cfrag(f, opt, hooks, ops, cp)) { + if (-1 == fsm_print_cfrag(f, opt, hooks, retlist, ops, cp)) { return -1; } } else { @@ -591,7 +596,7 @@ fsm_print_vmc(FILE *f, fprintf(f, ")\n"); fprintf(f, "{\n"); - if (-1 == fsm_print_cfrag(f, opt, hooks, ops, cp)) { + if (-1 == fsm_print_cfrag(f, opt, hooks, retlist, ops, cp)) { return -1; } diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 576d8c08d..e598c5da9 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -25,6 +25,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" static const char * @@ -129,7 +130,7 @@ print_end(FILE *f, case VM_END_SUCC: return print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, default_accept, NULL); @@ -321,10 +322,12 @@ static int fsm_print_vmdotfrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { assert(f != NULL); assert(opt != NULL); + assert(retlist != NULL); if (-1 == fsm_print_nodes(f, opt, hooks, ops)) { return -1; @@ -340,14 +343,16 @@ int fsm_print_vmdot(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); + assert(retlist != NULL); if (opt->fragment) { - if (-1 == fsm_print_vmdotfrag(f, opt, hooks, ops)) { + if (-1 == fsm_print_vmdotfrag(f, opt, hooks, retlist, ops)) { return -1; } } else { @@ -365,7 +370,7 @@ fsm_print_vmdot(FILE *f, fprintf(f, "\tstart [ shape = none, label = \"\" ];\n"); fprintf(f, "\tstart -> S0:i0:w [ style = bold ];\n"); - if (-1 == fsm_print_vmdotfrag(f, opt, hooks, ops)) { + if (-1 == fsm_print_vmdotfrag(f, opt, hooks, retlist, ops)) { return -1; } diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 7eff8d4e9..b7feafa69 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -25,6 +25,7 @@ #include "libfsm/internal.h" #include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" enum vmops_dialect { @@ -140,7 +141,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_SUCC: if (-1 == print_hook_accept(f, opt, hooks, - op->endids.ids, op->endids.count, + op->ret->ids, op->ret->count, default_accept, (void *) prefix)) { @@ -187,6 +188,7 @@ static int fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, const char *prefix) { @@ -195,6 +197,8 @@ fsm_print_vmopsfrag(FILE *f, assert(f != NULL); assert(opt != NULL); + (void) retlist; + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { if (-1 == print_label(f, op, opt)) { @@ -242,6 +246,7 @@ int fsm_print_vmops(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops, enum vmops_dialect dialect) { @@ -259,7 +264,7 @@ fsm_print_vmops(FILE *f, if (opt->fragment) { if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, opt, hooks, ops, prefix)) { + if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { return -1; } } @@ -271,7 +276,7 @@ fsm_print_vmops(FILE *f, fprintf(f, "#include \"%svmops.h\"\n", prefix); fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, opt, hooks, ops, prefix)) { + if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { return -1; } fprintf(f, "\t};\n"); @@ -429,26 +434,29 @@ int fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, hooks, ops, VMOPS_C); + return fsm_print_vmops(f, opt, hooks, retlist, ops, VMOPS_C); } int fsm_print_vmops_h(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, hooks, ops, VMOPS_H); + return fsm_print_vmops(f, opt, hooks, retlist, ops, VMOPS_H); } int fsm_print_vmops_main(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, hooks, ops, VMOPS_MAIN); + return fsm_print_vmops(f, opt, hooks, retlist, ops, VMOPS_MAIN); } diff --git a/src/libfsm/vm.c b/src/libfsm/vm.c index c9205815f..0eb4fa16c 100644 --- a/src/libfsm/vm.c +++ b/src/libfsm/vm.c @@ -18,6 +18,7 @@ #include "internal.h" #include "vm/vm.h" +#include "vm/retlist.h" #include "print/ir.h" // VM state: @@ -92,6 +93,7 @@ fsm_vm_compile_with_options(const struct fsm *fsm, static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct ir *ir; + struct ret_list retlist; struct fsm_dfavm *vm; assert(fsm != NULL); @@ -102,9 +104,15 @@ fsm_vm_compile_with_options(const struct fsm *fsm, return NULL; } + if (!build_retlist(&retlist, ir)) { + free_ir(fsm, ir); + return NULL; + } + a = zero; - if (!dfavm_compile_ir(&a, ir, vmopts)) { + if (!dfavm_compile_ir(&a, ir, &retlist, vmopts)) { + free_retlist(&retlist); free_ir(fsm, ir); return NULL; } @@ -116,6 +124,7 @@ fsm_vm_compile_with_options(const struct fsm *fsm, return NULL; } + free_retlist(&retlist); dfavm_opasm_finalize_op(&a); return vm; diff --git a/src/libfsm/vm/ir.c b/src/libfsm/vm/ir.c index f27df508d..608871b61 100644 --- a/src/libfsm/vm/ir.c +++ b/src/libfsm/vm/ir.c @@ -18,6 +18,7 @@ #include "libfsm/internal.h" +#include "retlist.h" #include "vm.h" #include "print/ir.h" @@ -277,7 +278,8 @@ opasm_free_list(struct dfavm_assembler_ir *a, struct dfavm_op_ir *op) } static struct dfavm_op_ir * -opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op_cmp cmp, unsigned char arg, +opasm_new(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + enum dfavm_op_instr instr, enum dfavm_op_cmp cmp, unsigned char arg, const struct ir_state *ir_state) { static const struct dfavm_op_ir zero; @@ -306,20 +308,22 @@ opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op if (ir_state != NULL) { op->example = ir_state->example; - op->endids.ids = ir_state->endids.ids; - op->endids.count = ir_state->endids.count; + op->ret = ir_state->isend + ? find_ret(retlist, ir_state->endids.ids, ir_state->endids.count) + : NULL; } return op; } static struct dfavm_op_ir * -opasm_new_fetch(struct dfavm_assembler_ir *a, unsigned state, enum dfavm_op_end end, +opasm_new_fetch(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + unsigned state, enum dfavm_op_end end, const struct ir_state *ir_state) { struct dfavm_op_ir *op; - op = opasm_new(a, VM_OP_FETCH, VM_CMP_ALWAYS, 0, ir_state); + op = opasm_new(a, retlist, VM_OP_FETCH, VM_CMP_ALWAYS, 0, ir_state); if (op == NULL) { return NULL; } @@ -331,12 +335,13 @@ opasm_new_fetch(struct dfavm_assembler_ir *a, unsigned state, enum dfavm_op_end } static struct dfavm_op_ir * -opasm_new_stop(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned char arg, enum dfavm_op_end end, +opasm_new_stop(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + enum dfavm_op_cmp cmp, unsigned char arg, enum dfavm_op_end end, const struct ir_state *ir_state) { struct dfavm_op_ir *op; - op = opasm_new(a, VM_OP_STOP, cmp, arg, ir_state); + op = opasm_new(a, retlist, VM_OP_STOP, cmp, arg, ir_state); if (op == NULL) { return NULL; } @@ -347,14 +352,15 @@ opasm_new_stop(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned cha } static struct dfavm_op_ir * -opasm_new_branch(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned char arg, uint32_t dest_state, +opasm_new_branch(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + enum dfavm_op_cmp cmp, unsigned char arg, uint32_t dest_state, const struct ir_state *ir_state) { struct dfavm_op_ir *op; assert(dest_state < a->nstates); - op = opasm_new(a, VM_OP_BRANCH, cmp, arg, ir_state); + op = opasm_new(a, retlist, VM_OP_BRANCH, cmp, arg, ir_state); if (op == NULL) { return NULL; } @@ -488,7 +494,8 @@ analyze_table(struct dfa_table *table) } static int -xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct dfavm_op_ir **opp) +xlate_table_ranges(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct dfa_table *table, struct dfavm_op_ir **opp) { int i,lo; int count = 0; @@ -508,8 +515,8 @@ xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct enum dfavm_op_cmp cmp = (i > lo+1) ? VM_CMP_LE : VM_CMP_EQ; op = (dst < 0) - ? opasm_new_stop(a, cmp, arg, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, cmp, arg, dst, table->ir_state); + ? opasm_new_stop(a, retlist, cmp, arg, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, cmp, arg, dst, table->ir_state); if (op == NULL) { return -1; @@ -526,8 +533,8 @@ xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct if (lo < FSM_SIGMA_COUNT) { int64_t dst = table->tbl[lo]; *opp = (dst < 0) - ? opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, VM_CMP_ALWAYS, 0, dst, table->ir_state); + ? opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, dst, table->ir_state); if (*opp == NULL) { return -1; } @@ -539,7 +546,8 @@ xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } static int -xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct dfavm_op_ir **opp) +xlate_table_cases(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct dfa_table *table, struct dfavm_op_ir **opp) { int i, count = 0; int64_t mdst = table->mode.to; @@ -554,8 +562,8 @@ xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } *opp = (dst < 0) - ? opasm_new_stop(a, VM_CMP_EQ, i, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, VM_CMP_EQ, i, dst, table->ir_state); + ? opasm_new_stop(a, retlist, VM_CMP_EQ, i, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, VM_CMP_EQ, i, dst, table->ir_state); if (*opp == NULL) { return -1; } @@ -564,8 +572,8 @@ xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } *opp = (mdst < 0) - ? opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, VM_CMP_ALWAYS, 0, mdst, table->ir_state); + ? opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, mdst, table->ir_state); if (*opp == NULL) { return -1; } @@ -576,7 +584,8 @@ xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } static int -initial_translate_table(struct dfavm_assembler_ir *a, struct dfa_table *table, struct dfavm_op_ir **opp) +initial_translate_table(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct dfa_table *table, struct dfavm_op_ir **opp) { int count, best_count; struct dfavm_op_ir *op, *best_op; @@ -604,13 +613,13 @@ initial_translate_table(struct dfavm_assembler_ir *a, struct dfa_table *table, s assert(dst >= 0); assert((size_t)dst < a->nstates); - *opp = opasm_new_stop(a, VM_CMP_NE, sym, VM_END_FAIL, table->ir_state); + *opp = opasm_new_stop(a, retlist, VM_CMP_NE, sym, VM_END_FAIL, table->ir_state); if (*opp == NULL) { return -1; } opp = &(*opp)->next; - *opp = opasm_new_branch(a, VM_CMP_ALWAYS, 0, dst, table->ir_state); + *opp = opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, dst, table->ir_state); if (*opp == NULL) { return -1; } @@ -620,10 +629,10 @@ initial_translate_table(struct dfavm_assembler_ir *a, struct dfa_table *table, s } best_op = NULL; - best_count = xlate_table_ranges(a, table, &best_op); + best_count = xlate_table_ranges(a, retlist, table, &best_op); op = NULL; - count = xlate_table_cases(a, table, &op); + count = xlate_table_cases(a, retlist, table, &op); if (count < best_count) { opasm_free_list(a,best_op); @@ -682,7 +691,8 @@ dfa_table_init(struct dfa_table *table, long default_dest, const struct ir_state } static int -initial_translate_partial(struct dfavm_assembler_ir *a, struct ir_state *st, struct dfavm_op_ir **opp) +initial_translate_partial(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct ir_state *st, struct dfavm_op_ir **opp) { struct dfa_table table; size_t i, ngrps; @@ -696,11 +706,12 @@ initial_translate_partial(struct dfavm_assembler_ir *a, struct ir_state *st, str group_to_table(&table, &st->u.partial.groups[i]); } - return initial_translate_table(a, &table, opp); + return initial_translate_table(a, retlist, &table, opp); } static int -initial_translate_dominant(struct dfavm_assembler_ir *a, struct ir_state *st, struct dfavm_op_ir **opp) +initial_translate_dominant(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct ir_state *st, struct dfavm_op_ir **opp) { struct dfa_table table; size_t i, ngrps; @@ -714,11 +725,12 @@ initial_translate_dominant(struct dfavm_assembler_ir *a, struct ir_state *st, st group_to_table(&table, &st->u.dominant.groups[i]); } - return initial_translate_table(a, &table, opp); + return initial_translate_table(a, retlist, &table, opp); } static int -initial_translate_error(struct dfavm_assembler_ir *a, struct ir_state *st, struct dfavm_op_ir **opp) +initial_translate_error(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct ir_state *st, struct dfavm_op_ir **opp) { struct dfa_table table; size_t i, ngrps; @@ -734,11 +746,13 @@ initial_translate_error(struct dfavm_assembler_ir *a, struct ir_state *st, struc group_to_table(&table, &st->u.error.groups[i]); } - return initial_translate_table(a, &table, opp); + return initial_translate_table(a, retlist, &table, opp); } static struct dfavm_op_ir * -initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_t ind) +initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, + const struct ret_list *retlist, + size_t ind) { struct ir_state *st; struct dfavm_op_ir **opp; @@ -747,22 +761,22 @@ initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_ opp = &a->ops[ind]; if (st->isend && st->strategy == IR_SAME && st->u.same.to == ind) { - *opp = opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_SUCC, st); + *opp = opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_SUCC, st); return a->ops[ind]; } - *opp = opasm_new_fetch(a, ind, (st->isend) ? VM_END_SUCC : VM_END_FAIL, st); + *opp = opasm_new_fetch(a, retlist, ind, (st->isend) ? VM_END_SUCC : VM_END_FAIL, st); opp = &(*opp)->next; assert(*opp == NULL); switch (st->strategy) { case IR_NONE: - *opp = opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, st); + *opp = opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, st); opp = &(*opp)->next; break; case IR_SAME: - *opp = opasm_new_branch(a, VM_CMP_ALWAYS, 0, st->u.same.to, st); + *opp = opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, st->u.same.to, st); opp = &(*opp)->next; break; @@ -774,19 +788,19 @@ initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_ * intelligently. */ case IR_PARTIAL: - if (initial_translate_partial(a, st, opp) < 0) { + if (initial_translate_partial(a, retlist, st, opp) < 0) { return NULL; } break; case IR_DOMINANT: - if (initial_translate_dominant(a, st, opp) < 0) { + if (initial_translate_dominant(a, retlist, st, opp) < 0) { return NULL; } break; case IR_ERROR: - if (initial_translate_error(a, st, opp) < 0) { + if (initial_translate_error(a, retlist, st, opp) < 0) { return NULL; } break; @@ -804,20 +818,6 @@ initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_ return a->ops[ind]; } -static int -initial_translate(const struct ir *ir, struct dfavm_assembler_ir *a) -{ - size_t i,n; - - n = a->nstates; - - for (i=0; i < n; i++) { - a->ops[i] = initial_translate_state(a, ir, i); - } - - return 0; -} - static void fixup_dests(struct dfavm_assembler_ir *a) { @@ -1063,8 +1063,15 @@ print_all_states(struct dfavm_assembler_ir *a) } int -dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_vm_compile_opts opts) +dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, const struct ret_list *retlist, + struct fsm_vm_compile_opts opts) { + size_t i; + + assert(a != NULL); + assert(ir != NULL); + assert(retlist != NULL); + a->nstates = ir->n; a->start = ir->start; @@ -1080,8 +1087,8 @@ dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_v return 0; } - if (initial_translate(ir, a) < 0) { - return 0; + for (i=0; i < a->nstates; i++) { + a->ops[i] = initial_translate_state(a, ir, retlist, i); } fixup_dests(a); @@ -1121,7 +1128,7 @@ dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_v return 0; } - a->ops[0] = opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, NULL); + a->ops[0] = opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, NULL); if (a->ops[0] == NULL) { return -1; } diff --git a/src/libfsm/vm/retlist.c b/src/libfsm/vm/retlist.c index 867c8b561..3202c0ff1 100644 --- a/src/libfsm/vm/retlist.c +++ b/src/libfsm/vm/retlist.c @@ -14,7 +14,9 @@ #include -#include "libfsm/vm/vm.h" +#include "libfsm/internal.h" + +#include "libfsm/print/ir.h" #include "libfsm/vm/retlist.h" static bool @@ -57,8 +59,8 @@ append_ret(struct ret_list *list, return true; } -int -cmp_ret_by_endid(const void *pa, const void *pb) +static int +cmp_ret(const void *pa, const void *pb) { const struct ret *a = pa; const struct ret *b = pb; @@ -70,56 +72,33 @@ cmp_ret_by_endid(const void *pa, const void *pb) } struct ret * -find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, - int (*cmp)(const void *pa, const void *pb)) +find_ret(const struct ret_list *list, + const fsm_end_id_t *ids, size_t count) { struct ret key; - assert(op != NULL); - assert(cmp != NULL); - - key.count = op->endids.count; - key.ids = op->endids.ids; + key.count = count; + key.ids = ids; - return bsearch(&key, list->a, list->count, sizeof *list->a, cmp); + return bsearch(&key, list->a, list->count, sizeof *list->a, cmp_ret); } bool -build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) +build_retlist(struct ret_list *list, const struct ir *ir) { - const struct dfavm_op_ir *op; + size_t i; assert(list != NULL); + assert(ir != NULL); list->count = 0; - for (op = a; op != NULL; op = op->next) { - switch (op->instr) { - case VM_OP_STOP: - if (op->u.stop.end_bits == VM_END_FAIL) { - /* %fail is special, don't add to retlist */ - continue; - } - - break; - - case VM_OP_FETCH: - if (op->u.fetch.end_bits == VM_END_FAIL) { - /* %fail is special, don't add to retlist */ - continue; - } - - break; - - case VM_OP_BRANCH: + for (i = 0; i < ir->n; i++) { + if (!ir->states[i].isend) { continue; - - default: - assert(!"unreached"); - abort(); } - if (!append_ret(list, op->endids.ids, op->endids.count)) { + if (!append_ret(list, ir->states[i].endids.ids, ir->states[i].endids.count)) { return false; } } @@ -128,14 +107,14 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) size_t j = 0; /* sort for both dedup and bsearch */ - qsort(list->a, list->count, sizeof *list->a, cmp_ret_by_endid); + qsort(list->a, list->count, sizeof *list->a, cmp_ret); /* deduplicate based on endids only. * j is the start of a run; i increments until we find * the start of the next run */ for (size_t i = 1; i < list->count; i++) { assert(i > j); - if (cmp_ret_by_endid(&list->a[j], &list->a[i]) == 0) { + if (cmp_ret(&list->a[j], &list->a[i]) == 0) { continue; } diff --git a/src/libfsm/vm/retlist.h b/src/libfsm/vm/retlist.h index 168bd0f70..5e67218b9 100644 --- a/src/libfsm/vm/retlist.h +++ b/src/libfsm/vm/retlist.h @@ -7,6 +7,8 @@ #ifndef FSM_INTERNAL_RETLIST_H #define FSM_INTERNAL_RETLIST_H +struct ir; + struct ret { size_t count; const fsm_end_id_t *ids; @@ -17,15 +19,11 @@ struct ret_list { struct ret *a; }; -int -cmp_ret_by_endid(const void *pa, const void *pb); - struct ret * -find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, - int (*cmp)(const void *pa, const void *pb)); +find_ret(const struct ret_list *list, const fsm_end_id_t *ids, size_t count); bool -build_retlist(struct ret_list *list, const struct dfavm_op_ir *a); +build_retlist(struct ret_list *list, const struct ir *ir); void free_retlist(struct ret_list *list); diff --git a/src/libfsm/vm/vm.h b/src/libfsm/vm/vm.h index bfb28a774..c96bfa84b 100644 --- a/src/libfsm/vm/vm.h +++ b/src/libfsm/vm/vm.h @@ -20,6 +20,8 @@ #define DFAVM_MAGIC "DFAVM$" struct ir; +struct ret; +struct ret_list; struct fsm_vm_compile_opts; struct dfavm_op_ir_pool; @@ -74,15 +76,8 @@ struct dfavm_op_ir { */ uint32_t index; - - -const char *example; - -struct { - fsm_end_id_t *ids; /* NULL -> 0 */ - size_t count; -} endids; - + const char *example; + const struct ret *ret; uint32_t num_incoming; // number of branches to this instruction int in_trace; @@ -198,7 +193,7 @@ const char * cmp_name(int cmp); int -dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_vm_compile_opts opts); +dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, const struct ret_list *retlist, struct fsm_vm_compile_opts opts); struct fsm_dfavm * dfavm_compile_vm(const struct dfavm_assembler_ir *a, struct fsm_vm_compile_opts opts); From 5092212a75e5c8ebbb430608adc9e97a1a7cbac0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 29 Jul 2024 00:07:25 +0100 Subject: [PATCH 216/373] Add AMBIG_MULTIPLE output for go. The generated code looks like this: ```go package fsm_fsm var ret0 []uint = []uint{1} var ret1 []uint = []uint{2} var ret2 []uint = []uint{1, 2} var ret3 []uint = []uint{0, 1, 2} func fsm_Match(data string) (bool, []uint) { var idx = ^uint(0) if idx++; idx >= uint(len(data)) { return true, ret0 } if data[idx] == 'a' { goto l3 } if data[idx] == 'b' { goto l2 } if data[idx] != 'c' { return false, nil } l0: // e.g. "c" if idx++; idx >= uint(len(data)) { return true, ret2 } if data[idx] <= '`' { return false, nil } if data[idx] <= 'b' { goto l1 } if data[idx] == 'c' { goto l0 } { return false, nil } l1: // e.g. "aa" if idx++; idx >= uint(len(data)) { return true, ret1 } if data[idx] <= '`' { return false, nil } if data[idx] <= 'c' { goto l1 } { return false, nil } l2: // e.g. "b" if idx++; idx >= uint(len(data)) { return true, ret3 } if data[idx] == 'a' { goto l1 } if data[idx] == 'b' { goto l2 } if data[idx] == 'c' { goto l1 } { return false, nil } l3: // e.g. "a" if idx++; idx >= uint(len(data)) { return true, ret1 } if data[idx] == 'a' { goto l1 } if data[idx] == 'b' { goto l2 } if data[idx] == 'c' { goto l1 } { return false, nil } } ``` --- src/libfsm/print/go.c | 75 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index fecafddb1..84942ea22 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -48,7 +48,8 @@ cmp_operator(int cmp) static int print_ids(FILE *f, - enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count, + size_t i) { switch (ambig) { case AMBIG_NONE: @@ -61,7 +62,7 @@ print_ids(FILE *f, return -1; } - fprintf(f, ", %u;", ids[0]); + fprintf(f, ", %u", ids[0]); break; case AMBIG_EARLIEST: @@ -69,13 +70,11 @@ print_ids(FILE *f, * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, ", %u;", ids[0]); + fprintf(f, ", %u", ids[0]); break; case AMBIG_MULTIPLE: - assert(!"unreached"); -// TODO: fprintf(f, ", ret%u[0:%u];", TODO: retlist index, count); -// ids[] + fprintf(f, ", ret%zu", i); break; default: @@ -91,16 +90,19 @@ default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { + size_t i; + assert(f != NULL); assert(opt != NULL); - assert(lang_opaque == NULL); + assert(lang_opaque != NULL); - (void) lang_opaque; (void) hook_opaque; + i = * (const size_t *) lang_opaque; + fprintf(f, "return true"); - if (-1 == print_ids(f, opt->ambig, ids, count)) { + if (-1 == print_ids(f, opt->ambig, ids, count, i)) { return -1; } @@ -118,7 +120,30 @@ default_reject(FILE *f, const struct fsm_options *opt, (void) lang_opaque; (void) hook_opaque; - fprintf(f, "{\n\t\treturn false\n\t}\n"); + fprintf(f, "{\n\t\treturn false"); + + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + fprintf(f, ", 0"); + break; + + case AMBIG_EARLIEST: + fprintf(f, ", 0"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, ", nil"); + break; + + default: + assert(!"unreached"); + abort(); + } + + fprintf(f, "\n\t}\n"); return 0; } @@ -153,20 +178,26 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, enum dfavm_op_end end_bits) { + size_t i; + switch (end_bits) { case VM_END_FAIL: return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: + assert(op->ret >= retlist->a); + + i = op->ret - retlist->a; + fprintf(f, "{\n"); fprintf(f, "\t\t"); if (-1 == print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, - default_accept, NULL)) - // TODO: index for retlist + default_accept, &i)) { return -1; } @@ -277,12 +308,12 @@ fsm_print_gofrag(FILE *f, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - print_end(f, op, opt, hooks, op->u.stop.end_bits); + print_end(f, op, opt, hooks, retlist, op->u.stop.end_bits); break; case VM_OP_FETCH: print_fetch(f, opt); - print_end(f, op, opt, hooks, op->u.fetch.end_bits); + print_end(f, op, opt, hooks, retlist, op->u.fetch.end_bits); break; case VM_OP_BRANCH: @@ -339,6 +370,20 @@ fsm_print_go(FILE *f, fprintf(f, "package %sfsm\n", package_prefix); fprintf(f, "\n"); + if (opt->ambig == AMBIG_MULTIPLE) { + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "var ret%zu []uint = []uint{", i); + for (size_t j = 0; j < retlist->a[i].count; j++) { + fprintf(f, "%u", retlist->a[i].ids[j]); + if (j + 1 < retlist->a[i].count) { + fprintf(f, ", "); + } + } + fprintf(f, "}\n"); + } + fprintf(f, "\n"); + } + fprintf(f, "func %sMatch", prefix); switch (opt->io) { @@ -376,7 +421,7 @@ fsm_print_go(FILE *f, break; case AMBIG_MULTIPLE: - fprintf(stdout, "(bool, uint[])"); + fprintf(stdout, "(bool, []uint)"); break; default: From 7cdb00b62c9a927b49b7a27edb6a66443b176998 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 31 Jul 2024 01:18:53 +0100 Subject: [PATCH 217/373] No need for the vmops_dialect enum here. --- src/libfsm/print/vmops.c | 364 ++++++++++++++++++++------------------- 1 file changed, 184 insertions(+), 180 deletions(-) diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index b7feafa69..ac3183071 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -28,12 +28,6 @@ #include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -enum vmops_dialect { - VMOPS_C, - VMOPS_H, - VMOPS_MAIN, -}; - static const char * cmp_operator(int cmp) { @@ -173,6 +167,7 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) fprintf(f, "\t\t{%sopEOF, 0, ", prefix); switch (opt->io) { + case FSM_IO_GETC: case FSM_IO_STR: case FSM_IO_PAIR: break; @@ -243,12 +238,11 @@ fsm_print_vmopsfrag(FILE *f, } int -fsm_print_vmops(FILE *f, +fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ret_list *retlist, - struct dfavm_op_ir *ops, - enum vmops_dialect dialect) + struct dfavm_op_ir *ops) { const char *prefix; @@ -263,183 +257,24 @@ fsm_print_vmops(FILE *f, } if (opt->fragment) { - if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { - return -1; - } + if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { + return -1; } } else { - switch (dialect) { - case VMOPS_C: - fprintf(f, "#include \n\n"); - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \"%svmops.h\"\n", prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { - return -1; - } - fprintf(f, "\t};\n"); - break; - - case VMOPS_H: - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#define %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \n\n"); - fprintf(f, "enum %svmOp { %sopEOF, %sopLT, %sopLE, %sopEQ, %sopNE, %sopGE, %sopGT, %sopALWAYS};\n", - prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix); - fprintf(f, "enum %sactionOp { %sactionRET, %sactionGOTO };\n", prefix, prefix, prefix); - fprintf(f, "struct %sop { enum %svmOp op; unsigned char c; enum %sactionOp action; int32_t arg; };\n\n", - prefix, prefix, prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - break; - - case VMOPS_MAIN: - fprintf(f, "#include \n"); - fprintf(f, "#include \n"); - fprintf(f, "#include \n\n"); - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \"%svmops.h\"\n", prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - fprintf(f, "extern struct %sop %sOps[];\n", prefix, prefix); - fprintf(f, "\n"); - - switch (opt->io) { - case FSM_IO_PAIR: - fprintf(f, "int %smatch(const char *b, const char *e)\n", prefix); - break; - - case FSM_IO_STR: - fprintf(f, "int %smatch(const char *s)\n", prefix); - break; - - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } - fprintf(f, "{\n"); - fprintf(f, "\tunsigned int i = 0;\n"); - fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); - fprintf(f, "\t/* Initialize to zero to prevent maybe-uninitialized warning. */\n"); - fprintf(f, "\tunsigned char c = 0;\n"); - fprintf(f, "\tint ok;\n"); - fprintf(f, "\tstruct %sop *ops = %sOps;\n", prefix, prefix); - - switch (opt->io) { - case FSM_IO_PAIR: - fprintf(f, "\tconst char *p = b;\n"); - break; - - case FSM_IO_STR: - fprintf(f, "\tconst char *p = s;\n"); - break; - - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } - - fprintf(f, "\n"); - fprintf(f, "\tfor (;;) {\n"); - fprintf(f, "\t\tok = 0;\n"); - fprintf(f, "\t\tswitch (ops[i].op) {\n"); - fprintf(f, "\t\tcase %sopEOF:\n", prefix); - - switch (opt->io) { - case FSM_IO_PAIR: - fprintf(f, "\t\t\tif (p < e) {\n"); - fprintf(f, "\t\t\t\t/* not at EOF */\n"); - fprintf(f, "\t\t\t\tc = *p++;\n"); - fprintf(f, "\t\t\t\ti++;\n"); - fprintf(f, "\t\t\t\tcontinue;\n"); - fprintf(f, "\t\t\t}\n"); - break; - - case FSM_IO_STR: - fprintf(f, "\t\t\tc = *p++;\n"); - fprintf(f, "\t\t\tif (c != '\\0') {\n"); - fprintf(f, "\t\t\t\t/* not at EOF */\n"); - fprintf(f, "\t\t\t\ti++;\n"); - fprintf(f, "\t\t\t\tcontinue;\n"); - fprintf(f, "\t\t\t}\n"); - break; - - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } - - fprintf(f, "\t\t\tok = 1;\n"); - fprintf(f, "\t\t\tbreak;\n"); - fprintf(f, "\t\tcase %sopLT: ok = c < ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopLE: ok = c <= ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopEQ: ok = c == ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopNE: ok = c != ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopGE: ok = c >= ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopGT: ok = c > ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopALWAYS: ok = 1; break;\n", prefix); - fprintf(f, "\t\t}\n"); - fprintf(f, "\t\tif (ok) {\n"); - fprintf(f, "\t\t\tif (ops[i].action == %sactionRET) {\n", prefix); - fprintf(f, "\t\t\t\treturn (int) (ops[i].arg);\n"); - fprintf(f, "\t\t\t}\n"); - fprintf(f, "\t\t\ti = ops[i].arg;\n"); - fprintf(f, "\t\t\tcontinue;\n"); - fprintf(f, "\t\t}\n"); - fprintf(f, "\t\ti++;\n"); - fprintf(f, "\t}\n"); - fprintf(f, "}\n"); - fprintf(f, "\n"); - fprintf(f, "#define %sBUFFER_SIZE (1024)\n", prefix); - fprintf(f, "\n"); - fprintf(f, "int main(void)\n"); - fprintf(f, "{\n"); - fprintf(f, "\tchar *buf, *p;\n"); - fprintf(f, "\tint r;\n"); - fprintf(f, "\n"); - fprintf(f, "\tbuf = malloc(%sBUFFER_SIZE);\n", prefix); - fprintf(f, "\tif (!buf) {\n"); - fprintf(f, "\t\tperror(\"malloc\");\n"); - fprintf(f, "\t\texit(1);\n"); - fprintf(f, "\t}\n\n"); - fprintf(f, "\tfor (;;) {\n"); - fprintf(f, "\t\tp = fgets(buf, %sBUFFER_SIZE, stdin);\n", prefix); - fprintf(f, "\t\tif (!p) {\n"); - fprintf(f, "\t\t\tbreak;\n"); - fprintf(f, "\t\t}\n"); - - switch (opt->io) { - case FSM_IO_PAIR: - fprintf(f, "\t\tr = %smatch(p, p + strlen(p));\n", prefix); - break; - case FSM_IO_STR: - fprintf(f, "\t\tr = %smatch(p);\n", prefix); - break; - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } - fprintf(f, "\t\tprintf(\"%%smatch\\n\", r ? \"no \" : \"\");\n"); - fprintf(f, "\t}\n"); - fprintf(f, "\treturn 0;\n"); - fprintf(f, "}\n"); - break; + fprintf(f, "#include \n\n"); + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \"%svmops.h\"\n", prefix); + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); + if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { + return -1; } + fprintf(f, "\t};\n"); } return 0; } -int -fsm_print_vmops_c(FILE *f, - const struct fsm_options *opt, - const struct fsm_hooks *hooks, - const struct ret_list *retlist, - struct dfavm_op_ir *ops) -{ - return fsm_print_vmops(f, opt, hooks, retlist, ops, VMOPS_C); -} - int fsm_print_vmops_h(FILE *f, const struct fsm_options *opt, @@ -447,7 +282,32 @@ fsm_print_vmops_h(FILE *f, const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, hooks, retlist, ops, VMOPS_H); + const char *prefix; + + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + (void) retlist; + (void) ops; + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#define %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \n\n"); + fprintf(f, "enum %svmOp { %sopEOF, %sopLT, %sopLE, %sopEQ, %sopNE, %sopGE, %sopGT, %sopALWAYS};\n", + prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix); + fprintf(f, "enum %sactionOp { %sactionRET, %sactionGOTO };\n", prefix, prefix, prefix); + fprintf(f, "struct %sop { enum %svmOp op; unsigned char c; enum %sactionOp action; int32_t arg; };\n\n", + prefix, prefix, prefix); + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + + return 0; } int @@ -457,6 +317,150 @@ fsm_print_vmops_main(FILE *f, const struct ret_list *retlist, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, opt, hooks, retlist, ops, VMOPS_MAIN); + const char *prefix; + + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + (void) retlist; + (void) ops; + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + fprintf(f, "#include \n"); + fprintf(f, "#include \n"); + fprintf(f, "#include \n\n"); + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \"%svmops.h\"\n", prefix); + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + fprintf(f, "extern struct %sop %sOps[];\n", prefix, prefix); + fprintf(f, "\n"); + + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "int %smatch(const char *b, const char *e)\n", prefix); + break; + + case FSM_IO_STR: + fprintf(f, "int %smatch(const char *s)\n", prefix); + break; + + case FSM_IO_GETC: + errno = ENOTSUP; + return -1; + } + fprintf(f, "{\n"); + fprintf(f, "\tunsigned int i = 0;\n"); + fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); + fprintf(f, "\t/* Initialize to zero to prevent maybe-uninitialized warning. */\n"); + fprintf(f, "\tunsigned char c = 0;\n"); + fprintf(f, "\tint ok;\n"); + fprintf(f, "\tstruct %sop *ops = %sOps;\n", prefix, prefix); + + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "\tconst char *p = b;\n"); + break; + + case FSM_IO_STR: + fprintf(f, "\tconst char *p = s;\n"); + break; + + case FSM_IO_GETC: + errno = ENOTSUP; + return -1; + } + + fprintf(f, "\n"); + fprintf(f, "\tfor (;;) {\n"); + fprintf(f, "\t\tok = 0;\n"); + fprintf(f, "\t\tswitch (ops[i].op) {\n"); + fprintf(f, "\t\tcase %sopEOF:\n", prefix); + + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "\t\t\tif (p < e) {\n"); + fprintf(f, "\t\t\t\t/* not at EOF */\n"); + fprintf(f, "\t\t\t\tc = *p++;\n"); + fprintf(f, "\t\t\t\ti++;\n"); + fprintf(f, "\t\t\t\tcontinue;\n"); + fprintf(f, "\t\t\t}\n"); + break; + + case FSM_IO_STR: + fprintf(f, "\t\t\tc = *p++;\n"); + fprintf(f, "\t\t\tif (c != '\\0') {\n"); + fprintf(f, "\t\t\t\t/* not at EOF */\n"); + fprintf(f, "\t\t\t\ti++;\n"); + fprintf(f, "\t\t\t\tcontinue;\n"); + fprintf(f, "\t\t\t}\n"); + break; + + case FSM_IO_GETC: + errno = ENOTSUP; + return -1; + } + + fprintf(f, "\t\t\tok = 1;\n"); + fprintf(f, "\t\t\tbreak;\n"); + fprintf(f, "\t\tcase %sopLT: ok = c < ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopLE: ok = c <= ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopEQ: ok = c == ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopNE: ok = c != ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopGE: ok = c >= ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopGT: ok = c > ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopALWAYS: ok = 1; break;\n", prefix); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t\tif (ok) {\n"); + fprintf(f, "\t\t\tif (ops[i].action == %sactionRET) {\n", prefix); + fprintf(f, "\t\t\t\treturn (int) (ops[i].arg);\n"); + fprintf(f, "\t\t\t}\n"); + fprintf(f, "\t\t\ti = ops[i].arg;\n"); + fprintf(f, "\t\t\tcontinue;\n"); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t\ti++;\n"); + fprintf(f, "\t}\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); + fprintf(f, "#define %sBUFFER_SIZE (1024)\n", prefix); + fprintf(f, "\n"); + fprintf(f, "int main(void)\n"); + fprintf(f, "{\n"); + fprintf(f, "\tchar *buf, *p;\n"); + fprintf(f, "\tint r;\n"); + fprintf(f, "\n"); + fprintf(f, "\tbuf = malloc(%sBUFFER_SIZE);\n", prefix); + fprintf(f, "\tif (!buf) {\n"); + fprintf(f, "\t\tperror(\"malloc\");\n"); + fprintf(f, "\t\texit(1);\n"); + fprintf(f, "\t}\n\n"); + fprintf(f, "\tfor (;;) {\n"); + fprintf(f, "\t\tp = fgets(buf, %sBUFFER_SIZE, stdin);\n", prefix); + fprintf(f, "\t\tif (!p) {\n"); + fprintf(f, "\t\t\tbreak;\n"); + fprintf(f, "\t\t}\n"); + + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "\t\tr = %smatch(p, p + strlen(p));\n", prefix); + break; + case FSM_IO_STR: + fprintf(f, "\t\tr = %smatch(p);\n", prefix); + break; + case FSM_IO_GETC: + errno = ENOTSUP; + return -1; + } + fprintf(f, "\t\tprintf(\"%%smatch\\n\", r ? \"no \" : \"\");\n"); + fprintf(f, "\t}\n"); + fprintf(f, "\treturn 0;\n"); + fprintf(f, "}\n"); + + return 0; } From 7e970d72302a1ecd6ba9aa550c34ef5e5681bcd5 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 31 Jul 2024 02:11:40 +0100 Subject: [PATCH 218/373] Whitespace. --- src/libfsm/print/vmops.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index ac3183071..201886a65 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -108,7 +108,7 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt static int print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const char *prefix) { - fprintf(f, "\t\t{%s%s, ", prefix, cmp_operator(op->cmp)); + fprintf(f, "%s%s, ", prefix, cmp_operator(op->cmp)); if (-1 == c_escputcharlit(f, opt, op->cmp_arg)) { return -1; } @@ -148,15 +148,13 @@ print_end(FILE *f, const struct dfavm_op_ir *op, abort(); } - fprintf(f, "},\n"); - return 0; } static int print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) { - fprintf(f, "%sactionGOTO, %" PRIu32 "},\n", prefix, op->u.br.dest_arg->index); + fprintf(f, "%sactionGOTO, %" PRIu32, prefix, op->u.br.dest_arg->index); return 0; } @@ -165,7 +163,7 @@ static int print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) { - fprintf(f, "\t\t{%sopEOF, 0, ", prefix); + fprintf(f, "%sopEOF, 0, ", prefix); switch (opt->io) { case FSM_IO_GETC: case FSM_IO_STR: @@ -200,6 +198,9 @@ fsm_print_vmopsfrag(FILE *f, return -1; } } + + fprintf(f, "\t\t{"); + switch (op->instr) { case VM_OP_STOP: if (-1 == print_cond(f, op, opt, prefix)) { @@ -232,6 +233,8 @@ fsm_print_vmopsfrag(FILE *f, assert(!"unreached"); break; } + + fprintf(f, "},\n"); } return 0; From f2bae911f330f17177d6c7e4394064c9c0380f71 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 2 Aug 2024 14:46:28 +0100 Subject: [PATCH 219/373] Bugfix; 1 means VM_END_SUCC here. --- src/libfsm/print/vmops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 201886a65..6f9238925 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -459,7 +459,7 @@ fsm_print_vmops_main(FILE *f, errno = ENOTSUP; return -1; } - fprintf(f, "\t\tprintf(\"%%smatch\\n\", r ? \"no \" : \"\");\n"); + fprintf(f, "\t\tprintf(\"%%smatch\\n\", r == 0 ? \"no \" : \"\");\n"); fprintf(f, "\t}\n"); fprintf(f, "\treturn 0;\n"); fprintf(f, "}\n"); From ac4a66a13dda38b38ef764f6747a731b0afd3618 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 2 Aug 2024 18:58:04 +0100 Subject: [PATCH 220/373] Ensure we don't pass NULL to memcmp() --- src/libfsm/vm/retlist.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libfsm/vm/retlist.c b/src/libfsm/vm/retlist.c index 3202c0ff1..97d4eb3f0 100644 --- a/src/libfsm/vm/retlist.c +++ b/src/libfsm/vm/retlist.c @@ -68,6 +68,13 @@ cmp_ret(const void *pa, const void *pb) if (a->count < b->count) { return -1; } if (a->count > b->count) { return +1; } + if (a->count == 0) { + return 0; + } + + assert(a->ids != NULL); + assert(b->ids != NULL); + return memcmp(a->ids, b->ids, a->count * sizeof *a->ids); } From 436d6e08c842440f1f55d010837bbb9b5958f3b0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 2 Aug 2024 19:06:22 +0100 Subject: [PATCH 221/373] Typo. --- src/libfsm/print/c.c | 12 ++++++------ src/libfsm/print/go.c | 6 +++--- src/libfsm/print/vmc.c | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 24a04e843..30a176199 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -545,13 +545,13 @@ fsm_print_c(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(stdout, ",\n"); - fprintf(stdout, "\tconst unsigned *id"); + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned *id"); break; case AMBIG_MULTIPLE: - fprintf(stdout, ",\n"); - fprintf(stdout, "\tconst unsigned **ids, size_t *count"); + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); break; default: @@ -560,8 +560,8 @@ fsm_print_c(FILE *f, } if (hooks->args != NULL) { - fprintf(stdout, ",\n"); - fprintf(stdout, "\t"); + fprintf(f, ",\n"); + fprintf(f, "\t"); if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 5a0ac9668..ee87538a8 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -349,7 +349,7 @@ fsm_print_go(FILE *f, } if (hooks->args != NULL) { - fprintf(stdout, ", "); + fprintf(f, ", "); if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; @@ -365,11 +365,11 @@ fsm_print_go(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(stdout, "(bool, uint)"); + fprintf(f, "(bool, uint)"); break; case AMBIG_MULTIPLE: - // TODO: fprintf(stdout, "(bool, uint[])"); + // TODO: fprintf(f, "(bool, uint[])"); errno = ENOTSUP; return -1; diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 206e333ae..e2dd15bad 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -565,13 +565,13 @@ fsm_print_vmc(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(stdout, ",\n"); - fprintf(stdout, "\tconst unsigned *id"); + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned *id"); break; case AMBIG_MULTIPLE: - fprintf(stdout, ",\n"); - fprintf(stdout, "\tconst unsigned **ids, size_t *count"); + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); break; default: @@ -580,8 +580,8 @@ fsm_print_vmc(FILE *f, } if (hooks->args != NULL) { - fprintf(stdout, ",\n"); - fprintf(stdout, "\t"); + fprintf(f, ",\n"); + fprintf(f, "\t"); if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; From e484294b1ffdbb9310fc30df75162a6ba8a4c51b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 2 Aug 2024 15:28:09 +0100 Subject: [PATCH 222/373] Support for enids for the vmops output. I've handled AMBIG_NONE here, but I haven't distinguished the other ambig modes. Other than AMBIG_NONE, the other modes are all presented as an array of ids, even if it's just a single element. That's because I don't see any reason to give these specialised APIs for the current use-cases for this generated code, which is supposed to be a direct representation of our VM opcodes. --- src/libfsm/print/vmops.c | 204 +++++++++++++++++++++++++++------------ 1 file changed, 140 insertions(+), 64 deletions(-) diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 6f9238925..ff031a08d 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -51,6 +51,7 @@ default_accept(FILE *f, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque) { const char *prefix; + size_t i; assert(f != NULL); assert(opt != NULL); @@ -58,13 +59,21 @@ default_accept(FILE *f, const struct fsm_options *opt, (void) hook_opaque; - prefix = lang_opaque; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } - // TODO: print ids (void) ids; (void) count; fprintf(f, "%sactionRET, 1", prefix); + if (opt->ambig != AMBIG_NONE) { + i = * (const size_t *) lang_opaque; + + fprintf(f, ", %zu", i); + } return 0; } @@ -77,13 +86,21 @@ default_reject(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(lang_opaque != NULL); + assert(lang_opaque == NULL); + (void) lang_opaque; (void) hook_opaque; - prefix = lang_opaque; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } fprintf(f, "%sactionRET, 0", prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ", 0"); + } return 0; } @@ -121,23 +138,26 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const char *prefix, + const struct ret_list *retlist, enum dfavm_op_end end_bits) { + size_t i; + switch (end_bits) { case VM_END_FAIL: - if (-1 == print_hook_reject(f, opt, hooks, default_reject, - (void *) prefix)) - { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } break; case VM_END_SUCC: + assert(op->ret >= retlist->a); + + i = op->ret - retlist->a; + if (-1 == print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, - default_accept, - (void *) prefix)) + default_accept, &i)) { return -1; } @@ -152,9 +172,12 @@ print_end(FILE *f, const struct dfavm_op_ir *op, } static int -print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) +print_branch(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op, const char *prefix) { fprintf(f, "%sactionGOTO, %" PRIu32, prefix, op->u.br.dest_arg->index); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ", 0"); + } return 0; } @@ -176,21 +199,55 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) return 0; } -/* TODO: eventually to be non-static */ -static int -fsm_print_vmopsfrag(FILE *f, +int +fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, const struct ret_list *retlist, - struct dfavm_op_ir *ops, - const char *prefix) + struct dfavm_op_ir *ops) { - struct dfavm_op_ir *op; + const struct dfavm_op_ir *op; + const char *prefix; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); - (void) retlist; + if (opt->fragment) { + errno = ENOTSUP; + return -1; + } + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + fprintf(f, "#include \n\n"); + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \"%svmops.h\"\n", prefix); + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + fprintf(f, "\n"); + + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "struct %sret %sRet[] = {\n", prefix, prefix); + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "\t{ (const unsigned []) { "); + for (size_t j = 0; j < retlist->a[i].count; j++) { + fprintf(f, "%u", retlist->a[i].ids[j]); + if (j + 1 < retlist->a[i].count) { + fprintf(f, ", "); + } + } + fprintf(f, " }, %zu },\n", retlist->a[i].count); + } + fprintf(f, "};\n"); + fprintf(f, "const size_t %sRet_count = sizeof %sRet / sizeof *%sRet;\n", prefix, prefix, prefix); + fprintf(f, "\n"); + } + + fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { @@ -199,14 +256,14 @@ fsm_print_vmopsfrag(FILE *f, } } - fprintf(f, "\t\t{"); + fprintf(f, "\t{"); switch (op->instr) { case VM_OP_STOP: if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, hooks, prefix, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, retlist, op->u.stop.end_bits)) { return -1; } break; @@ -215,7 +272,7 @@ fsm_print_vmopsfrag(FILE *f, if (-1 == print_fetch(f, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, hooks, prefix, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, retlist, op->u.fetch.end_bits)) { return -1; } break; @@ -224,7 +281,7 @@ fsm_print_vmopsfrag(FILE *f, if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_branch(f, op, prefix)) { + if (-1 == print_branch(f, opt, op, prefix)) { return -1; } break; @@ -237,43 +294,7 @@ fsm_print_vmopsfrag(FILE *f, fprintf(f, "},\n"); } - return 0; -} - -int -fsm_print_vmops_c(FILE *f, - const struct fsm_options *opt, - const struct fsm_hooks *hooks, - const struct ret_list *retlist, - struct dfavm_op_ir *ops) -{ - const char *prefix; - - assert(f != NULL); - assert(opt != NULL); - assert(hooks != NULL); - - if (opt->prefix != NULL) { - prefix = opt->prefix; - } else { - prefix = "fsm_"; - } - - if (opt->fragment) { - if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { - return -1; - } - } else { - fprintf(f, "#include \n\n"); - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \"%svmops.h\"\n", prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, opt, hooks, retlist, ops, prefix)) { - return -1; - } - fprintf(f, "\t};\n"); - } + fprintf(f, "\t};\n"); return 0; } @@ -294,6 +315,11 @@ fsm_print_vmops_h(FILE *f, (void) retlist; (void) ops; + if (opt->fragment) { + errno = ENOTSUP; + return -1; + } + if (opt->prefix != NULL) { prefix = opt->prefix; } else { @@ -303,11 +329,19 @@ fsm_print_vmops_h(FILE *f, fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); fprintf(f, "#define %sLIBFSM_VMOPS_H\n", prefix); fprintf(f, "#include \n\n"); + fprintf(f, "#include \n\n"); fprintf(f, "enum %svmOp { %sopEOF, %sopLT, %sopLE, %sopEQ, %sopNE, %sopGE, %sopGT, %sopALWAYS};\n", prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix); fprintf(f, "enum %sactionOp { %sactionRET, %sactionGOTO };\n", prefix, prefix, prefix); - fprintf(f, "struct %sop { enum %svmOp op; unsigned char c; enum %sactionOp action; int32_t arg; };\n\n", + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "struct %sret { const unsigned *ids; size_t count; };\n\n", prefix); + } + fprintf(f, "struct %sop { enum %svmOp op; unsigned char c; enum %sactionOp action; int32_t arg; int32_t ret; };\n\n", prefix, prefix, prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "extern struct %sret %sRet[];\n", prefix, prefix); + fprintf(f, "extern const size_t %sRet_count;\n", prefix); + } fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); return 0; @@ -329,6 +363,11 @@ fsm_print_vmops_main(FILE *f, (void) retlist; (void) ops; + if (opt->fragment) { + errno = ENOTSUP; + return -1; + } + if (opt->prefix != NULL) { prefix = opt->prefix; } else { @@ -342,21 +381,32 @@ fsm_print_vmops_main(FILE *f, fprintf(f, "#include \"%svmops.h\"\n", prefix); fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); fprintf(f, "extern struct %sop %sOps[];\n", prefix, prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "extern struct %sret %sRet[];\n", prefix, prefix); + fprintf(f, "extern const size_t %sRet_count;\n", prefix); + } fprintf(f, "\n"); + fprintf(f, "int %smatch(", prefix); switch (opt->io) { case FSM_IO_PAIR: - fprintf(f, "int %smatch(const char *b, const char *e)\n", prefix); + fprintf(f, "const char *b, const char *e"); break; case FSM_IO_STR: - fprintf(f, "int %smatch(const char *s)\n", prefix); + fprintf(f, "const char *s"); break; case FSM_IO_GETC: errno = ENOTSUP; return -1; } + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); + } + fprintf(f, ")\n"); + fprintf(f, "{\n"); fprintf(f, "\tunsigned int i = 0;\n"); fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); @@ -421,6 +471,12 @@ fsm_print_vmops_main(FILE *f, fprintf(f, "\t\t}\n"); fprintf(f, "\t\tif (ok) {\n"); fprintf(f, "\t\t\tif (ops[i].action == %sactionRET) {\n", prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "\t\t\t\tif (ops[i].arg) {\n"); + fprintf(f, "\t\t\t\t\t*ids = %sRet[ops[i].ret].ids;\n", prefix); + fprintf(f, "\t\t\t\t\t*count = %sRet[ops[i].ret].count;\n", prefix); + fprintf(f, "\t\t\t\t}\n"); + } fprintf(f, "\t\t\t\treturn (int) (ops[i].arg);\n"); fprintf(f, "\t\t\t}\n"); fprintf(f, "\t\t\ti = ops[i].arg;\n"); @@ -435,6 +491,10 @@ fsm_print_vmops_main(FILE *f, fprintf(f, "int main(void)\n"); fprintf(f, "{\n"); fprintf(f, "\tchar *buf, *p;\n"); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "\tconst unsigned *ids;\n"); + fprintf(f, "\tsize_t count, i;\n"); + } fprintf(f, "\tint r;\n"); fprintf(f, "\n"); fprintf(f, "\tbuf = malloc(%sBUFFER_SIZE);\n", prefix); @@ -448,18 +508,34 @@ fsm_print_vmops_main(FILE *f, fprintf(f, "\t\t\tbreak;\n"); fprintf(f, "\t\t}\n"); + fprintf(f, "\t\tr = %smatch(", prefix); switch (opt->io) { case FSM_IO_PAIR: - fprintf(f, "\t\tr = %smatch(p, p + strlen(p));\n", prefix); + fprintf(f, "p, p + strlen(p)"); break; case FSM_IO_STR: - fprintf(f, "\t\tr = %smatch(p);\n", prefix); + fprintf(f, "p"); break; case FSM_IO_GETC: + // TODO: getc from string buffer errno = ENOTSUP; return -1; } + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ", &ids, &count"); + } + fprintf(f, ");\n"); + fprintf(f, "\t\tprintf(\"%%smatch\\n\", r == 0 ? \"no \" : \"\");\n"); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "\t\tif (r) {\n"); + fprintf(f, "\t\t\tprintf(\"ids:\");\n"); + fprintf(f, "\t\t\tfor (i = 0; i < count; i++) {\n"); + fprintf(f, "\t\t\t\tprintf(\" %%u\", ids[i]);\n"); + fprintf(f, "\t\t\t}\n"); + fprintf(f, "\t\t\tprintf(\"\\n\");\n"); + fprintf(f, "\t\t}\n"); + } fprintf(f, "\t}\n"); fprintf(f, "\treturn 0;\n"); fprintf(f, "}\n"); From 2698127a1b3649d6024c1da570e7ef74213c2f0a Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 6 Aug 2024 00:16:18 +0100 Subject: [PATCH 223/373] First cut at AMBIG_MULTIPLE for llvm. It's a bit rough, this isn't what I want to end up with, but I wanted to commit this as a waypoint. --- src/libfsm/print/llvm.c | 47 +++++++++++++++++++++++++++++++---------- src/re/main.c | 29 +++++++++++++++++++------ src/retest/runner.c | 2 +- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 575d76f96..ef8c621c6 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -91,12 +91,19 @@ default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { + const char *prefix; size_t i; assert(f != NULL); assert(opt != NULL); assert(lang_opaque != NULL); + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm."; + } + (void) hook_opaque; i = * (const size_t *) lang_opaque; @@ -125,9 +132,9 @@ default_accept(FILE *f, const struct fsm_options *opt, break; case AMBIG_MULTIPLE: - // TODO: probably { i1, ptr_u8 } - assert(!"unimplemented"); - abort(); + (void) ids; + fprintf(f, "[{ i1 true, @%sr%zu, %zu }, %%ret%zu],\n", prefix, i, count, i); + break; default: assert(!"unreached"); @@ -155,10 +162,13 @@ default_reject(FILE *f, const struct fsm_options *opt, case AMBIG_ERROR: case AMBIG_EARLIEST: - case AMBIG_MULTIPLE: fprintf(f, "[{ false, undef }, %%fail]\n"); break; + case AMBIG_MULTIPLE: + fprintf(f, "[{ i1 false, %s undef, i32 undef }, %%fail]\n", ptr_i8); + break; + default: assert(!"unreached"); abort(); @@ -177,13 +187,13 @@ print_rettype(FILE *f, enum fsm_ambig ambig) case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(f, "{ i1, u32 }"); + fprintf(f, "{ i1, i32 }"); break; case AMBIG_MULTIPLE: - // TODO: probably { i1, ptr_u8 } - assert(!"unimplemented"); - abort(); + // success, ids, count + fprintf(f, "{ i1, %s, i32 }", ptr_i8); + break; default: assert(!"unreached"); @@ -609,7 +619,7 @@ fsm_print_llvm(FILE *f, if (opt->prefix != NULL) { prefix = opt->prefix; } else { - prefix = "fsm_"; + prefix = "fsm."; } if (hooks->cp != NULL) { @@ -624,8 +634,23 @@ fsm_print_llvm(FILE *f, } fprintf(f, "; generated\n"); -//XXX: type depends on ambig - fprintf(f, "define dso_local i1 @%smain", prefix); + + if (opt->ambig == AMBIG_MULTIPLE) { + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "@%sr%zu = internal unnamed_addr constant [%zu x i32] [", prefix, i, retlist->a[i].count); + for (size_t j = 0; j < retlist->a[i].count; j++) { + fprintf(f, "i32 %u", retlist->a[i].ids[j]); + if (j + 1 < retlist->a[i].count) { + fprintf(f, ", "); + } + } + fprintf(f, "]\n"); + } + } + + fprintf(f, "define dso_local "); + print_rettype(f, opt->ambig); + fprintf(f, " @%smain", prefix); switch (opt->io) { case FSM_IO_GETC: diff --git a/src/re/main.c b/src/re/main.c index 86258f318..34b8d05c1 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -420,25 +420,40 @@ accept_llvm(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - unsigned n; + const char *prefix; size_t i; assert(opt != NULL); assert(lang_opaque != NULL); assert(hook_opaque == NULL); + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm."; + } + (void) opt; (void) hook_opaque; - n = 0; + i = * (const size_t *) lang_opaque; - for (i = 0; i < count; i++) { - n |= 1U << ids[i]; - } + if (opt->ambig == AMBIG_MULTIPLE) { +// XXX +#define ptr_i8 "ptr" + fprintf(f, "[{ i1 true, %s @%sr%zu, i32 %zu }, %%ret%zu],", ptr_i8, prefix, i, count, i); +#undef ptr_i8 + } else { + unsigned n; - i = * (const size_t *) lang_opaque; + n = 0; + + for (i = 0; i < count; i++) { + n |= 1U << ids[i]; + } - fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, i); + fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, i); + } fprintf(f, " ; "); diff --git a/src/retest/runner.c b/src/retest/runner.c index ea5a54372..0dac9ab38 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -379,7 +379,7 @@ runner_init_compiled(struct fsm *fsm, case IMPL_LLVM: r->u.impl_llvm.h = h; - r->u.impl_llvm.func = (bool (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm_main"); + r->u.impl_llvm.func = (bool (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm.main"); break; case IMPL_GO: From 74265017081bd6a23914f444053ff5bd8d113ff0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 6 Aug 2024 23:48:58 +0100 Subject: [PATCH 224/373] Rework stop: to index into an array for return values. This means the phi instruction now only carries an array index. Many thanks to @mcy for advice and patient help here. --- src/libfsm/print/llvm.c | 147 +++++++++++++++++++++++----------------- src/re/main.c | 55 +++++++++------ 2 files changed, 117 insertions(+), 85 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index ef8c621c6..c02b5f46d 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -33,14 +33,16 @@ #define OPAQUE_POINTERS 1 -#ifdef OPAQUE_POINTERS // llvm >= 15 +#if OPAQUE_POINTERS // llvm >= 15 static const char *ptr_i8 = "ptr"; static const char *ptr_i32 = "ptr"; static const char *ptr_void = "ptr"; +static const char *ptr_rt = "ptr"; #else static const char *ptr_i8 = "i8*"; static const char *ptr_i32 = "i32*"; static const char *ptr_void = "i8*"; +static const char *ptr_rt = "%rt*"; #endif static const struct dfavm_op_ir fail; // used as a unqiue address only @@ -91,26 +93,16 @@ default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const char *prefix; - size_t i; - assert(f != NULL); assert(opt != NULL); - assert(lang_opaque != NULL); - - if (opt->prefix != NULL) { - prefix = opt->prefix; - } else { - prefix = "fsm."; - } + assert(lang_opaque == NULL); (void) hook_opaque; - - i = * (const size_t *) lang_opaque; + (void) lang_opaque; switch (opt->ambig) { case AMBIG_NONE: - fprintf(f, "[true, %%ret%zu],\n", i); + fprintf(f, "%%rt true"); break; case AMBIG_ERROR: @@ -120,7 +112,7 @@ default_accept(FILE *f, const struct fsm_options *opt, return -1; } - fprintf(f, "[{ true, %u }, %%ret%zu],\n", ids[0], i); + fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); break; case AMBIG_EARLIEST: @@ -128,12 +120,18 @@ default_accept(FILE *f, const struct fsm_options *opt, * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, "[{ true, i32 %u }, %%ret%zu],\n", ids[0], i); + fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); break; case AMBIG_MULTIPLE: - (void) ids; - fprintf(f, "[{ i1 true, @%sr%zu, %zu }, %%ret%zu],\n", prefix, i, count, i); + fprintf(f, "internal unnamed_addr constant [%zu x i32] [", count); + for (size_t j = 0; j < count; j++) { + fprintf(f, "i32 %u", ids[j]); + if (j + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, "]"); break; default: @@ -157,16 +155,16 @@ default_reject(FILE *f, const struct fsm_options *opt, switch (opt->ambig) { case AMBIG_NONE: - fprintf(f, "[false, %%fail]\n"); + fprintf(f, "%%rt false ; fail\n"); break; case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(f, "[{ false, undef }, %%fail]\n"); + fprintf(f, "%%rt { i1 false, i32 undef } ; fail\n"); break; case AMBIG_MULTIPLE: - fprintf(f, "[{ i1 false, %s undef, i32 undef }, %%fail]\n", ptr_i8); + fprintf(f, "%%rt { i1 false, %s undef, i32 undef } ; fail\n", ptr_i32); break; default: @@ -178,8 +176,10 @@ default_reject(FILE *f, const struct fsm_options *opt, } static int -print_rettype(FILE *f, enum fsm_ambig ambig) +print_rettype(FILE *f, const char *name, enum fsm_ambig ambig) { + fprintf(f, "%s = type ", name); + switch (ambig) { case AMBIG_NONE: fprintf(f, "i1"); @@ -187,12 +187,13 @@ print_rettype(FILE *f, enum fsm_ambig ambig) case AMBIG_ERROR: case AMBIG_EARLIEST: + // success, id fprintf(f, "{ i1, i32 }"); break; case AMBIG_MULTIPLE: // success, ids, count - fprintf(f, "{ i1, %s, i32 }", ptr_i8); + fprintf(f, "{ i1, %s, i32 }", ptr_i32); break; default: @@ -200,6 +201,8 @@ print_rettype(FILE *f, enum fsm_ambig ambig) abort(); } + fprintf(f, "\n"); + return 0; } @@ -415,10 +418,10 @@ print_fetch(FILE *f, const struct fsm_options *opt, static int fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, - const struct fsm_hooks *hooks, const struct ret_list *retlist, struct dfavm_op_ir *ops, - const char *cp) + const char *cp, + const char *prefix) { struct dfavm_op_ir *op; @@ -426,6 +429,7 @@ fsm_print_llvmfrag(FILE *f, assert(opt != NULL); assert(retlist != NULL); assert(cp != NULL); + assert(prefix != NULL); /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; @@ -450,12 +454,14 @@ fsm_print_llvmfrag(FILE *f, * This looks like: * * stop: - * %ret = phi i1 - * [true, %ret0], ; "abc" - * [true, %ret1], ; "xyz" - * [true, %ret2], ; "abc", "xyz" - * [false, %fail] - * ret i1 %ret + * %i = phi i64 + * [0, %ret0], + * [1, %ret1], + * [2, %ret2], + * [3, %fail] + * %p = getelementptr inbounds [4 x %rt], [4 x %rt]* @fsm.r, i64 0, i64 %i + * %ret = load %rt, ptr %p + * ret %rt %ret * fail: * br label %stop * ret0: @@ -465,6 +471,8 @@ fsm_print_llvmfrag(FILE *f, * ret2: * br label %stop * + * where @fsm.r is [4 x %rt] and %rt is the return type. + * * And we jump to stop: via the ret*: labels rather than * to a phi node directly. This helps for two reasons: * @@ -480,29 +488,16 @@ fsm_print_llvmfrag(FILE *f, */ print_label(f, true, "stop"); - fprintf(f, "\t%%ret = phi "); - print_rettype(f, opt->ambig); - fprintf(f, "\n"); - + fprintf(f, "\t%%i = phi i64\n"); for (size_t i = 0; i < retlist->count; i++) { - fprintf(f, "\t "); - - if (-1 == print_hook_accept(f, opt, hooks, - retlist->a[i].ids, retlist->a[i].count, - default_accept, &i)) - { - return -1; - } - } - - fprintf(f, "\t "); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { - return -1; + fprintf(f, "\t [%zu, %%ret%zu],\n", i, i); } + fprintf(f, "\t [%zu, %%fail]\n", retlist->count); - fprintf(f, "\tret "); - print_rettype(f, opt->ambig); - fprintf(f, " %%ret\n"); + fprintf(f, "\t%%p = getelementptr inbounds [%zu x %%rt], [%zu x %%rt]* @%sr, i64 0, i64 %%i\n", + retlist->count + 1, retlist->count + 1, prefix); + fprintf(f, "\t%%ret = load %%rt, %s %%p\n", ptr_rt); + fprintf(f, "\tret %%rt %%ret\n"); print_label(f, true, "fail"); fprintf(f, "\tbr "); @@ -629,28 +624,54 @@ fsm_print_llvm(FILE *f, } if (opt->fragment) { - fsm_print_llvmfrag(f, opt, hooks, retlist, ops, cp); + fsm_print_llvmfrag(f, opt, retlist, ops, cp, prefix); return 0; } fprintf(f, "; generated\n"); + print_rettype(f, "%rt", opt->ambig); + /* + * For AMBIG_MULTIPLE we emit a bunch of arrays and then point at them from + * each %rt. So we call the hook for the arrays, because that's where the id + * list is. For other ambig modes, we call the hook for the %rt instead. + */ if (opt->ambig == AMBIG_MULTIPLE) { for (size_t i = 0; i < retlist->count; i++) { - fprintf(f, "@%sr%zu = internal unnamed_addr constant [%zu x i32] [", prefix, i, retlist->a[i].count); - for (size_t j = 0; j < retlist->a[i].count; j++) { - fprintf(f, "i32 %u", retlist->a[i].ids[j]); - if (j + 1 < retlist->a[i].count) { - fprintf(f, ", "); - } + fprintf(f, "@%sr%zu = ", prefix, i); + if (-1 == print_hook_accept(f, opt, hooks, + retlist->a[i].ids, retlist->a[i].count, + default_accept, NULL)) + { + return -1; + } + fprintf(f, "\n"); + } + } + + fprintf(f, "@%sr = internal unnamed_addr constant [%zu x %%rt] [\n", prefix, retlist->count + 1); + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "\t "); + if (opt->ambig == AMBIG_MULTIPLE) { + fprintf(f, "%%rt { i1 true, %s bitcast ([%zu x i32]* @%sr%zu to %s), i32 %zu }", + ptr_i32, retlist->a[i].count, prefix, i, ptr_i32, retlist->a[i].count); + } else { + if (-1 == print_hook_accept(f, opt, hooks, + retlist->a[i].ids, retlist->a[i].count, + default_accept, NULL)) + { + return -1; } - fprintf(f, "]\n"); } + fprintf(f, ",\n"); + } + fprintf(f, "\t "); + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + return -1; } + fprintf(f, "\t]\n"); - fprintf(f, "define dso_local "); - print_rettype(f, opt->ambig); - fprintf(f, " @%smain", prefix); + fprintf(f, "define dso_local %%rt @%smain", prefix); switch (opt->io) { case FSM_IO_GETC: @@ -701,7 +722,7 @@ fsm_print_llvm(FILE *f, exit(EXIT_FAILURE); } - fsm_print_llvmfrag(f, opt, hooks, retlist, ops, cp); + fsm_print_llvmfrag(f, opt, retlist, ops, cp, prefix); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/re/main.c b/src/re/main.c index 34b8d05c1..f9539af51 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -420,39 +420,52 @@ accept_llvm(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - const char *prefix; size_t i; assert(opt != NULL); - assert(lang_opaque != NULL); + assert(lang_opaque == NULL); assert(hook_opaque == NULL); - if (opt->prefix != NULL) { - prefix = opt->prefix; - } else { - prefix = "fsm."; - } - (void) opt; (void) hook_opaque; - i = * (const size_t *) lang_opaque; + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "%%rt true"); + break; - if (opt->ambig == AMBIG_MULTIPLE) { -// XXX -#define ptr_i8 "ptr" - fprintf(f, "[{ i1 true, %s @%sr%zu, i32 %zu }, %%ret%zu],", ptr_i8, prefix, i, count, i); -#undef ptr_i8 - } else { - unsigned n; + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } - n = 0; + fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); + break; - for (i = 0; i < count; i++) { - n |= 1U << ids[i]; + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); + break; + + case AMBIG_MULTIPLE: + fprintf(f, "internal unnamed_addr constant [%zu x i32] [", count); + for (size_t j = 0; j < count; j++) { + fprintf(f, "i32 %u", ids[j]); + if (j + 1 < count) { + fprintf(f, ", "); + } } + fprintf(f, "]"); + break; - fprintf(f, "[u%#x, %%ret%zu],", (unsigned) n, i); + default: + assert(!"unreached"); + abort(); } fprintf(f, " ; "); @@ -467,8 +480,6 @@ accept_llvm(FILE *f, const struct fsm_options *opt, } } - fprintf(f, "\n"); - return 0; } From f69dbfc0a6f4216164d67d3d3e7a47fe18d627e6 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 00:59:52 +0100 Subject: [PATCH 225/373] Don't use `undef` for values the caller is expected to not access, use `poison`. Thanks to @mcy for this. --- src/libfsm/print/llvm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index c02b5f46d..35129d90a 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -160,11 +160,11 @@ default_reject(FILE *f, const struct fsm_options *opt, case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(f, "%%rt { i1 false, i32 undef } ; fail\n"); + fprintf(f, "%%rt { i1 false, i32 poison } ; fail\n"); break; case AMBIG_MULTIPLE: - fprintf(f, "%%rt { i1 false, %s undef, i32 undef } ; fail\n", ptr_i32); + fprintf(f, "%%rt { i1 false, %s poison, i32 poison } ; fail\n", ptr_i32); break; default: From 292fd1ce82cd162447c292770b962e558e342d2b Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 12:10:38 +0100 Subject: [PATCH 226/373] Split overriding comments to a separate hook. This allows callers to default the codegen for accepting states (in particular outputting the values for endids) independently of commenting caller-specific meanings for the IDs. --- include/fsm/print.h | 4 ++ src/libfsm/print.c | 20 +++++- src/libfsm/print/api.c | 2 +- src/libfsm/print/c.c | 24 +++++-- src/libfsm/print/dot.c | 2 +- src/libfsm/print/llvm.c | 60 +++++++++++----- src/libfsm/print/vmasm.c | 16 +++-- src/libfsm/print/vmdot.c | 10 ++- src/libfsm/print/vmops.c | 18 +++-- src/re/main.c | 146 +++++++++++---------------------------- 10 files changed, 159 insertions(+), 143 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index 453d3d915..c302b9524 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -72,6 +72,10 @@ struct fsm_hooks { int (*reject)(FILE *, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque); + int (*comment)(FILE *, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque); + /* only called for AMBIG_ERROR; see opt.ambig */ int (*conflict)(FILE *, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 34c8863c5..c15b15aef 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -59,6 +59,8 @@ print_hook_accept(FILE *f, void *lang_opaque, void *hook_opaque), void *lang_opaque) { + int r; + assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); @@ -72,10 +74,24 @@ print_hook_accept(FILE *f, } if (hooks->accept != NULL) { - return hooks->accept(f, opt, ids, count, + r = hooks->accept(f, opt, ids, count, lang_opaque, hooks->hook_opaque); } else if (default_accept != NULL) { - return default_accept(f, opt, ids, count, + r = default_accept(f, opt, ids, count, + lang_opaque, hooks->hook_opaque); + } else { + r = 0; + } + + if (r != 0) { + return r; + } + + if (opt->comments && hooks->comment != NULL) { + /* this space is a polyglot */ + fprintf(f, " "); + + r = hooks->comment(f, opt, ids, count, lang_opaque, hooks->hook_opaque); } diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index d7cb70f1f..43b8cb232 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -217,7 +217,7 @@ fsm_print_api(FILE *f, } else { fprintf(f, "\tfor (i = 0x%02x; i <= 0x%02x; i++) {", (unsigned int) lo, (unsigned int) hi - 1); - if (rangeclass(lo, hi - 1)) { + if (opt->comments && rangeclass(lo, hi - 1)) { fprintf(f, " /* '%c' .. '%c' */", (unsigned char) lo, (unsigned char) hi - 1); } fprintf(f, "\n"); diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 24a04e843..40e971e7f 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -352,12 +352,18 @@ print_endstates(FILE *f, /* no end states */ if (!ir_hasend(ir)) { - fprintf(f, "\treturn 0; /* unexpected EOT */\n"); + fprintf(f, "\treturn 0;"); + if (opt->comments) { + fprintf(f, " /* unexpected EOT */"); + } + fprintf(f, "\n"); return 0; } /* usual case */ - fprintf(f, "\t/* end states */\n"); + if (opt->comments) { + fprintf(f, "\t/* end states */\n"); + } fprintf(f, "\tswitch (state) {\n"); for (i = 0; i < ir->n; i++) { if (!ir->states[i].isend) { @@ -410,7 +416,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, fprintf(f, " /* e.g. \""); escputs(f, opt, c_escputc_str, ir->states[i].example); fprintf(f, "\" */"); - } else if (i == ir->start) { + } else if (i == ir->start && opt->comments) { fprintf(f, " /* start */"); } } @@ -423,7 +429,11 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, fprintf(f, "\n"); } fprintf(f, "\t\tdefault:\n"); - fprintf(f, "\t\t\t; /* unreached */\n"); + fprintf(f, "\t\t\t;"); + if (opt->comments) { + fprintf(f, " /* unreached */"); + } + fprintf(f, "\n"); fprintf(f, "\t\t}\n"); if (ferror(f)) { @@ -595,7 +605,11 @@ fsm_print_c(FILE *f, } if (ir->n == 0) { - fprintf(f, "\treturn 0; /* no matches */\n"); + fprintf(f, "\treturn 0;"); + if (opt->comments) { + fprintf(f, " /* no matches */"); + } + fprintf(f, "\n"); } else { if (-1 == fsm_print_c_body(f, ir, opt, hooks)) { return -1; diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index aeb79b254..710b5b01c 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -270,7 +270,7 @@ print_dotfrag(FILE *f, f_free(fsm->alloc, ids); } - /* TODO: show example here, unless !opt->comments */ + /* TODO: comment example per state */ if (-1 == print_state(f, opt, hooks, fsm, prefix, s)) { return -1; diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 35129d90a..57f0b4797 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -155,16 +155,16 @@ default_reject(FILE *f, const struct fsm_options *opt, switch (opt->ambig) { case AMBIG_NONE: - fprintf(f, "%%rt false ; fail\n"); + fprintf(f, "%%rt false"); break; case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(f, "%%rt { i1 false, i32 poison } ; fail\n"); + fprintf(f, "%%rt { i1 false, i32 poison }"); break; case AMBIG_MULTIPLE: - fprintf(f, "%%rt { i1 false, %s poison, i32 poison } ; fail\n", ptr_i32); + fprintf(f, "%%rt { i1 false, %s poison, i32 poison }", ptr_i32); break; default: @@ -172,6 +172,10 @@ default_reject(FILE *f, const struct fsm_options *opt, abort(); } + if (opt->comments) { + fprintf(f, " ; fail"); + } + return 0; } @@ -271,8 +275,11 @@ print_cond(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *op, fprintf(f, "icmp %s i8 %%c%u, ", cmp_operator(op->cmp), use(&frame->c)); llvm_escputcharlit(f, opt, op->cmp_arg); - fprintf(f, " ; "); - c_escputcharlit(f, opt, op->cmp_arg); // C escaping for a comment + + if (opt->comments) { + fprintf(f, " ; "); + c_escputcharlit(f, opt, op->cmp_arg); // C escaping for a comment + } fprintf(f, "\n"); } @@ -332,7 +339,11 @@ print_fetch(FILE *f, const struct fsm_options *opt, ptr_i8); print_decl(f, "r", decl(&frame->r)); - fprintf(f, "icmp eq i32 %%i%u, -1 ; EOF\n", n); + fprintf(f, "icmp eq i32 %%i%u, -1", n); + if (opt->comments) { + fprintf(f, " ; EOF"); + } + fprintf(f, "\n"); // XXX: we don't distinguish error from eof // https://github.com/katef/libfsm/issues/484 @@ -362,8 +373,11 @@ print_fetch(FILE *f, const struct fsm_options *opt, ptr_i8, n); print_decl(f, "r", decl(&frame->r)); - fprintf(f, "icmp eq i8 %%c%u, 0 ; EOT\n", - n); + fprintf(f, "icmp eq i8 %%c%u, 0", n); + if (opt->comments) { + fprintf(f, " ; EOT"); + } + fprintf(f, "\n"); print_branch(f, frame, end_bits == VM_END_FAIL ? &fail : NULL, @@ -385,8 +399,11 @@ print_fetch(FILE *f, const struct fsm_options *opt, ptr_i8, n); print_decl(f, "r", decl(&frame->r)); - fprintf(f, "icmp eq %s %%p%u, %%e ; EOF\n", - ptr_i8, n); + fprintf(f, "icmp eq %s %%p%u, %%e", ptr_i8, n); + if (opt->comments) { + fprintf(f, " ; EOT"); + } + fprintf(f, "\n"); print_branch(f, frame, end_bits == VM_END_FAIL ? &fail : NULL, @@ -516,16 +533,22 @@ fsm_print_llvmfrag(FILE *f, for (op = ops; op != NULL; op = op->next) { if (op->instr != VM_OP_STOP || op->cmp != VM_CMP_ALWAYS || op->u.stop.end_bits != VM_END_FAIL) { print_label(f, true, "l%" PRIu32, op->index); - } - if (op->example != NULL) { - /* C's escaping seems to be a subset of llvm's, and these are - * for comments anyway. So I'm borrowing this for C here */ - fprintf(f, "\t; e.g. \""); - escputs(f, opt, c_escputc_str, op->example); - fprintf(f, "\""); + /* + * We only show examples when there's a label for the block, + * otherwise it's confusing with the conditionally elided + * optimisations per-instruction below, which can result in + * no block code being emitted for a particular vm op. + */ + if (op->example != NULL) { + /* C's escaping seems to be a subset of llvm's, and these are + * for comments anyway. So I'm borrowing this for C here */ + fprintf(f, "\t; e.g. \""); + escputs(f, opt, c_escputc_str, op->example); + fprintf(f, "\""); - fprintf(f, "\n"); + fprintf(f, "\n"); + } } switch (op->instr) { @@ -669,6 +692,7 @@ fsm_print_llvm(FILE *f, if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } + fprintf(f, "\n"); fprintf(f, "\t]\n"); fprintf(f, "define dso_local %%rt @%smain", prefix); diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index ecab3763a..7848d88b7 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -153,12 +153,16 @@ print_asm_amd64(FILE *f, switch (opt->io) { case FSM_IO_STR: - fprintf(f, "// func %s%s(data string) int\n", prefix, "Match"); + if (opt->comments) { + fprintf(f, "// func %s%s(data string) int\n", prefix, "Match"); + } fprintf(f, "TEXT ·%s(SB), NOSPLIT, $0-24\n", "Match"); break; case FSM_IO_PAIR: - fprintf(f, "// func %s%s(data []byte) int\n", prefix, "Match"); + if (opt->comments) { + fprintf(f, "// func %s%s(data []byte) int\n", prefix, "Match"); + } fprintf(f, "TEXT ·%s%s(SB), NOSPLIT, $0-32\n", prefix, "Match"); break; @@ -195,7 +199,9 @@ print_asm_amd64(FILE *f, for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { fprintf(f, "%sl%u:\n", label_dot, op->index); - } else { + + // TODO: example + } else if (opt->comments) { fprintf(f, "%s l%u\n", comment, op->index); } @@ -210,7 +216,9 @@ print_asm_amd64(FILE *f, } if (op->cmp == VM_CMP_ALWAYS && op->next == NULL) { - fprintf(f, "\t%s elided jmp to %sfinish\n", comment, label_dot); + if (opt->comments) { + fprintf(f, "\t%s elided jmp to %sfinish\n", comment, label_dot); + } } else { const char *jmp_op; diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index e598c5da9..33faafcb8 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -268,9 +268,12 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir if (op->num_incoming > 0 || op == ops) { if (op != ops && can_fallthrough) { fprintf(f, "\t"); - fprintf(f, "S%lu:s -> S%" PRIu32 ":n [ style = bold ]; /* fallthrough */", + fprintf(f, "S%lu:s -> S%" PRIu32 ":n [ style = bold ];", block, op->index); + if (opt->comments) { + fprintf(f, " /* fallthrough */"); + } fprintf(f, "\n"); } @@ -306,11 +309,14 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir } else { /* relative branch within the same block, entry on the east */ /* XXX: would like to make these edges shorter, but I don't know how */ - fprintf(f, "S%lu:b%" PRIu32 ":e -> S%lu:b%" PRIu32 ":e [ constraint = false ]; /* relative */", + fprintf(f, "S%lu:b%" PRIu32 ":e -> S%lu:b%" PRIu32 ":e [ constraint = false ];", block, op->index, block, op->u.br.dest_arg->index); + if (opt->comments) { + fprintf(f, " /* relative */"); + } } fprintf(f, "\n"); diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index ff031a08d..dec2ea652 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -108,7 +108,9 @@ default_reject(FILE *f, const struct fsm_options *opt, static int print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { - fprintf(f, "\t\t/* l%" PRIu32 " */\n", op->index); + if (opt->comments) { + fprintf(f, "\t\t/* l%" PRIu32 " */\n", op->index); + } if (op->example != NULL) { fprintf(f, "\t\t/* e.g. \""); @@ -409,8 +411,10 @@ fsm_print_vmops_main(FILE *f, fprintf(f, "{\n"); fprintf(f, "\tunsigned int i = 0;\n"); - fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); - fprintf(f, "\t/* Initialize to zero to prevent maybe-uninitialized warning. */\n"); + if (opt->comments) { + fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); + fprintf(f, "\t/* Initialize to zero to prevent maybe-uninitialized warning. */\n"); + } fprintf(f, "\tunsigned char c = 0;\n"); fprintf(f, "\tint ok;\n"); fprintf(f, "\tstruct %sop *ops = %sOps;\n", prefix, prefix); @@ -438,7 +442,9 @@ fsm_print_vmops_main(FILE *f, switch (opt->io) { case FSM_IO_PAIR: fprintf(f, "\t\t\tif (p < e) {\n"); - fprintf(f, "\t\t\t\t/* not at EOF */\n"); + if (opt->comments) { + fprintf(f, "\t\t\t\t/* not at EOF */\n"); + } fprintf(f, "\t\t\t\tc = *p++;\n"); fprintf(f, "\t\t\t\ti++;\n"); fprintf(f, "\t\t\t\tcontinue;\n"); @@ -448,7 +454,9 @@ fsm_print_vmops_main(FILE *f, case FSM_IO_STR: fprintf(f, "\t\t\tc = *p++;\n"); fprintf(f, "\t\t\tif (c != '\\0') {\n"); - fprintf(f, "\t\t\t\t/* not at EOF */\n"); + if (opt->comments) { + fprintf(f, "\t\t\t\t/* not at EOF */\n"); + } fprintf(f, "\t\t\t\ti++;\n"); fprintf(f, "\t\t\t\tcontinue;\n"); fprintf(f, "\t\t\t}\n"); diff --git a/src/re/main.c b/src/re/main.c index f9539af51..5226acf36 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -357,20 +357,6 @@ accept_c(FILE *f, const struct fsm_options *opt, fprintf(f, "return %#x;", (unsigned) n); - fprintf(f, " /* "); - - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); - - fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ - - if (i + 1 < count) { - fprintf(f, ", "); - } - } - - fprintf(f, " */"); - return 0; } @@ -398,7 +384,25 @@ accept_rust(FILE *f, const struct fsm_options *opt, fprintf(f, "return Some(%#x)", (unsigned) n); - fprintf(f, " /* "); + return 0; +} + +static int +comment_c(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) +{ + size_t i; + + assert(opt != NULL); + assert(lang_opaque == NULL); + assert(hook_opaque == NULL); + + (void) opt; + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "/* "); for (i = 0; i < count; i++) { assert(ids[i] < matchc); @@ -416,7 +420,7 @@ accept_rust(FILE *f, const struct fsm_options *opt, } static int -accept_llvm(FILE *f, const struct fsm_options *opt, +comment_rust(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { @@ -427,48 +431,10 @@ accept_llvm(FILE *f, const struct fsm_options *opt, assert(hook_opaque == NULL); (void) opt; + (void) lang_opaque; (void) hook_opaque; - switch (opt->ambig) { - case AMBIG_NONE: - fprintf(f, "%%rt true"); - break; - - case AMBIG_ERROR: -// TODO: decide if we deal with this ahead of the call to print or not - if (count > 1) { - errno = EINVAL; - return -1; - } - - fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); - break; - - case AMBIG_EARLIEST: - /* - * The libfsm api guarentees these ids are unique, - * and only appear once each, and are sorted. - */ - fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); - break; - - case AMBIG_MULTIPLE: - fprintf(f, "internal unnamed_addr constant [%zu x i32] [", count); - for (size_t j = 0; j < count; j++) { - fprintf(f, "i32 %u", ids[j]); - if (j + 1 < count) { - fprintf(f, ", "); - } - } - fprintf(f, "]"); - break; - - default: - assert(!"unreached"); - abort(); - } - - fprintf(f, " ; "); + fprintf(f, "/* "); for (i = 0; i < count; i++) { assert(ids[i] < matchc); @@ -480,96 +446,68 @@ accept_llvm(FILE *f, const struct fsm_options *opt, } } + fprintf(f, " */"); + return 0; } static int -accept_dot(FILE *f, const struct fsm_options *opt, +comment_llvm(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { - fsm_state_t s; size_t i; assert(opt != NULL); - assert(lang_opaque != NULL); + assert(lang_opaque == NULL); assert(hook_opaque == NULL); (void) opt; + (void) lang_opaque; (void) hook_opaque; - s = * (fsm_state_t *) lang_opaque; - - fprintf(f, "label = <"); - - if (!opt->anonymous_states) { - fprintf(f, "%u", s); - - if (count > 0) { - fprintf(f, "
"); - } - } + fprintf(f, "; "); for (i = 0; i < count; i++) { assert(ids[i] < matchc); - fprintf(f, "#%u", ids[i]); + fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ if (i + 1 < count) { - fprintf(f, ","); - } - } - - fprintf(f, ">"); - - /* TODO: centralise to libfsm/print/dot.c */ - if (opt->comments) { - fprintf(f, " /* "); - - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); - - fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ - - if (i + 1 < count) { - fprintf(f, ", "); - } + fprintf(f, ", "); } - - fprintf(f, " */"); } return 0; } static int -accept_json(FILE *f, const struct fsm_options *opt, +comment_dot(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) { size_t i; assert(opt != NULL); - assert(lang_opaque == NULL); + assert(lang_opaque != NULL); assert(hook_opaque == NULL); (void) opt; - (void) lang_opaque; (void) hook_opaque; - fprintf(f, "[ "); + fprintf(f, "/* "); for (i = 0; i < count; i++) { assert(ids[i] < matchc); - fprintf(f, "%u", ids[i]); + fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ if (i + 1 < count) { fprintf(f, ", "); } } - fprintf(f, " ]"); + fprintf(f, " */"); return 0; } @@ -1172,33 +1110,31 @@ main(int argc, char *argv[]) } if (fsm_lang != FSM_PRINT_NONE) { - /* TODO: print examples in comments for end states; - * patterns in comments for the whole FSM */ - switch (fsm_lang) { case FSM_PRINT_NONE: break; case FSM_PRINT_C: case FSM_PRINT_VMC: - hooks.accept = accept_c; + hooks.accept = accept_c; + hooks.comment = comment_c; break; case FSM_PRINT_RUST: - hooks.accept = accept_rust; + hooks.accept = accept_rust; + hooks.comment = comment_rust; break; case FSM_PRINT_LLVM: - hooks.accept = accept_llvm; + hooks.comment = comment_llvm; break; case FSM_PRINT_DOT: case FSM_PRINT_VMDOT: - hooks.accept = patterns ? accept_dot : NULL; + hooks.comment = patterns ? comment_dot : NULL; break; case FSM_PRINT_JSON: - hooks.accept = patterns ? accept_json : NULL; break; default: From bb4e81dfc9af6ad2be1cff00b9146180abb0f2f6 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 13:54:04 +0100 Subject: [PATCH 227/373] No need to emit a bitmap here. Originally I'd intended this as a demonstration of how various applications can handle ambiguities differently. But now we have library support for AMBIG_MULTIPLE, I think this is just confusing. --- src/re/main.c | 56 --------------------------------------------------- 1 file changed, 56 deletions(-) diff --git a/src/re/main.c b/src/re/main.c index 5226acf36..bef79ea17 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -333,60 +333,6 @@ conflict(FILE *f, const struct fsm_options *opt, return 0; } -static int -accept_c(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque) -{ - unsigned n; - size_t i; - - assert(opt != NULL); - assert(lang_opaque == NULL); - assert(hook_opaque == NULL); - - (void) opt; - (void) lang_opaque; - (void) hook_opaque; - - n = 0; - - for (i = 0; i < count; i++) { - n |= 1U << ids[i]; - } - - fprintf(f, "return %#x;", (unsigned) n); - - return 0; -} - -static int -accept_rust(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque) -{ - unsigned n; - size_t i; - - assert(opt != NULL); - assert(lang_opaque == NULL); - assert(hook_opaque == NULL); - - (void) opt; - (void) lang_opaque; - (void) hook_opaque; - - n = 0; - - for (i = 0; i < count; i++) { - n |= 1U << ids[i]; - } - - fprintf(f, "return Some(%#x)", (unsigned) n); - - return 0; -} - static int comment_c(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, @@ -1116,12 +1062,10 @@ main(int argc, char *argv[]) case FSM_PRINT_C: case FSM_PRINT_VMC: - hooks.accept = accept_c; hooks.comment = comment_c; break; case FSM_PRINT_RUST: - hooks.accept = accept_rust; hooks.comment = comment_rust; break; From 6c179348ce37259207123364487d4a93af4f2d95 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 14:16:12 +0100 Subject: [PATCH 228/373] Use id count to indicate match failure, and >= 0 means success. The idea here is just to trim down %rt to: ``` %rt = type { ptr, i64 } ``` where we clearly don't need a uint64_t's count of unique ids. I've purposefully not done the same for single-id interfaces. I don't want to mix success/failure with an id *value*, because the values are opaque (i.e. the meaning of an id value is the responsibility of the caller). Whereas here for AMBIG_MULTIPLE I'm mixing success/failure with the count, not with the id values. Suggested by @mcy, thank you. --- src/libfsm/print/llvm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 57f0b4797..c0eb21e7e 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -164,7 +164,7 @@ default_reject(FILE *f, const struct fsm_options *opt, break; case AMBIG_MULTIPLE: - fprintf(f, "%%rt { i1 false, %s poison, i32 poison }", ptr_i32); + fprintf(f, "%%rt { %s poison, i64 -1 }", ptr_i32); break; default: @@ -196,8 +196,8 @@ print_rettype(FILE *f, const char *name, enum fsm_ambig ambig) break; case AMBIG_MULTIPLE: - // success, ids, count - fprintf(f, "{ i1, %s, i32 }", ptr_i32); + // ids, -1/count + fprintf(f, "{ %s, i64 }", ptr_i32); break; default: @@ -676,7 +676,7 @@ fsm_print_llvm(FILE *f, for (size_t i = 0; i < retlist->count; i++) { fprintf(f, "\t "); if (opt->ambig == AMBIG_MULTIPLE) { - fprintf(f, "%%rt { i1 true, %s bitcast ([%zu x i32]* @%sr%zu to %s), i32 %zu }", + fprintf(f, "%%rt { %s bitcast ([%zu x i32]* @%sr%zu to %s), i64 %zu }", ptr_i32, retlist->a[i].count, prefix, i, ptr_i32, retlist->a[i].count); } else { if (-1 == print_hook_accept(f, opt, hooks, From c44bfd2b82185df9ef1c201f76a89a3912634a2c Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 14:40:31 +0100 Subject: [PATCH 229/373] Clarification. --- include/fsm/print.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index c302b9524..434fe3384 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -87,7 +87,17 @@ struct fsm_hooks { /* * Print an FSM to the given file stream. The output is written in the format - * specified. + * specified by the lang enum. + * + * Not all languages support all options, and fsm_print will ENOTSUP where + * these are not possible. This is different to when an option is possible + * but simply not yet implemented, where fsm_print() will print a message + * to stderr and exit. + * + * The code generation for the typical case of matching input require the FSM + * to be a DFA, and will EINVAL if the FSM is not a DFA. As opposed to e.g. + * FSM_PRINT_API, which generates code for other purposes, and does not place + * particular expecations on the FSM. * * The output options may be NULL, indicating to use defaults. * From ba700692862d4d570f8924cf58e876d90ea06549 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 16:29:47 +0100 Subject: [PATCH 230/373] Add AMBIG_MULTIPLE output for Rust. The generated code looks like this: ```rust pub fn fsm_main(input: &str) -> Option<&'static [u32]> { use Label::*; static RET0: [u32; 1] = [1]; static RET1: [u32; 1] = [2]; static RET2: [u32; 3] = [0, 1, 2]; let mut bytes = input.bytes(); pub enum Label { Ls, L0, } let mut l = Ls; loop { match l { Ls => { // e.g. "" let c = match bytes.next() { None => return Some(&RET0) /* "x?" */, Some(c) => c, }; if c != b'x' { return None } let c = match bytes.next() { None => return Some(&RET2) /* "x", "x?", "x+" */, Some(c) => c, }; if c != b'x' { return None } l = L0; continue; } L0 => { // e.g. "xx" let c = match bytes.next() { None => return Some(&RET1) /* "x+" */, Some(c) => c, }; if c != b'x' { return None } l = L0; continue; } } } } ``` --- src/libfsm/print/rust.c | 92 ++++++++++++++++++++++++++++++----------- src/re/main.c | 2 +- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 07b8f7c29..b20d09e4d 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -51,7 +51,8 @@ cmp_operator(int cmp) static int print_ids(FILE *f, - enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count, + size_t i) { switch (ambig) { case AMBIG_NONE: @@ -63,8 +64,8 @@ print_ids(FILE *f, if (count > 1) { errno = EINVAL; return -1; - } - + } + fprintf(f, "return Some(%u)", ids[0]); break; @@ -75,16 +76,16 @@ print_ids(FILE *f, */ fprintf(f, "return Some(%u)", ids[0]); break; - + case AMBIG_MULTIPLE: - assert(!"unimplemented"); - abort(); - + fprintf(f, "return Some(&RET%zu)", i); + break; + default: assert(!"unreached"); abort(); } - + return 0; } @@ -92,29 +93,32 @@ static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, void *lang_opaque, void *hook_opaque) -{ +{ + size_t i; + assert(f != NULL); assert(opt != NULL); - assert(lang_opaque == NULL); - - (void) lang_opaque; + assert(lang_opaque != NULL); + (void) hook_opaque; - - if (-1 == print_ids(f, opt->ambig, ids, count)) { + + i = * (const size_t *) lang_opaque; + + if (-1 == print_ids(f, opt->ambig, ids, count, i)) { return -1; } return 0; } -static int +static int default_reject(FILE *f, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); - + (void) lang_opaque; (void) hook_opaque; @@ -163,17 +167,24 @@ static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct ret_list *retlist, enum dfavm_op_end end_bits) { + size_t i; + switch (end_bits) { case VM_END_FAIL: return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: + assert(op->ret >= retlist->a); + + i = op->ret - retlist->a; + return print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, default_accept, - NULL); + &i); default: assert(!"unreached"); @@ -221,6 +232,20 @@ fsm_print_rustfrag(FILE *f, /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; + if (opt->ambig == AMBIG_MULTIPLE) { + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, " static RET%zu: [u32; %zu] = [", i, retlist->a[i].count); + for (size_t j = 0; j < retlist->a[i].count; j++) { + fprintf(f, "%u", retlist->a[i].ids[j]); + if (j + 1 < retlist->a[i].count) { + fprintf(f, ", "); + } + } + fprintf(f, "];\n"); + } + fprintf(f, "\n"); + } + /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -322,7 +347,7 @@ fsm_print_rustfrag(FILE *f, if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, "{ "); } - if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, retlist, op->u.stop.end_bits)) { return -1; } if (op->cmp != VM_CMP_ALWAYS) { @@ -370,7 +395,7 @@ fsm_print_rustfrag(FILE *f, fprintf(f, " "); fprintf(f, "None => "); - print_end(f, op, opt, hooks, op->u.fetch.end_bits); + print_end(f, op, opt, hooks, retlist, op->u.fetch.end_bits); fprintf(f, ",\n"); fprintf(f, " "); @@ -453,22 +478,19 @@ fsm_print_rust(FILE *f, switch (opt->io) { case FSM_IO_GETC: /* e.g. dbg!(fsm_main("abc".as_bytes().iter().copied())); */ - fprintf(f, "(mut bytes: impl Iterator) -> Option<()> {\n"); - fprintf(f, " use Label::*;\n"); + fprintf(f, "(mut bytes: impl Iterator)"); break; case FSM_IO_STR: /* e.g. dbg!(fsm_main("xabces")); */ - fprintf(f, "(%sinput: &str) -> Option<()> {\n", + fprintf(f, "(%sinput: &str)", has_op(ops, VM_OP_FETCH) ? "" : "_"); - fprintf(f, " use Label::*;\n"); break; case FSM_IO_PAIR: /* e.g. dbg!(fsm_main("xabces".as_bytes())); */ - fprintf(f, "(%sinput: &[u8]) -> Option<()> {\n", + fprintf(f, "(%sinput: &[u8])", has_op(ops, VM_OP_FETCH) ? "" : "_"); - fprintf(f, " use Label::*;\n"); break; default: @@ -476,6 +498,26 @@ fsm_print_rust(FILE *f, exit(EXIT_FAILURE); } + fprintf(f, " -> "); + + switch (opt->ambig) { + case AMBIG_NONE: + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, "Option<()>"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, "Option<&'static [u32]>"); + break; + default: + fprintf(stderr, "unsupported ambig mode\n"); + exit(EXIT_FAILURE); + } + + fprintf(f, " {\n"); + fprintf(f, " use Label::*;\n"); + fsm_print_rustfrag(f, opt, hooks, retlist, ops, cp); fprintf(f, "}\n"); diff --git a/src/re/main.c b/src/re/main.c index bef79ea17..be8059d48 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -373,7 +373,7 @@ comment_rust(FILE *f, const struct fsm_options *opt, size_t i; assert(opt != NULL); - assert(lang_opaque == NULL); + assert(lang_opaque != NULL); assert(hook_opaque == NULL); (void) opt; From 218e90cec9cdfa01d01d7092088c517ac74e54cb Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 7 Aug 2024 20:51:50 +0100 Subject: [PATCH 231/373] Factor out print_ret(). --- src/libfsm/print/go.c | 26 ++++++++++++++++++-------- src/libfsm/print/rust.c | 26 ++++++++++++++++++-------- src/libfsm/print/vmops.c | 26 ++++++++++++++++++-------- 3 files changed, 54 insertions(+), 24 deletions(-) diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 84942ea22..43caa7712 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -232,6 +232,21 @@ print_fetch(FILE *f, const struct fsm_options *opt) } } +static void +print_ret(FILE *f, const unsigned *ids, size_t count) +{ + size_t i; + + fprintf(f, "[]uint{"); + for (i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, "}"); +} + /* TODO: eventually to be non-static */ static int fsm_print_gofrag(FILE *f, @@ -372,14 +387,9 @@ fsm_print_go(FILE *f, if (opt->ambig == AMBIG_MULTIPLE) { for (size_t i = 0; i < retlist->count; i++) { - fprintf(f, "var ret%zu []uint = []uint{", i); - for (size_t j = 0; j < retlist->a[i].count; j++) { - fprintf(f, "%u", retlist->a[i].ids[j]); - if (j + 1 < retlist->a[i].count) { - fprintf(f, ", "); - } - } - fprintf(f, "}\n"); + fprintf(f, "var ret%zu []uint = ", i); + print_ret(f, retlist->a[i].ids, retlist->a[i].count); + fprintf(f, "\n"); } fprintf(f, "\n"); } diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index b20d09e4d..96dca94f0 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -212,6 +212,21 @@ print_fetch(FILE *f) fprintf(f, "bytes.next()"); } +static void +print_ret(FILE *f, const unsigned *ids, size_t count) +{ + size_t i; + + fprintf(f, "["); + for (i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, "];"); +} + /* TODO: eventually to be non-static */ static int fsm_print_rustfrag(FILE *f, @@ -234,14 +249,9 @@ fsm_print_rustfrag(FILE *f, if (opt->ambig == AMBIG_MULTIPLE) { for (size_t i = 0; i < retlist->count; i++) { - fprintf(f, " static RET%zu: [u32; %zu] = [", i, retlist->a[i].count); - for (size_t j = 0; j < retlist->a[i].count; j++) { - fprintf(f, "%u", retlist->a[i].ids[j]); - if (j + 1 < retlist->a[i].count) { - fprintf(f, ", "); - } - } - fprintf(f, "];\n"); + fprintf(f, " static RET%zu: [u32; %zu] = ", i, retlist->a[i].count); + print_ret(f, retlist->a[i].ids, retlist->a[i].count); + fprintf(f, "\n"); } fprintf(f, "\n"); } diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index dec2ea652..3cfc1d7b6 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -201,6 +201,21 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) return 0; } +static void +print_ret(FILE *f, const unsigned *ids, size_t count) +{ + size_t i; + + fprintf(f, "{ (const unsigned []) { "); + for (i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, " }, %zu }", count); +} + int fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, @@ -235,14 +250,9 @@ fsm_print_vmops_c(FILE *f, if (opt->ambig != AMBIG_NONE) { fprintf(f, "struct %sret %sRet[] = {\n", prefix, prefix); for (size_t i = 0; i < retlist->count; i++) { - fprintf(f, "\t{ (const unsigned []) { "); - for (size_t j = 0; j < retlist->a[i].count; j++) { - fprintf(f, "%u", retlist->a[i].ids[j]); - if (j + 1 < retlist->a[i].count) { - fprintf(f, ", "); - } - } - fprintf(f, " }, %zu },\n", retlist->a[i].count); + fprintf(f, "\t"); + print_ret(f, retlist->a[i].ids, retlist->a[i].count); + fprintf(f, ",\n"); } fprintf(f, "};\n"); fprintf(f, "const size_t %sRet_count = sizeof %sRet / sizeof *%sRet;\n", prefix, prefix, prefix); From 719b1b25fcd66f8cdd49e56f78f098cefd577ee5 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 11 Aug 2024 23:14:50 +0100 Subject: [PATCH 232/373] Whoops... wrong exit status. --- src/libfsm/print/c.c | 2 +- src/libfsm/print/vmc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 927c5a875..7410c09e8 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -101,7 +101,7 @@ print_ids(FILE *f, fprintf(f, " };\n"); fprintf(f, "\t\t*ids = a;\n"); fprintf(f, "\t\t*count = %zu;\n", count); - fprintf(f, "\t\treturn 0;\n"); + fprintf(f, "\t\treturn 1;\n"); fprintf(f, "\t}"); break; diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 7ef82cc38..1548c2beb 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -94,7 +94,7 @@ print_ids(FILE *f, fprintf(f, " };\n"); fprintf(f, "\t\t*ids = a;\n"); fprintf(f, "\t\t*count = %zu;\n", count); - fprintf(f, "\t\treturn 0;\n"); + fprintf(f, "\t\treturn 1;\n"); fprintf(f, "\t}"); break; From 690f0e2180e66d4e54779882af6bf45b65a9f321 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 12 Aug 2024 22:24:51 +0100 Subject: [PATCH 233/373] Oops... this should've been done with the API change to return a bool. --- src/libfsm/print/c.c | 8 +++++--- src/libfsm/print/vmc.c | 12 +++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 7410c09e8..1be10df86 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -71,15 +71,17 @@ print_ids(FILE *f, return -1; } - fprintf(f, "return %u;", ids[0]); - break; + /* fallthrough */ case AMBIG_EARLIEST: /* * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, "return %u;", ids[0]); + fprintf(f, "{\n"); + fprintf(f, "\t\t*id = %u;\n", ids[0]); + fprintf(f, "\t\treturn 1;\n"); + fprintf(f, "\t}"); break; case AMBIG_MULTIPLE: diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 1548c2beb..b0fd77a08 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -63,16 +63,18 @@ print_ids(FILE *f, errno = EINVAL; return -1; } - - fprintf(f, "return %u;", ids[0]); - break; - + + /* fallthrough */ + case AMBIG_EARLIEST: /* * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, "return %u;", ids[0]); + fprintf(f, "{\n"); + fprintf(f, "\t\t*id = %u;\n", ids[0]); + fprintf(f, "\t\treturn 1;\n"); + fprintf(f, "\t}"); break; case AMBIG_MULTIPLE: From 0f4d838c85a0a31c6c3250fa975d421cd73b4d54 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 14 Aug 2024 16:54:35 +0100 Subject: [PATCH 234/373] Separate calling points for the comments hook. This gives better control over whitespace and punctuation between the hooks. For example we can output ", \n" with a comma between, and that sits more nicely for single-line comments. Previously these had to be " ,\n" --- include/fsm/print.h | 2 +- src/libfsm/print.c | 29 +++++++++++++++++++---------- src/libfsm/print.h | 6 ++++++ src/libfsm/print/awk.c | 15 +++++++++++++-- src/libfsm/print/c.c | 6 ++++++ src/libfsm/print/dot.c | 10 ++++++++++ src/libfsm/print/fsm.c | 8 +++++++- src/libfsm/print/go.c | 6 ++++++ src/libfsm/print/irdot.c | 2 -- src/libfsm/print/llvm.c | 18 +++++++++++++++++- src/libfsm/print/rust.c | 8 ++++++++ src/libfsm/print/sh.c | 15 +++++++++++++-- src/libfsm/print/vmasm.c | 5 +++++ src/libfsm/print/vmc.c | 15 +++++++++++++-- src/libfsm/print/vmdot.c | 11 +++++++++-- src/libfsm/print/vmops.c | 12 +++++++++++- src/re/main.c | 20 ++++++-------------- 17 files changed, 150 insertions(+), 38 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index 434fe3384..a485d88a9 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -74,7 +74,7 @@ struct fsm_hooks { int (*comment)(FILE *, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque); + void *hook_opaque); /* only called for AMBIG_ERROR; see opt.ambig */ int (*conflict)(FILE *, const struct fsm_options *opt, diff --git a/src/libfsm/print.c b/src/libfsm/print.c index c15b15aef..f8522c678 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -59,8 +59,6 @@ print_hook_accept(FILE *f, void *lang_opaque, void *hook_opaque), void *lang_opaque) { - int r; - assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); @@ -74,25 +72,36 @@ print_hook_accept(FILE *f, } if (hooks->accept != NULL) { - r = hooks->accept(f, opt, ids, count, + return hooks->accept(f, opt, ids, count, lang_opaque, hooks->hook_opaque); } else if (default_accept != NULL) { - r = default_accept(f, opt, ids, count, + return default_accept(f, opt, ids, count, lang_opaque, hooks->hook_opaque); - } else { - r = 0; } - if (r != 0) { - return r; + return 0; +} + +int +print_hook_comment(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const fsm_end_id_t *ids, size_t count) +{ + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + if (opt->ambig == AMBIG_ERROR) { + assert(count <= 1); } if (opt->comments && hooks->comment != NULL) { /* this space is a polyglot */ fprintf(f, " "); - r = hooks->comment(f, opt, ids, count, - lang_opaque, hooks->hook_opaque); + return hooks->comment(f, opt, ids, count, + hooks->hook_opaque); } return 0; diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 8e52e8c41..6fea83274 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -32,6 +32,12 @@ print_hook_accept(FILE *f, void *lang_opaque, void *hook_opaque), void *lang_opaque); +int +print_hook_comment(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const fsm_end_id_t *ids, size_t count); + int print_hook_reject(FILE *f, const struct fsm_options *opt, diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index 410290319..fad52c3ee 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -157,10 +157,21 @@ print_end(FILE *f, const struct dfavm_op_ir *op, return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, hooks, + if (-1 == print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, default_accept, - NULL); + NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } + + return 0; default: assert(!"unreached"); diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 1be10df86..bcea6ab5b 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -382,6 +382,12 @@ print_endstates(FILE *f, return -1; } + if (-1 == print_hook_comment(f, opt, hooks, + ir->states[i].endids.ids, ir->states[i].endids.count)) + { + return -1; + } + fprintf(f, "\n"); } diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index 710b5b01c..f4168e1d3 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -265,6 +265,16 @@ print_dotfrag(FILE *f, return -1; } + if (opt->comments && hooks->comment != NULL) { + fprintf(f, ","); + + if (-1 == print_hook_comment(f, opt, hooks, + ids, count)) + { + return -1; + } + } + fprintf(f, " ];\n"); f_free(fsm->alloc, ids); diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index cd42e01a3..0c93e4338 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -364,7 +364,13 @@ fsm_print_fsm(FILE *f, if (-1 == print_hook_accept(f, opt, hooks, ids, count, default_accept, - NULL)) + NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + ids, count)) { return -1; } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 04fe26c1d..50195b69f 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -202,6 +202,12 @@ print_end(FILE *f, const struct dfavm_op_ir *op, return -1; } + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } + fprintf(f, "\n\t}\n"); return 0; diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 186365203..cff37ee86 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -215,8 +215,6 @@ print_state(FILE *f, fprintf(f, "\n"); } - /* TODO: leaf callback for dot output */ - /* showing hook in addition to existing content */ if (cs->isend && hooks->accept != NULL) { fprintf(f, "\t\t @@ -359,7 +359,7 @@ Set the default &re_dialect.5re; for patterns. - This applies to all files unless overriden by &s.opt;. + This applies to all files unless overridden by &s.opt;. The default is pcre. @@ -424,7 +424,7 @@ requiring all regexps unioned to be non-overlapping. Formally, the requirement is that they are disjoint languages. Erroring for ambiguities applies after multiple regexps are joined, - either by union or by concaternation (&s.opt;). + either by union or by concatenation (&s.opt;).&u.opt; is implied by &n.opt;. From 5fa91fe858ee5161f4c9a09c05b761d61467fa89 Mon Sep 17 00:00:00 2001 From: John VanEnk Date: Fri, 23 Aug 2024 15:04:35 -0700 Subject: [PATCH 247/373] In `closure_free()`, use the FSM allocator instead of `free()`. --- src/fsm/main.c | 2 +- src/libfsm/closure.c | 5 +++-- src/libfsm/epsilons.c | 2 +- src/libfsm/internal.h | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/fsm/main.c b/src/fsm/main.c index 1d60ce5c7..194a18bcd 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -666,7 +666,7 @@ main(int argc, char *argv[]) printf("\n"); } - closure_free(closures, fsm->statecount); + closure_free(fsm, closures, fsm->statecount); return 0; } else { diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index b5b4f969f..3993afcda 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -189,7 +190,7 @@ epsilon_closure(struct fsm *fsm) } void -closure_free(struct state_set **closures, size_t n) +closure_free(struct fsm *fsm, struct state_set **closures, size_t n) { fsm_state_t s; @@ -197,6 +198,6 @@ closure_free(struct state_set **closures, size_t n) state_set_free(closures[s]); } - free(closures); + f_free(fsm->alloc, closures); } diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 51dd3e138..bda3b86b3 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -168,7 +168,7 @@ fsm_remove_epsilons(struct fsm *nfa) res = 1; cleanup: if (eclosures != NULL) { - closure_free(eclosures, state_count); + closure_free(nfa, eclosures, state_count); } return res; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index f45e070ff..1b08767a7 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -92,7 +92,7 @@ struct state_set ** epsilon_closure(struct fsm *fsm); void -closure_free(struct state_set **closures, size_t n); +closure_free(struct fsm *fsm, struct state_set **closures, size_t n); /* * Internal free function that invokes free(3) by default, or a user-provided From 84f383508d4506632dd456893421405ca81a079f Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 13:03:00 +0100 Subject: [PATCH 248/373] Wrong exit status for -G. Spotted by @pierreganty (#478), thank you --- src/libfsm/gen.c | 6 +++--- src/re/main.c | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 730b40724..790579dc8 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -193,7 +193,7 @@ gen_init_outer(struct fsm *fsm, size_t max_length, fsm_generate_matches_cb *cb, void *opaque, bool randomized, unsigned seed) { - int res = 0; + int res = false; if (fsm == NULL || cb == NULL || max_length == 0) { return false; } @@ -222,11 +222,11 @@ gen_init_outer(struct fsm *fsm, size_t max_length, goto cleanup; } - res = 1; + res = true; while (!ctx.done) { if (!gen_iter(&ctx)) { - res = 0; + res = false; break; } } diff --git a/src/re/main.c b/src/re/main.c index 78bc77a2f..1a0d5cbe1 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -1047,7 +1047,11 @@ main(int argc, char *argv[]) } if (generate_bounds > 0) { - return fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt); + if (!fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt)) { + exit(EXIT_FAILURE); + } + + return 0; } if (fsm_lang != FSM_PRINT_NONE) { From 60baaab8addbd06c65d0a2f8fb53eafd9b32c4bc Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 13:03:56 +0100 Subject: [PATCH 249/373] Wrong return type for AMBIG_ERROR/AMBIG_EARLIEST. Spotted by @sw17ch, thank you. --- src/libfsm/print/rust.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index b11fb2c5a..cf855d56d 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -520,14 +520,18 @@ fsm_print_rust(FILE *f, switch (opt->ambig) { case AMBIG_NONE: + fprintf(f, "Option<()>"); + break; + case AMBIG_ERROR: case AMBIG_EARLIEST: - fprintf(f, "Option<()>"); + fprintf(f, "Option"); break; case AMBIG_MULTIPLE: fprintf(f, "Option<&'static [u32]>"); break; + default: fprintf(stderr, "unsupported ambig mode\n"); exit(EXIT_FAILURE); From e7a21b47f40028d698a21733fccf9ecd61beb3cc Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 13:04:40 +0100 Subject: [PATCH 250/373] -C opt-level=3 makes a huge difference here. Spotted by @sw17ch, thank you. --- src/retest/runner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/retest/runner.c b/src/retest/runner.c index 0dac9ab38..49cd611af 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -158,7 +158,7 @@ compile(enum implementation impl, break; case IMPL_RUST: - if (0 != systemf("%s %s --crate-type dylib %s -o %s", + if (0 != systemf("%s %s -C opt-level=3 --crate-type dylib %s -o %s", "rustc", "--edition 2021", tmp_src, tmp_so)) { From edeb29131aca82e830feb55564537d62a30aaf96 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 13:30:47 +0100 Subject: [PATCH 251/373] Missing language. --- src/retest/main.c | 16 ++++++++-------- src/retest/reperf.c | 14 ++++++++++---- src/retest/runner.h | 8 ++++---- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/retest/main.c b/src/retest/main.c index 163e93bb9..53eba216c 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -1270,16 +1270,10 @@ main(int argc, char *argv[]) break; case 'l': - if (strcmp(optarg, "vm") == 0) { - impl = IMPL_INTERPRET; + if (strcmp(optarg, "asm") == 0) { + impl = IMPL_VMASM; } else if (strcmp(optarg, "c") == 0) { impl = IMPL_C; - } else if (strcmp(optarg, "asm") == 0) { - impl = IMPL_VMASM; - } else if (strcmp(optarg, "vmc") == 0) { - impl = IMPL_VMC; - } else if (strcmp(optarg, "vmops") == 0) { - impl = IMPL_VMOPS; } else if (strcmp(optarg, "go") == 0) { impl = IMPL_GO; } else if (strcmp(optarg, "goasm") == 0) { @@ -1288,6 +1282,12 @@ main(int argc, char *argv[]) impl = IMPL_LLVM; } else if (strcmp(optarg, "rust") == 0) { impl = IMPL_RUST; + } else if (strcmp(optarg, "vm") == 0) { + impl = IMPL_INTERPRET; + } else if (strcmp(optarg, "vmc") == 0) { + impl = IMPL_VMC; + } else if (strcmp(optarg, "vmops") == 0) { + impl = IMPL_VMOPS; } else { fprintf(stderr, "unknown argument to -l: %s\n", optarg); usage(); diff --git a/src/retest/reperf.c b/src/retest/reperf.c index 43c294821..4536bcf5f 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -1154,18 +1154,24 @@ main(int argc, char *argv[]) break; case 'l': - if (strcmp(optarg, "vm") == 0) { - impl = IMPL_INTERPRET; + if (strcmp(optarg, "asm") == 0) { + impl = IMPL_VMASM; } else if (strcmp(optarg, "c") == 0) { impl = IMPL_C; + } else if (strcmp(optarg, "go") == 0) { + impl = IMPL_GO; + } else if (strcmp(optarg, "goasm") == 0) { + impl = IMPL_GOASM; } else if (strcmp(optarg, "llvm") == 0) { impl = IMPL_LLVM; } else if (strcmp(optarg, "rust") == 0) { impl = IMPL_RUST; - } else if (strcmp(optarg, "asm") == 0) { - impl = IMPL_VMASM; + } else if (strcmp(optarg, "vm") == 0) { + impl = IMPL_INTERPRET; } else if (strcmp(optarg, "vmc") == 0) { impl = IMPL_VMC; + } else if (strcmp(optarg, "vmops") == 0) { + impl = IMPL_VMOPS; } else { fprintf(stderr, "unknown argument to -l: %s\n", optarg); usage(); diff --git a/src/retest/runner.h b/src/retest/runner.h index 3d53bc511..40b269eb0 100644 --- a/src/retest/runner.h +++ b/src/retest/runner.h @@ -27,13 +27,13 @@ enum error_type { enum implementation { IMPL_C, - IMPL_RUST, - IMPL_LLVM, IMPL_GO, IMPL_GOASM, - IMPL_VMC, - IMPL_VMASM, IMPL_INTERPRET, + IMPL_LLVM, + IMPL_RUST, + IMPL_VMASM, + IMPL_VMC, IMPL_VMOPS, }; From 3769f7743c673e623fd6f5b3519d3d250cfce471 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 15:05:32 +0100 Subject: [PATCH 252/373] Off by one. Previously this gave: ``` ; ./build/bin/re -r literal -G 3 abc ; ./build/bin/re -r literal -G 4 abc abc ; ``` and now -G 3 does construct "abc" (not including the newline, which is a property of the printing, not of the constructed string): ``` ; ./build/bin/re -r literal -G 3 abc abc ; ./build/bin/re -r literal -G 4 abc abc ; ``` and: ``` ; ./build/bin/re -r native -G 3 '^x+$' x xx xxx ``` Spotted by @pierreganty (#478), thank you --- src/libfsm/gen.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 790579dc8..aaf095674 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -139,6 +140,11 @@ int fsm_generate_matches(struct fsm *fsm, size_t max_length, fsm_generate_matches_cb *cb, void *opaque) { + if (max_length == 0) { + errno = EINVAL; + return 0; + } + INIT_TIMERS(); TIME(&pre); int res = gen_init_outer(fsm, max_length, cb, opaque, false, 0); @@ -562,8 +568,8 @@ sfs_step_edges(struct gen_ctx *ctx, struct gen_stack_frame *sf) sf->u.step_edges.initialized = true; } - if (ctx->buf_used + ctx->sed[sf->s_id] >= ctx->max_length) { - LOG(2, "PRUNING due to max length: used:%zu + sed[%d]:%u >= max_length:%zu\n", + if (ctx->buf_used + ctx->sed[sf->s_id] > ctx->max_length) { + LOG(2, "PRUNING due to max length: used:%zu + sed[%d]:%u > max_length:%zu\n", ctx->buf_used, sf->s_id, ctx->sed[sf->s_id], ctx->max_length); sf->t = GEN_SFS_LEAVING_STATE; return true; From 00ad53b60731c7464650af80cfd0fffa38dfa0ee Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 17:53:40 +0100 Subject: [PATCH 253/373] Bump actions for Node 20. --- .github/workflows/ci.yml | 66 ++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce24dbcf3..dfb6182f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Cache checkout - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} @@ -27,7 +27,7 @@ jobs: - name: Checkout if: steps.cache-checkout.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive path: ${{ env.wc }} @@ -49,7 +49,7 @@ jobs: steps: - name: Cache PCRE suite - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pcre with: path: pcre-suite/${{ env.pcre2 }} @@ -67,7 +67,7 @@ jobs: chmod -R ug-w pcre-suite - name: Cache converted PCRE tests - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-cvtpcre with: path: ${{ env.cvtpcre }} @@ -75,7 +75,7 @@ jobs: - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} @@ -155,14 +155,14 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} - name: Cache build - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} @@ -185,7 +185,7 @@ jobs: - name: Get number of CPU cores if: steps.cache-build.outputs.cache-hit != 'true' - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Make @@ -233,7 +233,7 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} @@ -241,7 +241,7 @@ jobs: # An arbitary build. - name: Fetch build - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} @@ -273,7 +273,7 @@ jobs: ${{ matrix.cc }} --version - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Make @@ -292,7 +292,7 @@ jobs: # kmkf duplicate install targets, it's not interesting for libfsm's CI, # so I'm retrying on error here. # github.com/katef/kmkf/issues/14 - name: Install - uses: nick-fields/retry@v2.8.3 + uses: nick-fields/retry@v3 with: timeout_seconds: 10 # required, but not a problem for the kmkf bug max_attempts: 3 @@ -322,7 +322,7 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} @@ -344,14 +344,14 @@ jobs: ${{ matrix.cc }} --version - name: Fetch build - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Test @@ -381,7 +381,7 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} @@ -403,7 +403,7 @@ jobs: ${{ matrix.cc }} --version - name: Fetch build - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} @@ -418,7 +418,7 @@ jobs: # still run fuzzing, just from empty, and do not save their seeds. - name: Restore seeds (mode ${{ matrix.mode }}) if: github.repository == 'katef/libfsm' - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 id: cache-seeds with: path: ${{ env.seeds }}-${{ matrix.mode }} @@ -429,7 +429,7 @@ jobs: run: mkdir -p ${{ env.seeds }}-${{ matrix.mode }} - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Fuzz @@ -455,7 +455,7 @@ jobs: # the same seeds for a given bug. # The explicit cache/restore and cache/save actions are just for that. - name: Save seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 if: always() with: path: ${{ env.seeds }}-${{ matrix.mode }} @@ -463,7 +463,7 @@ jobs: # nothing to do with the caching, I'm uploading the seeds so a developer can grab them to fuzz locally - name: Upload seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: seeds-${{ matrix.mode }}-${{ matrix.debug }} path: ${{ env.seeds }}-${{ matrix.mode }} @@ -513,14 +513,14 @@ jobs: go version - name: Fetch build - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Fetch converted PCRE tests - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-cvtpcre with: path: ${{ env.cvtpcre }} @@ -539,7 +539,7 @@ jobs: steps: - name: Cache docs - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-docs with: path: ${{ env.build }} @@ -554,7 +554,7 @@ jobs: - name: Fetch checkout if: steps.cache-docs.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} @@ -562,7 +562,7 @@ jobs: - name: Get number of CPU cores if: steps.cache-docs.outputs.cache-hit != 'true' - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Test docs @@ -594,7 +594,7 @@ jobs: steps: - name: Cache prefix - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-prefix with: path: ${{ env.prefix }} @@ -608,7 +608,7 @@ jobs: - name: Fetch checkout if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-checkout with: path: ${{ env.wc }} @@ -616,7 +616,7 @@ jobs: - name: Fetch build if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-build with: path: ${{ env.build }} @@ -624,7 +624,7 @@ jobs: - name: Fetch docs if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-docs with: path: ${{ env.build }} @@ -632,7 +632,7 @@ jobs: - name: Get number of CPU cores if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Install @@ -668,7 +668,7 @@ jobs: fpm -v - name: Fetch prefix - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-prefix with: path: ${{ env.prefix }} @@ -691,7 +691,7 @@ jobs: printf "package_file=%s\n" $(basename pkg/*) >> $GITHUB_ENV - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.package_file }} path: pkg/${{ env.package_file }} From d55794381a087368cd4bb1042670daceab53e6f3 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 25 Aug 2024 00:23:25 +0100 Subject: [PATCH 254/373] Happier cache lines. Probably the only change that actually affects something, `struct ir_state` is both important and now fits within a single cache line. I did this by eyeballing: ``` ; bmake -r -j 10 CC=clang DEBUG=1 && (for t in `cat /tmp/tags`; do pahole --class_name $t build/src/*/*.o build/src/adt/*.o ; done ) > /tmp/tt ``` This gives a ~2% improvement on total execution for rx -q /usr/share/bin/words --- include/adt/path.h | 3 ++- include/fsm/options.h | 12 ++++++------ src/adt/edgeset.c | 2 +- src/libfsm/clone.c | 13 +++++++------ src/libfsm/consolidate.c | 7 ++++--- src/libfsm/determinise_internal.h | 3 ++- src/libfsm/endids.c | 7 ++++--- src/libfsm/epsilons.c | 15 ++++++++------- src/libfsm/gen.c | 12 +++++++----- src/libfsm/internal.h | 12 ++++++------ src/libfsm/merge.c | 6 +++--- src/libfsm/print.c | 2 +- src/libfsm/print/c.c | 4 ++-- src/libfsm/print/ir.c | 10 +++++----- src/libfsm/print/ir.h | 9 ++++----- src/libfsm/print/irdot.c | 10 +++++----- src/libfsm/print/irjson.c | 10 +++++----- src/libfsm/vm/ir.c | 6 +++--- src/libfsm/vm/retlist.c | 2 +- src/libfsm/vm/vm.h | 5 +++-- src/libre/ast_analysis.c | 6 +++--- 21 files changed, 82 insertions(+), 74 deletions(-) diff --git a/include/adt/path.h b/include/adt/path.h index 63ea760fb..b9c2a4d5e 100644 --- a/include/adt/path.h +++ b/include/adt/path.h @@ -12,8 +12,9 @@ struct fsm_state; struct path { fsm_state_t state; - struct path *next; char c; + + struct path *next; }; struct path * diff --git a/include/fsm/options.h b/include/fsm/options.h index 90acdcfa8..cd162e9e3 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -81,17 +81,17 @@ struct fsm_options { */ unsigned int group_edges:1; - /* for generated code, what kind of I/O API to generate */ - enum fsm_io io; - - /* for generated code, how to handle multiple endids on an accepting state */ - enum fsm_ambig ambig; - /* a prefix for namespacing generated identifiers. NULL if not required. */ const char *prefix; /* the name of the enclosing package; NULL to use `prefix` (default). */ const char *package_prefix; + + /* for generated code, what kind of I/O API to generate */ + enum fsm_io io; + + /* for generated code, how to handle multiple endids on an accepting state */ + enum fsm_ambig ambig; }; #endif diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index 9658213c8..8e2554c33 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -35,8 +35,8 @@ struct edge_set { size_t ceil; /* nonzero */ size_t count; /* <= ceil */ struct edge_group { - fsm_state_t to; /* distinct */ uint64_t symbols[256/64]; + fsm_state_t to; /* distinct */ } *groups; /* sorted by .to */ }; diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index f6012f277..50f406825 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -86,7 +87,7 @@ fsm_clone(const struct fsm *fsm) struct copy_capture_actions_env { struct fsm *dst; - int ok; + bool ok; }; static int @@ -99,7 +100,7 @@ copy_capture_actions_cb(fsm_state_t state, if (!fsm_capture_add_action(env->dst, state, type, capture_id, to)) { - env->ok = 0; + env->ok = false; } return env->ok; @@ -108,7 +109,7 @@ copy_capture_actions_cb(fsm_state_t state, static int copy_capture_actions(struct fsm *dst, const struct fsm *src) { - struct copy_capture_actions_env env = { NULL, 1 }; + struct copy_capture_actions_env env = { NULL, true }; env.dst = dst; fsm_capture_action_iter(src, @@ -120,7 +121,7 @@ struct copy_end_ids_env { char tag; struct fsm *dst; const struct fsm *src; - int ok; + bool ok; }; static int @@ -134,7 +135,7 @@ copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) #endif if (!fsm_endid_set(env->dst, state, id)) { - env->ok = 0; + env->ok = false; return 0; } @@ -148,7 +149,7 @@ copy_end_ids(struct fsm *dst, const struct fsm *src) env.tag = 'c'; /* for clone */ env.dst = dst; env.src = src; - env.ok = 1; + env.ok = true; fsm_endid_iter(src, copy_end_ids_cb, &env); diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index fc15d4c26..49b08a821 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -39,7 +40,7 @@ struct consolidate_copy_capture_actions_env { struct fsm *dst; size_t mapping_count; const fsm_state_t *mapping; - int ok; + bool ok; }; static int @@ -186,7 +187,7 @@ consolidate_copy_capture_actions_cb(fsm_state_t state, if (!fsm_capture_add_action(env->dst, s, type, capture_id, t)) { - env->ok = 0; + env->ok = false; return 0; } @@ -204,7 +205,7 @@ consolidate_copy_capture_actions(struct fsm *dst, const struct fsm *src, env.dst = dst; env.mapping_count = mapping_count; env.mapping = mapping; - env.ok = 1; + env.ok = true; #if LOG_MAPPING for (i = 0; i < mapping_count; i++) { diff --git a/src/libfsm/determinise_internal.h b/src/libfsm/determinise_internal.h index 8fe35fcd8..8628f71b5 100644 --- a/src/libfsm/determinise_internal.h +++ b/src/libfsm/determinise_internal.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -85,7 +86,7 @@ struct det_copy_capture_actions_env { char tag; struct fsm *dst; struct reverse_mapping *reverse_mappings; - int ok; + bool ok; }; #define MAPPINGSTACK_DEF_CEIL 16 diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 695251f76..62f279ca3 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -755,7 +756,7 @@ struct carry_env { char tag; struct fsm *dst; fsm_state_t dst_state; - int ok; + bool ok; }; static int @@ -767,7 +768,7 @@ carry_iter_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) (void)state; if (!fsm_endid_set(env->dst, env->dst_state, id)) { - env->ok = 0; + env->ok = false; return 0; } return 1; @@ -798,7 +799,7 @@ fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, env.tag = 'C'; env.dst = dst_fsm; env.dst_state = dst_state; - env.ok = 1; + env.ok = true; if (!fsm_isend(src_fsm, s)) { continue; diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 51dd3e138..dcd46ddda 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -28,9 +29,9 @@ struct remap_env { char tag; + bool ok; const struct fsm_alloc *alloc; struct state_set **rmap; - int ok; size_t count; size_t ceil; @@ -182,7 +183,7 @@ remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) struct state_set **rmap; struct state_iter si; fsm_state_t si_s; - struct remap_env env = { 'R', NULL, NULL, 1, 0, 0, NULL }; + struct remap_env env = { 'R', true, NULL, NULL, 0, 0, NULL }; env.alloc = nfa->alloc; /* build a reverse mapping */ @@ -325,17 +326,17 @@ remap_capture_action_cb(fsm_state_t state, return 1; fail: - env->ok = 0; + env->ok = false; return 0; } struct collect_env { char tag; + bool ok; const struct fsm_alloc *alloc; size_t count; size_t ceil; fsm_end_id_t *ids; - int ok; }; static int @@ -353,7 +354,7 @@ collect_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) nids = f_realloc(env->alloc, env->ids, nceil * sizeof(*env->ids)); if (nids == NULL) { - env->ok = 0; + env->ok = false; return 0; } env->ceil = nceil; @@ -390,7 +391,7 @@ carry_endids(struct fsm *fsm, struct state_set *states, if (env.ids == NULL) { return 0; } - env.ok = 1; + env.ok = true; /* collect from states */ for (state_set_reset(states, &it); state_set_next(&it, &s); ) { @@ -407,7 +408,7 @@ carry_endids(struct fsm *fsm, struct state_set *states, /* add them */ for (i = 0; i < env.count; i++) { if (!fsm_endid_set(fsm, dst_state, env.ids[i])) { - env.ok = 0; + env.ok = false; goto cleanup; } } diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 730b40724..5de396058 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -74,22 +74,24 @@ struct gen_ctx { const struct fsm *fsm; size_t max_length; fsm_generate_matches_cb *cb; - void *opaque; + + bool done; size_t buf_ceil; size_t buf_used; char *buf; + void *opaque; + + unsigned depth; + unsigned steps; + /* This is used to avoid useless cycles -- if a state * was reached since the same match_count, then don't * explore it again. */ unsigned match_count; unsigned *state_counts; - unsigned depth; - unsigned steps; - bool done; - /* Shortest end distance for a state: sed[s_id] */ unsigned *sed; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index f45e070ff..fd1ad877e 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -44,11 +44,14 @@ struct state_array; #define FSM_CAPTURE_MAX INT_MAX struct fsm_edge { - fsm_state_t state; /* destination */ + fsm_state_t state:24; /* destination. :24 for packing */ unsigned char symbol; }; struct fsm_state { + struct edge_set *edges; + struct state_set *epsilons; + unsigned int end:1; /* If 0, then this state has no need for checking @@ -57,9 +60,6 @@ struct fsm_state { /* meaningful within one particular transformation only */ unsigned int visited:1; - - struct edge_set *edges; - struct state_set *epsilons; }; struct fsm { @@ -68,10 +68,10 @@ struct fsm { size_t statealloc; /* number of elements allocated */ size_t statecount; /* number of elements populated */ - size_t endcount; + size_t endcount:31; /* :31 for packing */ - fsm_state_t start; unsigned int hasstart:1; + fsm_state_t start; struct fsm_capture_info *capture_info; struct endid_info *endid_info; diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index f9ec44593..b2a556c66 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -27,8 +27,8 @@ struct copy_capture_env { char tag; + bool ok; struct fsm *dst; - int ok; }; static int @@ -134,7 +134,7 @@ copy_capture_cb(fsm_state_t state, if (!fsm_capture_add_action(env->dst, state, type, capture_id, to)) { - env->ok = 0; + env->ok = false; return 0; } @@ -147,7 +147,7 @@ copy_capture_actions(struct fsm *dst, struct fsm *src) struct copy_capture_env env; env.tag = 'C'; env.dst = dst; - env.ok = 1; + env.ok = true; fsm_capture_action_iter(src, copy_capture_cb, &env); diff --git a/src/libfsm/print.c b/src/libfsm/print.c index f8522c678..501b12b96 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -354,7 +354,7 @@ fsm_print(FILE *f, const struct fsm *fsm, continue; } - assert(ir->states[i].endids.count <= 1); + assert(ir->states[i].count <= 1); } } diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index bcea6ab5b..2199b66c3 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -375,7 +375,7 @@ print_endstates(FILE *f, fprintf(f, "\tcase S%u: ", i); if (-1 == print_hook_accept(f, opt, hooks, - ir->states[i].endids.ids, ir->states[i].endids.count, + ir->states[i].ids, ir->states[i].count, default_accept, NULL)) { @@ -383,7 +383,7 @@ print_endstates(FILE *f, } if (-1 == print_hook_comment(f, opt, hooks, - ir->states[i].endids.ids, ir->states[i].endids.count)) + ir->states[i].ids, ir->states[i].count)) { return -1; } diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 457716dcc..b3831b164 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -541,8 +541,8 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) assert(i < ir->n); ir->states[i].isend = fsm_isend(fsm, i); - ir->states[i].endids.ids = NULL; - ir->states[i].endids.count = 0; + ir->states[i].ids = NULL; + ir->states[i].count = 0; if (fsm_isend(fsm, i)) { fsm_end_id_t *ids; @@ -563,8 +563,8 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) assert(res == 1); } - ir->states[i].endids.ids = ids; - ir->states[i].endids.count = count; + ir->states[i].ids = ids; + ir->states[i].count = count; } if (make_state(fsm, i, &ir->states[i]) == -1) { @@ -629,7 +629,7 @@ free_ir(const struct fsm *fsm, struct ir *ir) for (i = 0; i < ir->n; i++) { f_free(fsm->alloc, (void *) ir->states[i].example); - f_free(fsm->alloc, (void *) ir->states[i].endids.ids); + f_free(fsm->alloc, (void *) ir->states[i].ids); switch (ir->states[i].strategy) { case IR_TABLE: diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index 8be4dfc5e..bbfd5006e 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -53,12 +53,11 @@ struct ir_error { struct ir_state { const char *example; - unsigned int isend:1; - struct { - fsm_end_id_t *ids; /* NULL -> 0 */ - size_t count; - } endids; + fsm_end_id_t *ids; /* NULL -> 0 */ + size_t count:31; // :31 for packing + + unsigned int isend:1; enum ir_strategy strategy; union { diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 098a44e30..4541eabc9 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -195,13 +195,13 @@ print_state(FILE *f, fprintf(f, "\t\t \n", ir_indexof(ir, cs), strategy_name(cs->strategy)); - if (cs->isend && cs->endids.count > 0) { + if (cs->isend && cs->count > 0) { fprintf(f, "\t\t + + + + +
L-L%u"); - print_cond(f, op, opt); + print_cond(f, opt, op); fprintf(f, "", op->index); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, opt, ir->states, op, op->u.stop.end_bits)) { return -1; } fprintf(f, "fetch", op->index); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, opt, ir->states, op, op->u.fetch.end_bits)) { return -1; } fprintf(f, ""); - print_cond(f, op, opt); + print_cond(f, opt, op); fprintf(f, "", op->index); print_branch(f, op); @@ -233,19 +238,23 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, static void fsm_print_edges(FILE *f, const struct fsm_options *opt, - const struct dfavm_assembler_ir *a) + const struct ir *ir, const struct dfavm_op_ir *ops) { const struct dfavm_op_ir *op; unsigned long block; int can_fallthrough; + assert(f != NULL); + assert(opt != NULL); + assert(ir != NULL); + (void) opt; can_fallthrough = 1; - for (op = a->linked; op != NULL; op = op->next) { - if (op->num_incoming > 0 || op == a->linked) { - if (op != a->linked && can_fallthrough) { + for (op = ops; op != NULL; op = op->next) { + if (op->num_incoming > 0 || op == ops) { + if (op != ops && can_fallthrough) { fprintf(f, "\t"); fprintf(f, "S%lu:s -> S%" PRIu32 ":n [ style = bold ]; /* fallthrough */", block, @@ -298,49 +307,33 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, /* TODO: eventually to be non-static */ static int -fsm_print_vmdotfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) +fsm_print_vmdotfrag(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); - a = zero; - - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - - if (-1 == fsm_print_nodes(f, ir, opt, &a)) { + if (-1 == fsm_print_nodes(f, opt, ir, ops)) { return -1; } fprintf(f, "\n"); - fsm_print_edges(f, opt, &a); - - dfavm_opasm_finalize_op(&a); + fsm_print_edges(f, opt, ir, ops); return 0; } - -static int -fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt) +int +fsm_print_vmdot(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(ir != NULL); if (opt->fragment) { - if (-1 == fsm_print_vmdotfrag(f, ir, opt)) { + if (-1 == fsm_print_vmdotfrag(f, opt, ir, ops)) { return -1; } } else { @@ -358,7 +351,7 @@ fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options fprintf(f, "\tstart [ shape = none, label = \"\" ];\n"); fprintf(f, "\tstart -> S0:i0:w [ style = bold ];\n"); - if (-1 == fsm_print_vmdotfrag(f, ir, opt)) { + if (-1 == fsm_print_vmdotfrag(f, opt, ir, ops)) { return -1; } @@ -366,34 +359,6 @@ fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_vmdot(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - r = fsm_print_vmdot_complete(f, ir, fsm->opt); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 11972ffa5..e0c7eda38 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -35,7 +35,7 @@ enum vmops_dialect { }; static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { assert(f != NULL); assert(leaf_opaque == NULL); @@ -98,8 +98,10 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const char *prefix, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, + const struct ir_state *ir_states, + const char *prefix, + enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { fprintf(f, "%sactionRET, -1},\n", prefix); @@ -114,7 +116,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "%sactionRET, %td", prefix, op->ir_state - ir->states); + fprintf(f, "%sactionRET, %td", prefix, op->ir_state - ir_states); } fprintf(f, "},\n"); @@ -148,36 +150,23 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) /* TODO: eventually to be non-static */ static int -fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix, +fsm_print_vmopsfrag(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, + const char *prefix, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { - FSM_VM_COMPILE_DEFAULT_FLAGS, - FSM_VM_COMPILE_VM_V1, - NULL - }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - a = zero; - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. * We'll need that in order to be able to use the leaf callback here. */ (void) leaf; (void) leaf_opaque; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { if (-1 == print_label(f, op, opt)) { return -1; @@ -188,7 +177,7 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, prefix, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, prefix, op->u.stop.end_bits)) { return -1; } break; @@ -197,7 +186,7 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, if (-1 == print_fetch(f, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, opt, prefix, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, ir->states, prefix, op->u.fetch.end_bits)) { return -1; } break; @@ -217,38 +206,37 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } } - dfavm_opasm_finalize_op(&a); - return 0; } int -fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) +fsm_print_vmops(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops, + enum vmops_dialect dialect) { - struct ir *ir; + int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); + assert(ir != NULL); - ir = make_ir(fsm); - if (ir == NULL) { - return -1; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; } - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; + if (opt->leaf != NULL) { + leaf = opt->leaf; } else { - prefix = "fsm_"; + leaf = print_leaf; } - if (fsm->opt->fragment) { + if (opt->fragment) { if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, ir, fsm->opt, prefix, - fsm->opt->leaf != NULL ? fsm->opt->leaf : leaf, fsm->opt->leaf_opaque)) + if (-1 == fsm_print_vmopsfrag(f, opt, ir, ops, prefix, + leaf, opt->leaf_opaque)) { return -1; } @@ -261,8 +249,8 @@ fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) fprintf(f, "#include \"%svmops.h\"\n", prefix); fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, ir, fsm->opt, prefix, - fsm->opt->leaf != NULL ? fsm->opt->leaf : leaf, fsm->opt->leaf_opaque)) + if (-1 == fsm_print_vmopsfrag(f, opt, ir, ops, prefix, + leaf, opt->leaf_opaque)) { return -1; } @@ -291,7 +279,7 @@ fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) fprintf(f, "extern struct %sop %sOps[];\n", prefix, prefix); fprintf(f, "\n"); - switch (fsm->opt->io) { + switch (opt->io) { case FSM_IO_PAIR: fprintf(f, "int %smatch(const char *b, const char *e)\n", prefix); break; @@ -312,7 +300,7 @@ fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) fprintf(f, "\tint ok;\n"); fprintf(f, "\tstruct %sop *ops = %sOps;\n", prefix, prefix); - switch (fsm->opt->io) { + switch (opt->io) { case FSM_IO_PAIR: fprintf(f, "\tconst char *p = b;\n"); break; @@ -332,7 +320,7 @@ fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) fprintf(f, "\t\tswitch (ops[i].op) {\n"); fprintf(f, "\t\tcase %sopEOF:\n", prefix); - switch (fsm->opt->io) { + switch (opt->io) { case FSM_IO_PAIR: fprintf(f, "\t\t\tif (p < e) {\n"); fprintf(f, "\t\t\t\t/* not at EOF */\n"); @@ -395,7 +383,7 @@ fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) fprintf(f, "\t\t\tbreak;\n"); fprintf(f, "\t\t}\n"); - switch (fsm->opt->io) { + switch (opt->io) { case FSM_IO_PAIR: fprintf(f, "\t\tr = %smatch(p, p + strlen(p));\n", prefix); break; @@ -414,30 +402,27 @@ fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) } } - free_ir(fsm, ir); - - if (ferror(f)) { - return -1; - } - return 0; } int -fsm_print_vmops_c(FILE *f, const struct fsm *fsm) +fsm_print_vmops_c(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, fsm, VMOPS_C); + return fsm_print_vmops(f, opt, ir, ops, VMOPS_C); } int -fsm_print_vmops_h(FILE *f, const struct fsm *fsm) +fsm_print_vmops_h(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, fsm, VMOPS_H); + return fsm_print_vmops(f, opt, ir, ops, VMOPS_H); } int -fsm_print_vmops_main(FILE *f, const struct fsm *fsm) +fsm_print_vmops_main(FILE *f, const struct fsm_options *opt, + const struct ir *ir, struct dfavm_op_ir *ops) { - return fsm_print_vmops(f, fsm, VMOPS_MAIN); + return fsm_print_vmops(f, opt, ir, ops, VMOPS_MAIN); } From 28c85583df53c21594dbc675bda40ca75736428d Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 12 Jul 2024 10:22:45 +0100 Subject: [PATCH 201/373] Return bool for match/failure from generated code. This drops information about which endids match. My intention is to output that later, but I wanted to think about one thing at a time, because this gets confusing. Here I'm making things worse before they get better. This applies for the codegen that doesn't use the endleaf callbacks. I'm going to call these "default" output. One observation here is that not all FSMs will have endids attached, and we still need to indicate success/failure. So that's why I'm not using the populated endids to indicate a match. --- src/libfsm/print/awk.c | 6 +-- src/libfsm/print/c.c | 17 +++----- src/libfsm/print/go.c | 12 +++--- src/libfsm/print/llvm.c | 31 ++++++-------- src/libfsm/print/rust.c | 12 +++--- src/libfsm/print/sh.c | 2 +- src/libfsm/print/vmasm.c | 91 +++++++++++++++------------------------- src/libfsm/print/vmc.c | 8 ++-- src/libfsm/print/vmops.c | 6 +-- src/retest/runner.c | 25 ++++++----- src/retest/runner.h | 6 +-- 11 files changed, 86 insertions(+), 130 deletions(-) diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index eb62ec6a8..32bac6da0 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -41,9 +41,7 @@ print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaq (void) ids; (void) count; - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return -1;"); + fprintf(f, "return 0;"); return 0; } @@ -93,7 +91,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { - fprintf(f, "return -1"); + fprintf(f, "return 0"); return 0; } diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index a25ce42d0..1d17abb70 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -62,9 +62,7 @@ print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaq (void) count; (void) leaf_opaque; - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return -1; /* leaf */"); + fprintf(f, "return 0; /* leaf */"); return 0; } @@ -121,15 +119,10 @@ print_groups(FILE *f, const struct fsm_options *opt, print_ranges(f, opt, groups[j].ranges, groups[j].n); - /* TODO: pad S%u out to maximum state width */ if (groups[j].to != csi) { fprintf(f, " state = S%u;", groups[j].to); } fprintf(f, " break;\n"); - - /* TODO: if greedy, and fsm_isend(fsm, state->edges[i].sl->state) then: - fprintf(f, " return %s%s;\n", prefix.tok, state->edges[i].sl->state's token); - */ } } @@ -276,7 +269,7 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) /* no end states */ if (!ir_hasend(ir)) { - fprintf(f, "\treturn -1; /* unexpected EOT */\n"); + fprintf(f, "\treturn 0; /* unexpected EOT */\n"); return 0; } @@ -298,12 +291,12 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) return -1; } } else { - fprintf(f, "return %u;", i); + fprintf(f, "return 1;"); } fprintf(f, "\n"); } - fprintf(f, "\tdefault: return -1; /* unexpected EOT */\n"); + fprintf(f, "\tdefault: return 0; /* unexpected EOT */\n"); fprintf(f, "\t}\n"); return 0; @@ -476,7 +469,7 @@ fsm_print_c(FILE *f, const struct fsm_options *opt, } if (ir->n == 0) { - fprintf(f, "\treturn -1; /* no matches */\n"); + fprintf(f, "\treturn 0; /* no matches */\n"); } else { if (-1 == fsm_print_c_body(f, ir, opt, leaf, opt->leaf_opaque)) { return -1; diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 63fb30ba0..25a00db8b 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -38,9 +38,7 @@ print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaq (void) count; (void) leaf_opaque; - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); + fprintf(f, "return 0;"); return 0; } @@ -95,7 +93,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { - fprintf(f, "{\n\t\treturn -1\n\t}\n"); + fprintf(f, "{\n\t\treturn false\n\t}\n"); return 0; } @@ -110,7 +108,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return %td", op->ir_state - ir_states); + fprintf(f, "return true"); } fprintf(f, "\n\t}\n"); @@ -289,11 +287,11 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, switch (opt->io) { case FSM_IO_PAIR: - fprintf(f, "(data []byte) int {\n"); + fprintf(f, "(data []byte) bool {\n"); break; case FSM_IO_STR: - fprintf(f, "(data string) int {\n"); + fprintf(f, "(data string) bool {\n"); break; default: diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index bbc61d97f..7c6b5c52b 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -80,7 +80,7 @@ print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaq /* XXX: this should be FSM_UNKNOWN or something non-EOF, * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); + fprintf(f, "ret false;"); return 0; } @@ -529,12 +529,12 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, * This looks like: * * stop: - * %ret = phi i32 - * [u0x1, %ret0], ; "abc" - * [u0x2, %ret1], ; "xyz" - * [u0x3, %ret2], ; "abc", "xyz" - * [-1, %fail] - * ret i32 %ret + * %ret = phi i1 + * [true, %ret0], ; "abc" + * [true, %ret1], ; "xyz" + * [true, %ret2], ; "abc", "xyz" + * [false, %fail] + * ret i1 %ret * fail: * br label %stop * ret0: @@ -558,7 +558,7 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, * llvm doesn't find this optimisation for us. */ print_label(f, true, "stop"); - fprintf(f, "\t%%ret = phi i32\n"); + fprintf(f, "\t%%ret = phi i1\n"); for (size_t i = 0; i < retlist.count; i++) { fprintf(f, "\t "); @@ -570,12 +570,11 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "[%td, %%ret%zu],\n", - retlist.a[i].ir_state - ir->states, i); + fprintf(f, "[true, %%ret%zu],\n", i); } } - fprintf(f, "\t%s[-1, %%fail]\n", " "); - fprintf(f, "\tret i32 %%ret\n"); + fprintf(f, "\t%s[false, %%fail]\n", " "); + fprintf(f, "\tret i1 %%ret\n"); print_label(f, true, "fail"); fprintf(f, "\tbr "); @@ -716,11 +715,11 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, if (opt->fragment) { fsm_print_llvmfrag(f, opt, ir, ops, cp, leaf, opt->leaf_opaque); - goto error; + return 0; } fprintf(f, "; generated\n"); - fprintf(f, "define dso_local i32 @%smain", prefix); + fprintf(f, "define dso_local i1 @%smain", prefix); switch (opt->io) { case FSM_IO_GETC: @@ -778,9 +777,5 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, fprintf(f, "\n"); return 0; - -error: - - return -1; } diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 0adb8e71f..5eb352ad4 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -41,9 +41,7 @@ print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaq (void) count; (void) leaf_opaque; - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); + fprintf(f, "return None;"); return 0; } @@ -120,7 +118,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return Some(%td)", op->ir_state - ir_states); + fprintf(f, "return Some(())"); } return 0; @@ -406,20 +404,20 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, switch (opt->io) { case FSM_IO_GETC: /* e.g. dbg!(fsm_main("abc".as_bytes().iter().copied())); */ - fprintf(f, "(mut bytes: impl Iterator) -> Option {\n"); + fprintf(f, "(mut bytes: impl Iterator) -> Option<()> {\n"); fprintf(f, " use Label::*;\n"); break; case FSM_IO_STR: /* e.g. dbg!(fsm_main("xabces")); */ - fprintf(f, "(%sinput: &str) -> Option {\n", + fprintf(f, "(%sinput: &str) -> Option<()> {\n", has_op(ops, VM_OP_FETCH) ? "" : "_"); fprintf(f, " use Label::*;\n"); break; case FSM_IO_PAIR: /* e.g. dbg!(fsm_main("xabces".as_bytes())); */ - fprintf(f, "(%sinput: &[u8]) -> Option {\n", + fprintf(f, "(%sinput: &[u8]) -> Option<()> {\n", has_op(ops, VM_OP_FETCH) ? "" : "_"); fprintf(f, " use Label::*;\n"); break; diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 21a8b6105..a637dc46a 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -135,7 +135,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "matched %td", op->ir_state - ir_states); + fprintf(f, "matched"); } return 0; diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 6597f2c53..4d5204dc2 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -156,22 +156,21 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, case VM_OP_STOP: { const char *jmp_op; + bool r = op->u.stop.end_bits == VM_END_SUCC; - if (op->u.stop.end_bits == VM_END_SUCC) { - unsigned end_st = op->ir_state - ir->states; - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $0x%02x, %%%s\n", end_st, ret_reg); - break; - case AMD64_NASM: - fprintf(f, "\tMOV %s, %02xh\n", ret_reg, end_st); - break; - case AMD64_GO: - fprintf(f, "\tMOVQ $0x%02x, %s\n", end_st, ret_reg); - break; - - } + switch (dialect) { + case AMD64_ATT: + fprintf(f, "\tmovl $%d, %%%s\n", r, ret_reg); + break; + case AMD64_NASM: + fprintf(f, "\tMOV %s, %d\n", ret_reg, r); + break; + case AMD64_GO: + fprintf(f, "\tMOVQ $%d, %s\n", r, ret_reg); + break; + } + if (r) { /* endleaf in addition to existing instructions */ if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, @@ -181,19 +180,6 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, return -1; } } - } else { - /* TODO: we don't have a way to override the -1 (or its API) */ - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $-1, %%%s\n", ret_reg); - break; - case AMD64_NASM: - fprintf(f, "\tMOV %s, -1\n", ret_reg); - break; - case AMD64_GO: - fprintf(f, "\tMOVQ $-1, %s\n", ret_reg); - break; - } } if (op->cmp == VM_CMP_ALWAYS && op->next == NULL) { @@ -232,41 +218,32 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, case VM_OP_FETCH: { - if (op->u.fetch.end_bits == VM_END_SUCC) { - unsigned end_st = op->ir_state - ir->states; - - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $0x%02x,%%%s\n", (unsigned)end_st, ret_reg); - break; - - case AMD64_NASM: - fprintf(f, "\tMOV %s, %02xh\n", ret_reg, end_st); - break; - - case AMD64_GO: - fprintf(f, "\tMOVL $0x%02x, %s\n", end_st, ret_reg); - break; - - } - } else { - - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $-1, %%%s\n", ret_reg); - break; + bool r = op->u.fetch.end_bits == VM_END_SUCC; - case AMD64_NASM: - fprintf(f, "\tMOV %s, -1\n", ret_reg); - break; + switch (dialect) { + case AMD64_ATT: + fprintf(f, "\tmovl $%d, %%%s\n", r, ret_reg); + break; + case AMD64_NASM: + fprintf(f, "\tMOV %s, %d\n", ret_reg, r); + break; + case AMD64_GO: + fprintf(f, "\tMOVQ $%d, %s\n", r, ret_reg); + break; + } - case AMD64_GO: - fprintf(f, "\tMOVQ $-1, %s\n", ret_reg); - break; + if (r) { + /* endleaf in addition to existing instructions */ + if (opt->endleaf != NULL) { + if (-1 == opt->endleaf(f, + op->ir_state->endids.ids, op->ir_state->endids.count, + opt->endleaf_opaque)) + { + return -1; + } } } - switch (dialect) { case AMD64_ATT: fprintf(f, "\tcmpq %%%s,%%%s\n", stp_reg, stn_reg); diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 9e22fe3bc..a36c502da 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -38,9 +38,7 @@ print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaq (void) count; (void) leaf_opaque; - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); + fprintf(f, "return 0;"); return 0; } @@ -93,7 +91,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { - fprintf(f, "return -1;"); + fprintf(f, "return 0;"); return 0; } @@ -105,7 +103,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "return %td;", op->ir_state - ir_states); + fprintf(f, "return 1;"); } return 0; diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index e0c7eda38..b90b13074 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -104,7 +104,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { - fprintf(f, "%sactionRET, -1},\n", prefix); + fprintf(f, "%sactionRET, 0},\n", prefix); return 0; } @@ -116,7 +116,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, return -1; } } else { - fprintf(f, "%sactionRET, %td", prefix, op->ir_state - ir_states); + fprintf(f, "%sactionRET, 1", prefix); } fprintf(f, "},\n"); @@ -394,7 +394,7 @@ fsm_print_vmops(FILE *f, const struct fsm_options *opt, errno = ENOTSUP; return -1; } - fprintf(f, "\t\tprintf(\"%%smatch\\n\", (r == -1) ? \"no \" : \"\");\n"); + fprintf(f, "\t\tprintf(\"%%smatch\\n\", r ? \"no \" : \"\");\n"); fprintf(f, "\t}\n"); fprintf(f, "\treturn 0;\n"); fprintf(f, "}\n"); diff --git a/src/retest/runner.c b/src/retest/runner.c index d6b364d43..95f89aa91 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -115,13 +115,12 @@ print(const struct fsm *fsm, enum implementation impl, fprintf(f, "use std::slice;\n"); fprintf(f, "\n"); - // XXX: fsm_main shouldn't return Option, Option or a bitfield type fprintf(f, "#[no_mangle]\n"); - fprintf(f, "pub extern \"C\" fn retest_trampoline(ptr: *const c_uchar, len: usize) -> i64 {\n"); + fprintf(f, "pub extern \"C\" fn retest_trampoline(ptr: *const c_uchar, len: usize) -> bool {\n"); fprintf(f, " let a: &[u8] = unsafe { slice::from_raw_parts(ptr, len as usize) };\n"); fprintf(f, " match fsm_main(a) {\n"); - fprintf(f, " Some(u) => u as i64,\n"); // XXX: we lose info for the i64 ABI - fprintf(f, " None => -1,\n"); + fprintf(f, " Some(()) => true,\n"); + fprintf(f, " None => false,\n"); fprintf(f, " }\n"); fprintf(f, "}\n"); } @@ -369,21 +368,21 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation case IMPL_RUST: r->u.impl_rust.h = h; - r->u.impl_rust.func = (int64_t (*)(const unsigned char *, size_t)) (uintptr_t) dlsym(h, "retest_trampoline"); + r->u.impl_rust.func = (bool (*)(const unsigned char *, size_t)) (uintptr_t) dlsym(h, "retest_trampoline"); break; case IMPL_LLVM: r->u.impl_llvm.h = h; - r->u.impl_llvm.func = (int (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm_main"); + r->u.impl_llvm.func = (bool (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm_main"); break; case IMPL_GO: case IMPL_GOASM: r->u.impl_go.h = h; - r->u.impl_go.func = (int64_t (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "retest_trampoline"); + r->u.impl_go.func = (bool (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "retest_trampoline"); if (r->u.impl_go.func == NULL) { /* Sometime we need a leading underscore. */ - r->u.impl_go.func = (int64_t (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "_retest_trampoline"); + r->u.impl_go.func = (bool (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "_retest_trampoline"); } break; @@ -496,24 +495,24 @@ fsm_runner_run(const struct fsm_runner *r, const char *s, size_t n) case IMPL_VMC: case IMPL_VMOPS: assert(r->u.impl_c.func != NULL); - return r->u.impl_c.func(s, s+n) >= 0; + return r->u.impl_c.func(s, s + n); case IMPL_RUST: assert(r->u.impl_rust.func != NULL); - return r->u.impl_rust.func((const unsigned char *)s, n) >= 0; + return r->u.impl_rust.func((const unsigned char *) s, n); case IMPL_LLVM: assert(r->u.impl_llvm.func != NULL); - return r->u.impl_llvm.func(s, s+n) >= 0; + return r->u.impl_llvm.func(s, s + n); case IMPL_GO: case IMPL_GOASM: assert(r->u.impl_go.func != NULL); - return r->u.impl_go.func((const unsigned char *)s, n) >= 0; + return r->u.impl_go.func((const unsigned char *) s, n); case IMPL_VMASM: assert(r->u.impl_asm.func != NULL); - return r->u.impl_asm.func((const unsigned char *)s, n) >= 0; + return r->u.impl_asm.func((const unsigned char *) s, n); case IMPL_INTERPRET: assert(r->u.impl_vm.vm != NULL); diff --git a/src/retest/runner.h b/src/retest/runner.h index abe88435f..3bd1531ab 100644 --- a/src/retest/runner.h +++ b/src/retest/runner.h @@ -48,17 +48,17 @@ struct fsm_runner { struct { /* pub extern "C" fn f(ptr: *const c_uchar, len: usize) -> usize */ void *h; - int64_t (*func)(const unsigned char *, size_t); + bool (*func)(const unsigned char *, size_t); } impl_rust; struct { void *h; - int (*func)(const char *, const char *); + bool (*func)(const char *, const char *); } impl_llvm; struct { void *h; - int64_t (*func)(const unsigned char *, int64_t); + bool (*func)(const unsigned char *, int64_t); } impl_go; struct { From 97ec4e3416a98da29165e38ccb0795687966b32b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 12 Jul 2024 18:45:10 +0100 Subject: [PATCH 202/373] No need for ir_state here. I've carried over only the fields we actually use to the vm ir struct. This is a bit brutal, I'm trying to avoid sacrificing functionality, but when it comes to a choice, at the moment I care more about keeping things separate than not cutting features. However the only thing I think I dropped is the state numbering for -l vmdot output. --- src/libfsm/print.c | 10 +----- src/libfsm/print.h | 2 +- src/libfsm/print/awk.c | 20 ++++------- src/libfsm/print/go.c | 29 ++++++---------- src/libfsm/print/llvm.c | 69 +++++++++--------------------------- src/libfsm/print/rust.c | 24 +++++-------- src/libfsm/print/sh.c | 17 ++++----- src/libfsm/print/vmasm.c | 44 +++++++++-------------- src/libfsm/print/vmc.c | 29 ++++++---------- src/libfsm/print/vmdot.c | 75 ++++++++++++---------------------------- src/libfsm/print/vmops.c | 39 ++++++++------------- src/libfsm/vm/ir.c | 48 +++++++++++++++---------- src/libfsm/vm/v1.c | 2 ++ src/libfsm/vm/v2.c | 2 ++ src/libfsm/vm/vm.h | 11 ++++-- 15 files changed, 160 insertions(+), 261 deletions(-) diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 1f3a461ef..789d2825a 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -101,15 +101,7 @@ fsm_print(FILE *f, const struct fsm *fsm, enum fsm_print_lang lang) } if (print_vm != NULL) { - /* - * TODO: I'm passing ir here for sake of emitting a state index by - * op->ir_state - ir_states for returning from the generated code. - * - * I want to remove this after moving the generated API to just - * return a boolean, and for callers to think in terms of endid - * sets instead. - */ - r = print_vm(f, fsm->opt, ir, a.linked); + r = print_vm(f, fsm->opt, a.linked); } dfavm_opasm_finalize_op(&a); diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 7c4a6a976..72e028aec 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -13,7 +13,7 @@ struct dfavm_op_ir; typedef int fsm_print_f(FILE *f, const struct fsm_options *opt, const struct fsm *fsm); typedef int ir_print_f(FILE *f, const struct fsm_options *opt, const struct ir *ir); -typedef int vm_print_f(FILE *f, const struct fsm_options *opt, const struct ir *ir, struct dfavm_op_ir *ops); +typedef int vm_print_f(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops); vm_print_f fsm_print_amd64_att; vm_print_f fsm_print_amd64_go; diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index 32bac6da0..bf06d471f 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -27,8 +27,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - #define START UINT32_MAX static int @@ -97,7 +95,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -110,10 +108,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, /* awk can't return an array */ fprintf(f, "return \""); - for (i = 0; i < op->ir_state->endids.count; i++) { - fprintf(f, "%u", op->ir_state->endids.ids[i]); + for (i = 0; i < op->endids.count; i++) { + fprintf(f, "%u", op->endids.ids[i]); - if (i < op->ir_state->endids.count - 1) { + if (i < op->endids.count - 1) { fprintf(f, ","); } } @@ -190,9 +188,9 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, print_label(f, op); fprintf(f, ":"); - if (op->ir_state != NULL && op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " /* e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\" */"); } @@ -247,8 +245,7 @@ fsm_print_awkfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_awk(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_awk(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; @@ -256,9 +253,6 @@ fsm_print_awk(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); - - (void) ir; if (opt->prefix != NULL) { prefix = opt->prefix; diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 25a00db8b..05ce812e6 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -26,8 +26,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - static int print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { @@ -66,10 +64,10 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt { fprintf(f, "l%" PRIu32 ":", op->index); - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " // e.g. \""); /* Go's string escape rules are a superset of C's. */ - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); } } @@ -89,7 +87,6 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - const struct ir_state *ir_states, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { @@ -102,7 +99,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -138,8 +135,7 @@ print_fetch(FILE *f, const struct fsm_options *opt) /* TODO: eventually to be non-static */ static int -fsm_print_gofrag(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +fsm_print_gofrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -148,7 +144,6 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); assert(cp != NULL); /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. @@ -184,7 +179,7 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, } else { switch (opt->io) { case FSM_IO_PAIR: - if (ir->n > 0) { + if (ops != NULL) { /* start idx at -1 unsigned so after first increment we're correct at index 0 */ fprintf(f, "\tvar idx = ^uint(0)\n"); fprintf(f, "\n"); @@ -192,7 +187,7 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, break; case FSM_IO_STR: - if (ir->n > 0) { + if (ops != NULL) { /* start idx at -1 unsigned so after first increment we're correct at index 0 */ fprintf(f, "\tvar idx = ^uint(0)\n"); fprintf(f, "\n"); @@ -216,12 +211,12 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - print_end(f, op, opt, ir->states, op->u.stop.end_bits); + print_end(f, op, opt, op->u.stop.end_bits); break; case VM_OP_FETCH: print_fetch(f, opt); - print_end(f, op, opt, ir->states, op->u.fetch.end_bits); + print_end(f, op, opt, op->u.fetch.end_bits); break; case VM_OP_BRANCH: @@ -241,8 +236,7 @@ fsm_print_gofrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_go(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_go(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; @@ -253,7 +247,6 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -274,7 +267,7 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, } if (opt->fragment) { - if (-1 == fsm_print_gofrag(f, opt, ir, ops, cp, + if (-1 == fsm_print_gofrag(f, opt, ops, cp, leaf, opt->leaf_opaque)) { return -1; @@ -299,7 +292,7 @@ fsm_print_go(FILE *f, const struct fsm_options *opt, return -1; } - if (-1 == fsm_print_gofrag(f, opt, ir, ops, cp, + if (-1 == fsm_print_gofrag(f, opt, ops, cp, leaf, opt->leaf_opaque)) { return -1; diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 7c6b5c52b..c94e511aa 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -29,8 +29,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - #define OPAQUE_POINTERS 1 #ifdef OPAQUE_POINTERS // llvm >= 15 @@ -312,7 +310,6 @@ print_fetch(FILE *f, const struct fsm_options *opt, struct ret { size_t count; const fsm_end_id_t *ids; - const struct ir_state *ir_state; // TODO: remove when we return endids only }; struct ret_list { @@ -322,8 +319,7 @@ struct ret_list { static bool append_ret(struct ret_list *list, - const fsm_end_id_t *ids, size_t count, - const struct ir_state *ir_state) + const fsm_end_id_t *ids, size_t count) { const size_t low = 16; /* must be power of 2 */ const size_t factor = 2; /* must be even */ @@ -355,11 +351,6 @@ append_ret(struct ret_list *list, list->a[list->count].ids = ids; list->a[list->count].count = count; - /* .ir_state is not part of the "key" for struct ret, - * we ignore it for comparisons and effectively pick - * an arbitrary ir_state for output */ - list->a[list->count].ir_state = ir_state; - list->count++; return true; @@ -374,25 +365,9 @@ cmp_ret_by_endid(const void *pa, const void *pb) if (a->count < b->count) { return -1; } if (a->count > b->count) { return +1; } - /* .ir_state intentionally ignored */ - return memcmp(a->ids, b->ids, a->count * sizeof *a->ids); } -static int -cmp_ret_by_state(const void *pa, const void *pb) -{ - const struct ret *a = pa; - const struct ret *b = pb; - - if (a->ir_state < b->ir_state) { return -1; } - if (a->ir_state > b->ir_state) { return +1; } - - /* .ids intentionally ignored */ - - return 0; -} - static struct ret * find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, int (*cmp)(const void *pa, const void *pb)) @@ -402,9 +377,8 @@ find_ret(const struct ret_list *list, const struct dfavm_op_ir *op, assert(op != NULL); assert(cmp != NULL); - key.count = op->ir_state->endids.count; - key.ids = op->ir_state->endids.ids; - key.ir_state = op->ir_state; + key.count = op->endids.count; + key.ids = op->endids.ids; return bsearch(&key, list->a, list->count, sizeof *list->a, cmp); } @@ -442,10 +416,7 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) abort(); } - if (!append_ret(list, - op->ir_state->endids.ids, op->ir_state->endids.count, - op->ir_state)) - { + if (!append_ret(list, op->endids.ids, op->endids.count)) { return false; } } @@ -455,8 +426,7 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) /* TODO: eventually to be non-static */ static int -fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -466,7 +436,6 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); assert(cp != NULL); /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. @@ -492,8 +461,7 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, build_retlist(&retlist, ops); /* sort for both dedup and bsearch */ - qsort(retlist.a, retlist.count, sizeof *retlist.a, - opt->endleaf != NULL ? cmp_ret_by_endid : cmp_ret_by_state); + qsort(retlist.a, retlist.count, sizeof *retlist.a, cmp_ret_by_endid); /* * If we're going by endleaf, we deal with endids. @@ -595,11 +563,11 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, print_label(f, true, "l%" PRIu32, op->index); } - if (op->ir_state != NULL && op->ir_state->example != NULL) { + if (op->example != NULL) { /* C's escaping seems to be a subset of llvm's, and these are * for comments anyway. So I'm borrowing this for C here */ fprintf(f, "\t; e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); fprintf(f, "\n"); @@ -624,12 +592,11 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, /* handled above */ } else { assert(retlist.count > 0); - const struct ret *ret = find_ret(&retlist, op, - opt->endleaf != NULL ? cmp_ret_by_endid : cmp_ret_by_state); + const struct ret *ret = find_ret(&retlist, op, cmp_ret_by_endid); assert(ret != NULL); assert(ret >= retlist.a && ret <= (retlist.a + retlist.count)); - assert(ret->count == op->ir_state->endids.count); - assert(0 == memcmp(ret->ids, op->ir_state->endids.ids, ret->count)); + assert(ret->count == op->endids.count); + assert(0 == memcmp(ret->ids, op->endids.ids, ret->count)); fprintf(f, "\tbr "); print_label(f, false, "ret%u", ret - retlist.a); fprintf(f, "\n"); @@ -643,12 +610,11 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, /* handled in print_fetch() */ } else { assert(retlist.count > 0); - const struct ret *ret = find_ret(&retlist, op, - opt->endleaf != NULL ? cmp_ret_by_endid : cmp_ret_by_state); + const struct ret *ret = find_ret(&retlist, op, cmp_ret_by_endid); assert(ret != NULL); assert(ret >= retlist.a && ret <= (retlist.a + retlist.count)); - assert(ret->count == op->ir_state->endids.count); - assert(0 == memcmp(ret->ids, op->ir_state->endids.ids, ret->count)); + assert(ret->count == op->endids.count); + assert(0 == memcmp(ret->ids, op->endids.ids, ret->count)); fprintf(f, "\tbr "); print_label(f, false, "ret%u", ret - retlist.a); fprintf(f, "\n"); @@ -684,8 +650,7 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_llvm(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; @@ -713,7 +678,7 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, } if (opt->fragment) { - fsm_print_llvmfrag(f, opt, ir, ops, cp, + fsm_print_llvmfrag(f, opt, ops, cp, leaf, opt->leaf_opaque); return 0; } @@ -770,7 +735,7 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, exit(EXIT_FAILURE); } - fsm_print_llvmfrag(f, opt, ir, ops, cp, + fsm_print_llvmfrag(f, opt, ops, cp, leaf, opt->leaf_opaque); fprintf(f, "}\n"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 5eb352ad4..da5b15a93 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -27,8 +27,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - #define START UINT32_MAX static int @@ -102,7 +100,6 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - const struct ir_state *ir_states, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { @@ -112,7 +109,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -146,8 +143,7 @@ print_fetch(FILE *f) /* TODO: eventually to be non-static */ static int -fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -157,7 +153,6 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); assert(cp != NULL); /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. @@ -248,11 +243,11 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, print_label(f, op); fprintf(f, " => {"); - if (op->ir_state != NULL && op->ir_state->example != NULL) { + if (op->example != NULL) { /* C's escaping seems to be a subset of rust's, and these are * for comments anyway. So I'm borrowing this for C here */ fprintf(f, " // e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); } @@ -269,7 +264,7 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, "{ "); } - if (-1 == print_end(f, op, opt, ir->states, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { return -1; } fprintf(f, ";"); @@ -318,7 +313,7 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, fprintf(f, " "); fprintf(f, "None => "); - print_end(f, op, opt, ir->states, op->u.fetch.end_bits); + print_end(f, op, opt, op->u.fetch.end_bits); fprintf(f, ",\n"); fprintf(f, " "); @@ -363,8 +358,7 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_rust(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; @@ -392,7 +386,7 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, } if (opt->fragment) { - fsm_print_rustfrag(f, opt, ir, ops, cp, + fsm_print_rustfrag(f, opt, ops, cp, leaf, opt->leaf_opaque); goto error; } @@ -427,7 +421,7 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, opt, ir, ops, cp, + fsm_print_rustfrag(f, opt, ops, cp, leaf, opt->leaf_opaque); fprintf(f, "}\n"); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index a637dc46a..89ec3897f 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -27,8 +27,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - static const char * str_operator(int cmp) { @@ -76,9 +74,9 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt * arbitrary characters, and nobody can read $'...' style strings. * So I'm just using C-style strings here because it's simple enough. */ - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " # e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); } } @@ -119,7 +117,6 @@ print_cond(FILE *f, const struct dfavm_op_ir *op) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - const struct ir_state *ir_states, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { @@ -129,7 +126,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -154,14 +151,12 @@ print_fetch(FILE *f) } int -fsm_print_sh(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_sh(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { struct dfavm_op_ir *op; assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); if (opt->io != FSM_IO_STR) { errno = ENOTSUP; @@ -215,14 +210,14 @@ fsm_print_sh(FILE *f, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op); - if (-1 == print_end(f, op, opt, ir->states, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { return -1; } break; case VM_OP_FETCH: print_fetch(f); - if (-1 == print_end(f, op, opt, ir->states, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, op->u.fetch.end_bits)) { return -1; } break; diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 4d5204dc2..0ae125f8f 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -25,8 +25,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - enum asm_dialect { AMD64_ATT, AMD64_NASM, @@ -34,8 +32,7 @@ enum asm_dialect { }; static int -print_asm_amd64(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *prefix, enum asm_dialect dialect) { @@ -48,7 +45,6 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, const char *label_dot = "."; - const struct ir_state *curr_st = NULL; const struct dfavm_op_ir *op; char *comment; @@ -62,7 +58,6 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); switch (dialect) { case AMD64_ATT: comment = "#"; break; @@ -141,15 +136,10 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, } for (op = ops; op != NULL; op = op->next) { - if (curr_st != op->ir_state) { - unsigned state = op->ir_state - ir->states; - if (op->num_incoming > 0) { - fprintf(f, "%sstate_%u:\n", label_dot, state); - } else { - fprintf(f, "%s state %u\n", comment, state); - } - - curr_st = op->ir_state; + if (op->num_incoming > 0) { + fprintf(f, "%sl%u:\n", label_dot, op->index); + } else { + fprintf(f, "%s l%u\n", comment, op->index); } switch (op->instr) { @@ -174,7 +164,7 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, /* endleaf in addition to existing instructions */ if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -236,7 +226,7 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, /* endleaf in addition to existing instructions */ if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -273,7 +263,6 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, case VM_OP_BRANCH: { const char *jmp_op; - unsigned dest_state = op->u.br.dest_arg->ir_state - ir->states; if (op->cmp != VM_CMP_ALWAYS) { switch (dialect) { @@ -301,7 +290,7 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, case VM_CMP_NE: jmp_op = (dialect == AMD64_ATT) ? "jne " : "JNE "; break; } - fprintf(f, "\t%-3s %sstate_%u\n", jmp_op, label_dot, dest_state); + fprintf(f, "\t%-3s %sl%u\n", jmp_op, label_dot, op->u.br.dest_arg->index); } break; @@ -347,8 +336,7 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, } static int -print_vmasm_encoding(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +print_vmasm_encoding(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, enum asm_dialect dialect) { const char *prefix; @@ -374,27 +362,27 @@ print_vmasm_encoding(FILE *f, const struct fsm_options *opt, prefix = "fsm_"; } - return print_asm_amd64(f, opt, ir, ops, prefix, dialect); + return print_asm_amd64(f, opt, ops, prefix, dialect); } int fsm_print_amd64_att(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) + struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, ir, ops, AMD64_ATT); + return print_vmasm_encoding(f, opt, ops, AMD64_ATT); } int fsm_print_amd64_nasm(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) + struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, ir, ops, AMD64_NASM); + return print_vmasm_encoding(f, opt, ops, AMD64_NASM); } int fsm_print_amd64_go(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) + struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, ir, ops, AMD64_GO); + return print_vmasm_encoding(f, opt, ops, AMD64_GO); } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index a36c502da..ba5c61995 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -26,8 +26,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - static int print_leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) { @@ -66,9 +64,9 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt { fprintf(f, "l%" PRIu32 ":", op->index); - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " /* e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\" */"); } } @@ -87,7 +85,6 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) static int print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - const struct ir_state *ir_states, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { @@ -97,7 +94,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; @@ -280,8 +277,7 @@ walk_sequence(struct dfavm_op_ir *op, /* TODO: eventually to be non-static */ static int -fsm_print_cfrag(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops, +fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, const char *cp, int (*leaf)(FILE *, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque), const void *leaf_opaque) @@ -290,7 +286,6 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); assert(cp != NULL); /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. @@ -359,7 +354,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, ir->states, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { return -1; } break; @@ -377,7 +372,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, fprintf(f, "if (e - p < %zu || 0 != memcmp(p, \"", n); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, ir->states, end_bits)) { + if (-1 == print_end(f, NULL, opt, end_bits)) { return -1; } fprintf(f, "\n"); @@ -390,7 +385,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, fprintf(f, "if (0 != strncmp(p, \""); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, ir->states, end_bits)) { + if (-1 == print_end(f, NULL, opt, end_bits)) { return -1; } fprintf(f, "\n"); @@ -401,7 +396,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, op = tail; } else { print_fetch(f, opt); - if (-1 == print_end(f, op, opt, ir->states, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, op->u.fetch.end_bits)) { return -1; } if (opt->io == FSM_IO_PAIR) { @@ -438,8 +433,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, } int -fsm_print_vmc(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { int (*leaf)(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque); const char *prefix; @@ -449,7 +443,6 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -464,7 +457,7 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, } if (opt->fragment) { - if (-1 == fsm_print_cfrag(f, opt, ir, ops, cp, + if (-1 == fsm_print_cfrag(f, opt, ops, cp, leaf, opt->leaf_opaque)) { return -1; @@ -489,7 +482,7 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, break; } - if (-1 == fsm_print_cfrag(f, opt, ir, ops, cp, + if (-1 == fsm_print_cfrag(f, opt, ops, cp, leaf, opt->leaf_opaque)) { return -1; diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 460cd6e72..c647cb988 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -26,17 +26,6 @@ #include "libfsm/vm/vm.h" -#include "ir.h" - -static unsigned int -ir_indexof(const struct ir *ir, const struct ir_state *cs) -{ - assert(ir != NULL); - assert(cs != NULL); - - return cs - &ir->states[0]; -} - static const char * cmp_operator(int cmp) { @@ -84,7 +73,6 @@ print_cond(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op) static int print_end(FILE *f, const struct fsm_options *opt, - const struct ir_state *ir_states, const struct dfavm_op_ir *op, enum dfavm_op_end end_bits) { if (end_bits == VM_END_FAIL) { @@ -94,21 +82,21 @@ print_end(FILE *f, const struct fsm_options *opt, if (opt->endleaf != NULL) { if (-1 == opt->endleaf(f, - op->ir_state->endids.ids, op->ir_state->endids.count, + op->endids.ids, op->endids.count, opt->endleaf_opaque)) { return -1; } } else { - fprintf(f, "ret %td", op->ir_state - ir_states); + fprintf(f, "match"); - if (op->ir_state->endids.count > 0) { + if (op->endids.count > 0) { fprintf(f, " / "); - for (size_t i = 0; i < op->ir_state->endids.count; i++) { - fprintf(f, "#%u", op->ir_state->endids.ids[i]); + for (size_t i = 0; i < op->endids.count; i++) { + fprintf(f, "#%u", op->endids.ids[i]); - if (i < op->ir_state->endids.count - 1) { + if (i < op->endids.count - 1) { fprintf(f, " "); } } @@ -125,20 +113,18 @@ print_branch(FILE *f, const struct dfavm_op_ir *op) } static int -fsm_print_nodes(FILE *f, const struct fsm_options *opt, - const struct ir *ir, struct dfavm_op_ir *ops) +fsm_print_nodes(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) { struct dfavm_op_ir *op; - const struct ir_state *ir_state; + const char *example; unsigned block; assert(f != NULL); assert(opt != NULL); - assert(ir != NULL); block = 0; - ir_state = NULL; + example = NULL; for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0 || op == ops) { @@ -158,24 +144,23 @@ fsm_print_nodes(FILE *f, const struct fsm_options *opt, } else { fprintf(f, "\t\t\tL%uStateCondOp
"); + fprintf(f, "\t\t\t"); fprintf(f, "e.g. \""); - escputs(f, opt, dot_escputc_html, op->ir_state->example); + escputs(f, opt, dot_escputc_html, op->example); fprintf(f, "\""); fprintf(f, "
"); - if (op->ir_state != NULL) { - fprintf(f, op->ir_state->isend ? "(S%u)" : "S%u", ir_indexof(ir, op->ir_state)); - } else { - fprintf(f, "(none)"); - } - fprintf(f, ""); print_cond(f, opt, op); fprintf(f, "", op->index); - if (-1 == print_end(f, opt, ir->states, op, op->u.stop.end_bits)) { + if (-1 == print_end(f, opt, op, op->u.stop.end_bits)) { return -1; } fprintf(f, "fetch", op->index); - if (-1 == print_end(f, opt, ir->states, op, op->u.fetch.end_bits)) { + if (-1 == print_end(f, opt, op, op->u.fetch.end_bits)) { return -1; } fprintf(f, "
"); - if (-1 == opt->endleaf(f, + + if (-1 == print_hook_accept(f, opt, cs->endids.ids, cs->endids.count, - opt->endleaf_opaque)) + NULL, + NULL)) { return -1; } + fprintf(f, "
"); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, cs->endids.ids, cs->endids.count, NULL, NULL)) @@ -326,13 +328,16 @@ print_state(FILE *f, const struct fsm_options *opt, } int -fsm_print_ir(FILE *f, const struct fsm_options *opt, +fsm_print_ir(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir *ir) { size_t i; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(ir != NULL); fprintf(f, "digraph G {\n"); @@ -349,7 +354,7 @@ fsm_print_ir(FILE *f, const struct fsm_options *opt, } for (i = 0; i < ir->n; i++) { - if (-1 == print_state(f, opt, ir, &ir->states[i])) { + if (-1 == print_state(f, opt, hooks, ir, &ir->states[i])) { return -1; } } diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index a15b47015..06c4c4208 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -114,11 +114,14 @@ print_groups(FILE *f, const struct fsm_options *opt, } static int -print_state(FILE *f, const struct fsm_options *opt, +print_state(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir_state *cs) { assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(cs != NULL); fprintf(f, "\t\t{\n"); @@ -137,10 +140,10 @@ print_state(FILE *f, const struct fsm_options *opt, } /* showing hook in addition to existing content */ - if (cs->isend && opt->hooks.accept != NULL) { + if (cs->isend && hooks->accept != NULL) { fprintf(f, "\t\t\t\"endleaf\": "); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, cs->endids.ids, cs->endids.count, NULL, NULL)) { @@ -166,8 +169,6 @@ print_state(FILE *f, const struct fsm_options *opt, fprintf(f, "\n"); } - /* TODO: leaf callback for json output */ - switch (cs->strategy) { case IR_NONE: break; @@ -215,13 +216,16 @@ print_state(FILE *f, const struct fsm_options *opt, } int -fsm_print_irjson(FILE *f, const struct fsm_options *opt, +fsm_print_irjson(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir *ir) { size_t i; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(ir != NULL); fprintf(f, "{\n"); @@ -230,7 +234,7 @@ fsm_print_irjson(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\"states\": [\n"); for (i = 0; i < ir->n; i++) { - if (-1 == print_state(f, opt, &ir->states[i])) { + if (-1 == print_state(f, opt, hooks, &ir->states[i])) { return -1; } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 1c5d9da4e..025d1fe6f 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -204,13 +204,17 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, } int -fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) +fsm_print_json(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm) { fsm_state_t start; fsm_state_t i; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(fsm != NULL); fprintf(f, "{\n"); @@ -242,7 +246,7 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) } /* showing hook in addition to existing content */ - if (opt->hooks.accept != NULL) { + if (hooks->accept != NULL) { int notfirst; notfirst = 0; @@ -258,22 +262,25 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) } count = fsm_endid_count(fsm, i); + if (count == 0) { + ids = NULL; + } else { + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; + } - ids = f_malloc(fsm->alloc, count * sizeof *ids); - if (ids == NULL) { - return -1; + res = fsm_endid_get(fsm, i, count, ids); + assert(res == 1); } - res = fsm_endid_get(fsm, i, count, ids); - assert(res == 1); - if (notfirst) { fprintf(f, ", "); } fprintf(f, "{ %u, ", i); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, ids, count, NULL, NULL)) { @@ -282,7 +289,9 @@ fsm_print_json(FILE *f, const struct fsm_options *opt, const struct fsm *fsm) fprintf(f, " }"); - f_free(fsm->alloc, ids); + if (count > 0) { + f_free(fsm->alloc, ids); + } notfirst = 1; } diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 6cf49db28..ac87090df 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -98,7 +98,7 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { size_t i; @@ -106,6 +106,8 @@ default_accept(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(lang_opaque != NULL); + (void) hook_opaque; + i = * (const size_t *) lang_opaque; switch (opt->ambig) { @@ -146,13 +148,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; switch (opt->ambig) { case AMBIG_NONE: @@ -516,7 +519,10 @@ build_retlist(struct ret_list *list, const struct dfavm_op_ir *a) /* TODO: eventually to be non-static */ static int -fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +fsm_print_llvmfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, const char *cp) { struct ret_list retlist; @@ -613,7 +619,7 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o for (size_t i = 0; i < retlist.count; i++) { fprintf(f, "\t "); - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, retlist.a[i].ids, retlist.a[i].count, default_accept, &i)) { @@ -622,7 +628,7 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o } fprintf(f, "\t "); - if (-1 == print_hook_reject(f, opt, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { return -1; } @@ -736,13 +742,17 @@ fsm_print_llvmfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o } int -fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_llvm(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { const char *prefix; const char *cp; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -750,14 +760,14 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->hooks.cp != NULL) { - cp = opt->hooks.cp; + if (hooks->cp != NULL) { + cp = hooks->cp; } else { cp = "c"; /* XXX */ } if (opt->fragment) { - fsm_print_llvmfrag(f, opt, ops, cp); + fsm_print_llvmfrag(f, opt, hooks, ops, cp); return 0; } @@ -814,7 +824,7 @@ fsm_print_llvm(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) exit(EXIT_FAILURE); } - fsm_print_llvmfrag(f, opt, ops, cp); + fsm_print_llvmfrag(f, opt, hooks, ops, cp); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index eefe84b11..5c35747d9 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -53,7 +53,7 @@ print_ids(FILE *f, enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) { switch (ambig) { - case AMBIG_NONE: + case AMBIG_NONE: fprintf(f, "return Some(())"); break; @@ -89,35 +89,37 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { - assert(f != NULL); - assert(opt != NULL); - assert(lang_opaque == NULL); - - (void) lang_opaque; - - if (-1 == print_ids(f, opt->ambig, ids, count)) { - return -1; - } - - return 0; + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + if (-1 == print_ids(f, opt->ambig, ids, count)) { + return -1; + } + + return 0; } - + static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { - assert(f != NULL); - assert(opt != NULL); - assert(lang_opaque == NULL); - - (void) lang_opaque; + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; fprintf(f, "return None"); - return 0; + return 0; } static int @@ -157,15 +159,17 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, default_reject, NULL); + return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, + return print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, NULL); @@ -198,7 +202,10 @@ print_fetch(FILE *f) /* TODO: eventually to be non-static */ static int -fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +fsm_print_rustfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, const char *cp) { struct dfavm_op_ir *op; @@ -312,7 +319,7 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, "{ "); } - if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } if (op->cmp != VM_CMP_ALWAYS) { @@ -360,7 +367,7 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o fprintf(f, " "); fprintf(f, "None => "); - print_end(f, op, opt, op->u.fetch.end_bits); + print_end(f, op, opt, hooks, op->u.fetch.end_bits); fprintf(f, ",\n"); fprintf(f, " "); @@ -405,13 +412,17 @@ fsm_print_rustfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *o } int -fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_rust(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { const char *prefix; const char *cp; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -419,14 +430,14 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) prefix = "fsm_"; } - if (opt->hooks.cp != NULL) { - cp = opt->hooks.cp; + if (hooks->cp != NULL) { + cp = hooks->cp; } else { cp = "c"; /* XXX */ } if (opt->fragment) { - fsm_print_rustfrag(f, opt, ops, cp); + fsm_print_rustfrag(f, opt, hooks, ops, cp); goto error; } @@ -460,7 +471,7 @@ fsm_print_rust(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, opt, ops, cp); + fsm_print_rustfrag(f, opt, hooks, ops, cp); fprintf(f, "}\n"); fprintf(f, "\n"); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index a01369b24..05692df59 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -107,13 +107,14 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "matched"); @@ -126,13 +127,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "fail"); @@ -192,15 +194,17 @@ print_cond(FILE *f, const struct dfavm_op_ir *op) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, default_reject, NULL); + return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, + return print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, NULL); @@ -226,12 +230,16 @@ print_fetch(FILE *f) } int -fsm_print_sh(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_sh(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { struct dfavm_op_ir *op; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->io != FSM_IO_STR) { errno = ENOTSUP; @@ -285,14 +293,14 @@ fsm_print_sh(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) switch (op->instr) { case VM_OP_STOP: print_cond(f, op); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } break; case VM_OP_FETCH: print_fetch(f); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.fetch.end_bits)) { return -1; } break; diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 95250d8a0..c5ac87492 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -33,7 +33,9 @@ enum asm_dialect { }; static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, enum asm_dialect dialect, const char *ret_reg, enum dfavm_op_end end_bits) { @@ -41,13 +43,13 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, switch (end_bits) { case VM_END_FAIL: - if (-1 == print_hook_reject(f, opt, NULL, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL)) { return -1; } break; case VM_END_SUCC: - if (-1 == print_hook_accept(f, opt, + if (-1 == print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, NULL, NULL)) { @@ -83,7 +85,10 @@ print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } static int -print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +print_asm_amd64(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, const char *prefix, enum asm_dialect dialect) { @@ -196,7 +201,7 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, switch (op->instr) { case VM_OP_STOP: { - if (-1 == print_end(f, op, opt, + if (-1 == print_end(f, op, opt, hooks, dialect, ret_reg, op->u.stop.end_bits)) { @@ -241,7 +246,7 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, case VM_OP_FETCH: { - if (-1 == print_end(f, op, opt, + if (-1 == print_end(f, op, opt, hooks, dialect, ret_reg, op->u.fetch.end_bits)) { @@ -350,13 +355,17 @@ print_asm_amd64(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, } static int -print_vmasm_encoding(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +print_vmasm_encoding(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, enum asm_dialect dialect) { const char *prefix; assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (dialect == AMD64_GO) { if (opt->io != FSM_IO_STR && opt->io != FSM_IO_PAIR) { @@ -376,27 +385,33 @@ print_vmasm_encoding(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir prefix = "fsm_"; } - return print_asm_amd64(f, opt, ops, prefix, dialect); + return print_asm_amd64(f, opt, hooks, ops, prefix, dialect); } int -fsm_print_amd64_att(FILE *f, const struct fsm_options *opt, +fsm_print_amd64_att(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, ops, AMD64_ATT); + return print_vmasm_encoding(f, opt, hooks, ops, AMD64_ATT); } int -fsm_print_amd64_nasm(FILE *f, const struct fsm_options *opt, +fsm_print_amd64_nasm(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, ops, AMD64_NASM); + return print_vmasm_encoding(f, opt, hooks, ops, AMD64_NASM); } int -fsm_print_amd64_go(FILE *f, const struct fsm_options *opt, +fsm_print_amd64_go(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, opt, ops, AMD64_GO); + return print_vmasm_encoding(f, opt, hooks, ops, AMD64_GO); } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index b69203052..206e333ae 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -108,13 +108,14 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; if (-1 == print_ids(f, opt->ambig, ids, count)) { return -1; @@ -125,13 +126,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "return 0;"); @@ -163,15 +165,17 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, default_reject, NULL); + return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, + return print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, NULL); @@ -353,7 +357,10 @@ walk_sequence(struct dfavm_op_ir *op, /* TODO: eventually to be non-static */ static int -fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, +fsm_print_cfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, const char *cp) { struct dfavm_op_ir *op; @@ -423,7 +430,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } break; @@ -441,7 +448,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, fprintf(f, "if (e - p < %zu || 0 != memcmp(p, \"", n); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, end_bits)) { + if (-1 == print_end(f, NULL, opt, hooks, end_bits)) { return -1; } fprintf(f, "\n"); @@ -454,7 +461,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, fprintf(f, "if (0 != strncmp(p, \""); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, end_bits)) { + if (-1 == print_end(f, NULL, opt, hooks, end_bits)) { return -1; } fprintf(f, "\n"); @@ -465,7 +472,7 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, op = tail; } else { print_fetch(f, opt); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits)) { + if (-1 == print_end(f, op, opt, hooks, op->u.fetch.end_bits)) { return -1; } if (opt->io == FSM_IO_PAIR) { @@ -502,7 +509,10 @@ fsm_print_cfrag(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops, } int -fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_vmc(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { const char *prefix; @@ -511,6 +521,7 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); if (opt->prefix != NULL) { prefix = opt->prefix; @@ -519,7 +530,7 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) } if (opt->fragment) { - if (-1 == fsm_print_cfrag(f, opt, ops, cp)) { + if (-1 == fsm_print_cfrag(f, opt, hooks, ops, cp)) { return -1; } } else { @@ -568,11 +579,11 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) abort(); } - if (opt->hooks.args != NULL) { + if (hooks->args != NULL) { fprintf(stdout, ",\n"); fprintf(stdout, "\t"); - if (-1 == print_hook_args(f, opt, NULL, NULL)) { + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; } } @@ -580,7 +591,7 @@ fsm_print_vmc(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) fprintf(f, ")\n"); fprintf(f, "{\n"); - if (-1 == fsm_print_cfrag(f, opt, ops, cp)) { + if (-1 == fsm_print_cfrag(f, opt, hooks, ops, cp)) { return -1; } diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 90a8b0971..576d8c08d 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -48,13 +48,14 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "match"); @@ -75,13 +76,14 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, - void *lang_opaque) + void *lang_opaque, void *hook_opaque) { assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); (void) lang_opaque; + (void) hook_opaque; fprintf(f, "fail"); @@ -116,15 +118,17 @@ print_cond(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op) } static int -print_end(FILE *f, const struct fsm_options *opt, +print_end(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct dfavm_op_ir *op, enum dfavm_op_end end_bits) { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, default_reject, NULL); + return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, + return print_hook_accept(f, opt, hooks, op->endids.ids, op->endids.count, default_accept, NULL); @@ -142,7 +146,10 @@ print_branch(FILE *f, const struct dfavm_op_ir *op) } static int -fsm_print_nodes(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) +fsm_print_nodes(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { struct dfavm_op_ir *op; const char *example; @@ -204,7 +211,7 @@ fsm_print_nodes(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *ops) print_cond(f, opt, op); fprintf(f, "", op->index); - if (-1 == print_end(f, opt, op, op->u.stop.end_bits)) { + if (-1 == print_end(f, opt, hooks, op, op->u.stop.end_bits)) { return -1; } fprintf(f, "fetch", op->index); - if (-1 == print_end(f, opt, op, op->u.fetch.end_bits)) { + if (-1 == print_end(f, opt, hooks, op, op->u.fetch.end_bits)) { return -1; } fprintf(f, "
"); diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index c0eb21e7e..ec931769c 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -668,6 +668,13 @@ fsm_print_llvm(FILE *f, { return -1; } + + if (-1 == print_hook_comment(f, opt, hooks, + retlist->a[i].ids, retlist->a[i].count)) + { + return -1; + } + fprintf(f, "\n"); } } @@ -678,6 +685,7 @@ fsm_print_llvm(FILE *f, if (opt->ambig == AMBIG_MULTIPLE) { fprintf(f, "%%rt { %s bitcast ([%zu x i32]* @%sr%zu to %s), i64 %zu }", ptr_i32, retlist->a[i].count, prefix, i, ptr_i32, retlist->a[i].count); + fprintf(f, ","); } else { if (-1 == print_hook_accept(f, opt, hooks, retlist->a[i].ids, retlist->a[i].count, @@ -685,8 +693,16 @@ fsm_print_llvm(FILE *f, { return -1; } + + fprintf(f, ","); + + if (-1 == print_hook_comment(f, opt, hooks, + retlist->a[i].ids, retlist->a[i].count)) + { + return -1; + } } - fprintf(f, ",\n"); + fprintf(f, "\n"); } fprintf(f, "\t "); if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 96dca94f0..b11fb2c5a 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -364,6 +364,14 @@ fsm_print_rustfrag(FILE *f, fprintf(f, " }"); } + if (op->u.stop.end_bits == VM_END_SUCC) { + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } + } + if (op->cmp == VM_CMP_ALWAYS) { /* the code for fallthrough would be unreachable */ fallthrough = false; diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 5ebca2f85..4b9c647c6 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -205,10 +205,21 @@ print_end(FILE *f, const struct dfavm_op_ir *op, return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, hooks, + if (-1 == print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, default_accept, - NULL); + NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } + + return 0; default: assert(!"unreached"); diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 7848d88b7..dc9b76ee7 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -56,6 +56,11 @@ print_end(FILE *f, const struct dfavm_op_ir *op, { return -1; } + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } break; default: diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index b0fd77a08..c43073dca 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -178,10 +178,21 @@ print_end(FILE *f, const struct dfavm_op_ir *op, return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, hooks, + if (-1 == print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, default_accept, - NULL); + NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } + + return 0; default: assert(!"unreached"); diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 33faafcb8..e01fc9011 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -129,10 +129,17 @@ print_end(FILE *f, return print_hook_reject(f, opt, hooks, default_reject, NULL); case VM_END_SUCC: - return print_hook_accept(f, opt, hooks, + if (-1 == print_hook_accept(f, opt, hooks, op->ret->ids, op->ret->count, default_accept, - NULL); + NULL)) + { + return -1; + } + + /* no print_hook_comment() for dot output */ + + return 0; default: assert(!"unreached"); diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 3cfc1d7b6..d95263062 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -303,7 +303,17 @@ fsm_print_vmops_c(FILE *f, break; } - fprintf(f, "},\n"); + fprintf(f, "},"); + + if (op->instr == VM_OP_STOP && op->u.stop.end_bits == VM_END_SUCC) { + if (-1 == print_hook_comment(f, opt, hooks, + op->ret->ids, op->ret->count)) + { + return -1; + } + } + + fprintf(f, "\n"); } fprintf(f, "\t};\n"); diff --git a/src/re/main.c b/src/re/main.c index be8059d48..53abe2d52 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -336,16 +336,14 @@ conflict(FILE *f, const struct fsm_options *opt, static int comment_c(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque) + void *hook_opaque) { size_t i; assert(opt != NULL); - assert(lang_opaque == NULL); assert(hook_opaque == NULL); (void) opt; - (void) lang_opaque; (void) hook_opaque; fprintf(f, "/* "); @@ -368,19 +366,17 @@ comment_c(FILE *f, const struct fsm_options *opt, static int comment_rust(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque) + void *hook_opaque) { size_t i; assert(opt != NULL); - assert(lang_opaque != NULL); assert(hook_opaque == NULL); (void) opt; - (void) lang_opaque; (void) hook_opaque; - fprintf(f, "/* "); + fprintf(f, "// "); for (i = 0; i < count; i++) { assert(ids[i] < matchc); @@ -392,24 +388,20 @@ comment_rust(FILE *f, const struct fsm_options *opt, } } - fprintf(f, " */"); - return 0; } static int comment_llvm(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque) + void *hook_opaque) { size_t i; assert(opt != NULL); - assert(lang_opaque == NULL); assert(hook_opaque == NULL); (void) opt; - (void) lang_opaque; (void) hook_opaque; fprintf(f, "; "); @@ -430,12 +422,11 @@ comment_llvm(FILE *f, const struct fsm_options *opt, static int comment_dot(FILE *f, const struct fsm_options *opt, const fsm_end_id_t *ids, size_t count, - void *lang_opaque, void *hook_opaque) + void *hook_opaque) { size_t i; assert(opt != NULL); - assert(lang_opaque != NULL); assert(hook_opaque == NULL); (void) opt; @@ -1066,6 +1057,7 @@ main(int argc, char *argv[]) break; case FSM_PRINT_RUST: + case FSM_PRINT_GO: /* close enough */ hooks.comment = comment_rust; break; From ab0a4119ceb8f6c125fe7dd0ff3dc3b771427921 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 16 Aug 2024 10:32:27 +0100 Subject: [PATCH 235/373] Typo. Spotted by Scott. --- include/fsm/print.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index a485d88a9..ed9699e07 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -94,7 +94,7 @@ struct fsm_hooks { * but simply not yet implemented, where fsm_print() will print a message * to stderr and exit. * - * The code generation for the typical case of matching input require the FSM + * The code generation for the typical case of matching input requires the FSM * to be a DFA, and will EINVAL if the FSM is not a DFA. As opposed to e.g. * FSM_PRINT_API, which generates code for other purposes, and does not place * particular expecations on the FSM. From f0fea2772b75a692d872517a14b12bc6ee381231 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 16 Aug 2024 12:08:16 +0100 Subject: [PATCH 236/373] -t for AMBIG_EARLIEST --- src/rx/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index e708990b0..28ec6f11b 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -790,7 +790,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQqsuvx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); + printf("usage: %s: [-ciQqstuvx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); printf(" %s -h\n", name); } @@ -844,7 +844,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:E:e:ik:F:l:n:r:sR:Qquvx"), c != -1) { + while (c = getopt(argc, argv, "h" "C:cd:E:e:ik:F:l:n:r:R:stQquvx"), c != -1) { switch (c) { case 'C': charset = optarg; @@ -937,6 +937,10 @@ main(int argc, char *argv[]) quiet = true; break; + case 't': + opt.ambig = AMBIG_EARLIEST; + break; + case 'u': opt.ambig = AMBIG_MULTIPLE; break; From 26832cce21ab9ec64472cbbb072a249d9669d04e Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 16 Aug 2024 13:05:05 +0100 Subject: [PATCH 237/373] Graphviz output for the DFA. --- src/rx/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rx/main.c b/src/rx/main.c index 28ec6f11b..90f5e617e 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -705,6 +705,7 @@ print_name(const char *name, enum fsm_ambig ambig) enum fsm_ambig mask; } a[] = { { "c", FSM_PRINT_VMC, AMBIG_ANY }, + { "dot", FSM_PRINT_DOT, AMBIG_ANY }, { "fsm", FSM_PRINT_FSM, AMBIG_ANY }, { "rust", FSM_PRINT_RUST, AMBIG_ANY }, { "go", FSM_PRINT_GO, AMBIG_ANY }, From d062ec9ed7248e7d5d1f289a227e920a9192701b Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 16 Aug 2024 17:48:17 +0100 Subject: [PATCH 238/373] -a and -w for anonymous states and fragment output. --- src/rx/main.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 90f5e617e..c09c7dd41 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -791,7 +791,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-ciQqstuvx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); + printf("usage: %s: [-aciQqstuvwx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); printf(" %s -h\n", name); } @@ -845,8 +845,12 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "C:cd:E:e:ik:F:l:n:r:R:stQquvx"), c != -1) { + while (c = getopt(argc, argv, "h" "aC:cd:E:e:ik:F:l:n:r:R:stQquvwx"), c != -1) { switch (c) { + case 'a': + opt.anonymous_states = false; + break; + case 'C': charset = optarg; break; @@ -952,6 +956,10 @@ main(int argc, char *argv[]) verbose = true; break; + case 'w': + opt.fragment = true; + break; + case 'x': unanchored_literals = true; break; From 5e1044007afb69514d0cbe19a9cf0234777042d3 Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 16 Aug 2024 17:59:27 +0100 Subject: [PATCH 239/373] -X for always_hex --- src/rx/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index c09c7dd41..79966422a 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -791,7 +791,7 @@ usage(const char *name) name = p != NULL ? p + 1 : name; } - printf("usage: %s: [-aciQqstuvwx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); + printf("usage: %s: [-aciQqstuvwXx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); printf(" %s -h\n", name); } @@ -845,7 +845,7 @@ main(int argc, char *argv[]) const char *name = argv[0]; int c; - while (c = getopt(argc, argv, "h" "aC:cd:E:e:ik:F:l:n:r:R:stQquvwx"), c != -1) { + while (c = getopt(argc, argv, "h" "aC:cd:E:e:ik:F:l:n:r:R:stQquvwXx"), c != -1) { switch (c) { case 'a': opt.anonymous_states = false; @@ -960,6 +960,10 @@ main(int argc, char *argv[]) opt.fragment = true; break; + case 'X': + opt.always_hex = true; + break; + case 'x': unanchored_literals = true; break; From 7607456620fe72051607621df6a1c320f718c2bb Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 17 Aug 2024 10:32:24 +0100 Subject: [PATCH 240/373] -Fb for consistency wrt -Fi. --- src/re/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/re/main.c b/src/re/main.c index 53abe2d52..78bc77a2f 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -465,6 +465,10 @@ parse_flags(const char *arg, enum re_flags *flags) { for (; *arg; arg++) { switch (*arg) { + case 'b': + *flags = *flags | RE_ANCHORED; + break; + case 'i': *flags = *flags | RE_ICASE; break; From a925708bdc0df11c3433d25fd325e7a87fedffa9 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 17 Aug 2024 10:33:44 +0100 Subject: [PATCH 241/373] Documentation for -F flags and -k io api. --- man/lx.1/lx.1.xml | 13 +++++++++ man/re.1/re.1.xml | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/man/lx.1/lx.1.xml b/man/lx.1/lx.1.xml index 19423e67b..b9af035c1 100644 --- a/man/lx.1/lx.1.xml +++ b/man/lx.1/lx.1.xml @@ -18,12 +18,14 @@ tokbuf"> getc"> + io"> -n"> -Q"> -X"> -b &api_tokbuf.arg;"> -g &api_getc.arg;"> + -k &api_io.arg;"> -l &lang.arg;"> -e &prefix.arg;"> -t &prefix.arg;"> @@ -62,6 +64,7 @@ &X.opt; &b.opt; &g.opt; + &k.opt; &l.opt; &e.opt; &t.opt; @@ -223,6 +226,16 @@ Dot output is not subject to checks for semantic errors + + &k.opt; + + + Set the IO &api; for output, + per the io option for &fsm_print.3;. + The default is getc. + + + &e.opt; diff --git a/man/re.1/re.1.xml b/man/re.1/re.1.xml index ef2112500..1e01795f9 100644 --- a/man/re.1/re.1.xml +++ b/man/re.1/re.1.xml @@ -13,8 +13,10 @@ stdout"> + io"> re"> file"> + flags"> dialect"> prefix"> length"> @@ -33,7 +35,9 @@ -X"> -e &prefix.arg;"> -E &prefix.arg;"> + -F &flags.arg;"> -G &length.arg;"> + -k &io.arg;"> -a"> -d"> @@ -97,6 +101,7 @@ query &a.opt; &c.opt; + &k.opt; &w.opt; &X.opt; &e.opt; @@ -299,6 +304,60 @@ group (for dialects which have them) (rather than a "whole" regexp) + + &F.opt; + + + Set RE_* flags per &re.3;. + Note not all enum re_flags constants are meaningful + to expose to the &cli;. + The default is no flags set. + The available flags are: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagValueMeaning
b (&b.opt;)RE_ANCHOREDAnchor patterns at both start and end, + as if by ^ and $ + (\Z for PCRE).
i (&i.opt;)RE_ICASEPatterns are case insensitive.
sRE_SINGLESingle-line mode. + Equivalent to PCRE_DOTALL.
xRE_EXTENDED + Eextended mode. + Equivalent to PCRE_EXTENDED.
+ + + &G.opt; @@ -307,6 +366,16 @@ group (for dialects which have them) (rather than a "whole" regexp) + + &k.opt; + + + Set the IO &api; for output, + per the io option for &fsm_print.3;. + The default is getc. + + + &q.opt; From c26c5fd649c821fcbd81bd077b46acdb71b01e80 Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 17 Aug 2024 11:28:48 +0100 Subject: [PATCH 242/373] A little more progress on invocation. --- man/re.1/re.1.xml | 49 +++++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/man/re.1/re.1.xml b/man/re.1/re.1.xml index 1e01795f9..ad5254348 100644 --- a/man/re.1/re.1.xml +++ b/man/re.1/re.1.xml @@ -53,7 +53,6 @@ -x"> -h"> - -v"> ]> @@ -174,9 +173,6 @@ query &h.opt; - @@ -189,6 +185,8 @@ query which are “executed” to match against input strings, or used to provide example strings which would match when given as input. + This tool provides an interface to the &re.3; &api;. + The &re.1; tool constructs a Finite State Machine from Regular Expression syntax. - -XXX: when executing, -strings are single or multiple with --. -can't we feed a string to stdin, too? -XXX: now with -x instead: Unlike &grep.1;, strings are matched from -command line arguments rather than &stdin.lit;. -TODO: with no strings to match, will just validate fsm instead. -validate is the wrong word - - + When executing, text to match is given either as strings + from command line arguments, or is given as filenames with &x.opt;. + Multiple arguments may be given using --. + Unlike &grep.1;, &re.1; does not read text from &stdin.lit;. + + With no text to match, &re.1; will just validate + + the pattern syntax instead. + + @@ -536,17 +538,6 @@ group (for dialects which have them) (rather than a "whole" regexp) - - &h.opt; From 9668a769cbded3b364c0313ca11f92d32609079f Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 17 Aug 2024 20:38:50 +0100 Subject: [PATCH 243/373] Manpage for rx(1) --- Makefile | 1 + README.md | 3 +- man/re.1/re.1.xml | 2 +- man/rx.1/Makefile | 4 + man/rx.1/rx.1.xml | 482 ++++++++++++++++++++++++++++++++++++++++++ share/dtd/ent-ref.dtd | 1 + src/rx/main.c | 1 - 7 files changed, 491 insertions(+), 3 deletions(-) create mode 100644 man/rx.1/Makefile create mode 100644 man/rx.1/rx.1.xml diff --git a/Makefile b/Makefile index 997499f98..177f8456a 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ PKG += libtheft .if !defined(NODOC) SUBDIR += man/fsm.1 SUBDIR += man/re.1 +SUBDIR += man/rx.1 SUBDIR += man/lx.1 SUBDIR += man/fsm_print.3 SUBDIR += man/libfsm.3 diff --git a/README.md b/README.md index 48ec80949..545725fe9 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ You get: * libfsm — library for manipulating FSM (NFA and DFA) * libre — library for compiling regular expressions to NFA * fsm(1) — command line interface for FSM - * re(1) — command line interface for executing regular expressions + * re(1) — command line interface for regular expressions + * rx(1) — command line interface for compiling sets of regular expressions * lx(1) — lexer generator lx is an attempt to produce a simple, expressive, and unobtrusive diff --git a/man/re.1/re.1.xml b/man/re.1/re.1.xml index ef2112500..a37e0656e 100644 --- a/man/re.1/re.1.xml +++ b/man/re.1/re.1.xml @@ -194,7 +194,7 @@ query and minimised to its canonical &dfa; by default. The &nfa; may be preserved by the &n.opt; option. - multiple regexps may be given; these are unioned by default, + Multiple regexps may be given; these are unioned by default, or concatenated when the &s.opt; option is given. &re.1; is invoked in one of four modes of operation: diff --git a/man/rx.1/Makefile b/man/rx.1/Makefile new file mode 100644 index 000000000..54c2fe984 --- /dev/null +++ b/man/rx.1/Makefile @@ -0,0 +1,4 @@ +.include "../../share/mk/top.mk" + +MAN += man/rx.1/rx.1.xml + diff --git a/man/rx.1/rx.1.xml b/man/rx.1/rx.1.xml new file mode 100644 index 000000000..7e076d54e --- /dev/null +++ b/man/rx.1/rx.1.xml @@ -0,0 +1,482 @@ + + + + + + stdin"> + stdout"> + stderr"> + + + charset"> + file"> + flags"> + dialect"> + io"> + prefix"> + limit"> + + -a"> + -b"> + -C &charset.arg;"> + -c"> + -d &file.arg;"> + -E &prefix.arg;"> + -e &prefix.arg;"> + -F &flags.arg;"> + -i"> + -k &io.arg;"> + -Q"> + -q"> + -R &charset.arg;"> + -r &dialect.arg;"> + -s"> + -t"> + -u"> + -l &lang.arg;"> + -n &limit.arg;"> + -v"> + -w"> + -X"> + + -h"> +]> + + + + rx + + + + rx + 1 + + + + rx + Compile sets of Regular Expressions + + + + + rx + + + &a.opt; + &c.opt; + &i.opt; + &Q.opt; + &q.opt; + &s.opt; + &t.opt; + &u.opt; + &v.opt; + &w.opt; + &X.opt; + &x.opt; + + + &C.opt; + &k.opt; + &l.opt; + &r.opt; + &R.opt; + &d.opt; + &E.opt; + &e.opt; + + &file.arg; + + + + + Description + + &rx.1; is a command line too to compile sets of + regular expressions to executable code. + Patterns are read from one or more files, with multiple patterns per file. + Patterns that &rx.1; declines to handle are written out + to a file given by , if present. + Generated code is printed out to &stdout.lit;. + + Input files have one pattern per line. + Each pattern has an associated id. + ids are assigned depending on the number of input files. + For a single file, ids are assigned per pattern + (that is, the id is the line number within the file). + For multiple files, ids are assigned per file + (that is, the same id is shared by all patterns within a file). + + Pattern ids are made available to the generated code + when successfully matching a set of one or more patterns. + You can see these with output. + It is possible for a given text string to match patterns + associated with different ids. + There are several ways to deal with this, + which of these is appropriate depends on the application: + + + + Error about it at compile time. + This is the default for &rx.1; + To use this mode, ensure your patterns don't overlap. + In particular you can use rx -q as a lint to find conflicts. + + + + Give conflicting patterns the same id in the first place. + This would be the case for a lexer, + where you might have multiple spellings that + produce the same token. + + + + Allow ambiguious patterns, + and the generated &api; returns a set of ids. + See &u.opt;. + + + + Earliest line number (lower id) wins. + This would suit a firewall-like application where it doesn't matter which + See &t.opt;. + + + + Longest match or most specific regex wins. + This doesn't work for &dfa; and so is not provided by &rx.1;. + + + + + &rx.1; places patterns into one of several categories: + literal strings, general regexps, parse errors, + unsupported syntax, unsatisfiable expressions, + expressions rejected due to + or character set violations. + These categories are internal, and do not appear in the + generated code. However they may be seen with the &v.opt; output, + and apply for the &n.opt; limit on the number of general regexps. + + + + Options + + + + &a.opt; + + + Number states, + per the anonymous_states option for &fsm_print.3;. + The default is to anonymise states, + which is the opposite to the equivalent &a.opt; option for &fsm.1;. + + + + + &C.opt; + + + Intersect the constructed &fsm; down to a given + character set. + This effectively cuts regex constructions like + .* down to [[:charset:]]*. + The default is to allow all character values. + + + + + &c.opt; + + + Use RFC 9110 HTTP -field-name tokens + as the intersection character set + (as if by ). + A valid HTTP header also has at least one character, + but &rx.1; does not enforce this. + + + + + &d.opt; + + + Output declined patterns to a file, one per line. + Patterns are declined by &rx.1; either because they're unsupported, + or (in the absence of ) because of an error + compiling the pattern. + + + + + + &E.opt; + + + Set a package prefix for output, + per the package_prefix option for &fsm_print.3;. + The default is to output with the value of prefix. + + + + + &e.opt; + + + Set a prefix for output, + per the prefix option for &fsm_print.3;. + The default is to output with no prefix. + + + + + &i.opt; + + + Strict mode. + This causes &rx.1; to error on parse errors and for unsatisfiable + expressions (such as $x). + + Strict mode does not error on regexps that are parsed correctly + but unsupported, these go to the declined list (see &d.opt;). + + + + + &k.opt; + + + Set the IO &api; for output, + per the io option for &fsm_print.3;. + The default is getc. + + + + + &l.opt; + + + Set the output language for generated code. + See &fsm_lang.5fsm; for the supported language identifiers. + The default is vmc. + + Compiled code is printed is to &stdout.lit;, + &q.opt; disables output. + + &rx.1; does not provide a way to disable + consolidate_edges, + or to disable disable_comments. + See &fsm_print.3; for information on these. + + + + + &F.opt; + + + Set RE_* flags per &re.3;. + Note not all enum re_flags constants are meaningful + to expose to the &cli;. + Flags apply to all patterns. + The default is no flags set. + The available flags are: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagValueMeaning
b (&b.opt;)RE_ANCHOREDAnchor patterns at both start and end, + as if by ^ and $ + (\Z for PCRE).
i (&i.opt;)RE_ICASEPatterns are case insensitive.
sRE_SINGLESingle-line mode. + Equivalent to PCRE_DOTALL.
xRE_EXTENDED + Eextended mode. + Equivalent to PCRE_EXTENDED.
+ + + + + &n.opt; + + + &rx.1; categorizes patterns internally when constructing + intermediate state machines. + allows a limit on the general category of patterns. + The default is no limit. + Categorization is visible per-pattern by &v.opt;. + + + + + &R.opt; + + + Reject character set. + When a pattern contains a character in + The default is to accept all characters. + + + + + &r.opt; + + + Set the default &re_dialect.5re; for patterns. + This applies to all files unless overriden by &s.opt;. + The default is pcre. + + + + + &s.opt; + + + Override default dialect per-file by file extension. + + + + + &Q.opt; + + + Print resource consumption stats to &stderr.lit;. + + + + + &q.opt; + + + Quiet (do not print generated code). + The default is to print generated code to &stdout.lit;. + + + + + &t.opt; + + + Resolve ambiguities by picking the earliest (lowest) id + for a set of conflicts. + The default is to error for conflicts. + + + + + &u.opt; + + + Allow ambiguities. + This means patterns with different ids may match the same text. + The default is to error for conflicts. + + It's possible to have multiple patterns with the same id + (i.e. by being in the same file when using multi-file mode), + and these are not considered a conflict because they key + to the same id. + + + + + &u.opt; + + + Allow ambiguities between regexps, + such that multiple regexps may match the same text. + The default is to error for ambiguities, + requiring all regexps unioned to be non-overlapping. + Formally, the requirement is that they are disjoint languages. + Erroring for ambiguities applies after multiple regexps are joined, + either by union or by concaternation (&s.opt;). + + &u.opt; is implied by &n.opt;. + + + + + &v.opt; + + + Verbose mode. + This prints information about every pattern and associated &fsm; + to &stderr.lit; during compilation. + In particular this shows the internal categorization + per pattern, which is relevant for the limit. + + + + + &w.opt; + + + Output a fragment, + per the fragment option for &fsm_print.3;. + The default is to output a complete file. + + + + + &X.opt; + + + Always encode byte values in hexadecimal, + per the always_hex option for &fsm_print.3;. + The default is to only use hex for non-printable characters. + + + + + &h.opt; + + + Print command-line usage help. + + + + + + + See Also + + &re.1;, &fsm.3;, + &native.5re;, &literal.5re;, &lx.5;. + + + diff --git a/share/dtd/ent-ref.dtd b/share/dtd/ent-ref.dtd index fb91fa88c..0e71f6eba 100644 --- a/share/dtd/ent-ref.dtd +++ b/share/dtd/ent-ref.dtd @@ -31,6 +31,7 @@ lx %vol.1;"> re %vol.1;"> +rx %vol.1;"> fsm %vol.1;"> libre %vol.3;"> diff --git a/src/rx/main.c b/src/rx/main.c index 79966422a..9a78f8357 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -837,7 +837,6 @@ main(int argc, char *argv[]) // TODO: prepend comment with rx invocation to output // TODO: cli option for memory limit to decline patterns (implement via allocation hooks) -// TODO: manpage // TODO: consider de-duplicating arrays. not sure there's any reason // TODO: decide if i want dialect==RE_LITERAL unanchored or not, maybe a cli flag either way From c08782ac041e1fc058620335874783db298462d9 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 20 Aug 2024 15:32:07 +0100 Subject: [PATCH 244/373] Missing flag, this is now present on main. --- src/rx/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rx/main.c b/src/rx/main.c index 9a78f8357..5ed4508b9 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -759,6 +759,10 @@ parse_flags(const char *arg, enum re_flags *flags) { for (; *arg; arg++) { switch (*arg) { + case 'b': + *flags = *flags | RE_ANCHORED; + break; + case 'i': *flags = *flags | RE_ICASE; break; From d73682bea494e25bf1aef8992c94353fdbdd6f06 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 20 Aug 2024 23:24:22 +0100 Subject: [PATCH 245/373] Stray assertion. I'm not sure why I thought this was neccessary; I tested and the trie code seems happy with an empty string. Which is fortunate, because I'm a believer in recursive datastructures. --- src/libre/re_strings.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libre/re_strings.c b/src/libre/re_strings.c index dc554c801..39a9b5c24 100644 --- a/src/libre/re_strings.c +++ b/src/libre/re_strings.c @@ -68,7 +68,6 @@ int re_strings_add_raw(struct re_strings *g, const void *p, size_t n, const fsm_end_id_t *endid) { assert(p != NULL); - assert(n > 0); return trie_add_word((struct trie_graph *) g, p, n, endid) != NULL; } From 8c9b209bc8928623060d1aaff921456f30ed7c65 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 22 Aug 2024 12:44:26 +0100 Subject: [PATCH 246/373] Spelling. --- man/rx.1/rx.1.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/man/rx.1/rx.1.xml b/man/rx.1/rx.1.xml index 7e076d54e..5ce8d5b27 100644 --- a/man/rx.1/rx.1.xml +++ b/man/rx.1/rx.1.xml @@ -137,7 +137,7 @@ - Allow ambiguious patterns, + Allow ambiguous patterns, and the generated &api; returns a set of ids. See &u.opt;. @@ -154,7 +154,7 @@ - + &rx.1; places patterns into one of several categories: literal strings, general regexps, parse errors, unsupported syntax, unsatisfiable expressions, @@ -324,7 +324,7 @@ 		x RE_EXTENDED - Eextended mode. + Extended mode. Equivalent to PCRE_EXTENDED.
S%u%s
end id"); - for (size_t i = 0; i < cs->endids.count; i++) { - fprintf(f, "#%u", cs->endids.ids[i]); + for (size_t i = 0; i < cs->count; i++) { + fprintf(f, "#%u", cs->ids[i]); - if (i < cs->endids.count - 1) { + if (i < (size_t) cs->count - 1) { fprintf(f, " "); } } @@ -220,7 +220,7 @@ print_state(FILE *f, fprintf(f, "\t\t
"); if (-1 == print_hook_accept(f, opt, hooks, - cs->endids.ids, cs->endids.count, + cs->ids, cs->count, NULL, NULL)) { diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 06c4c4208..147ba263f 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -127,12 +127,12 @@ print_state(FILE *f, fprintf(f, "\t\t{\n"); fprintf(f, "\t\t\t\"end\": %s,\n", cs->isend ? "true" : "false"); - if (cs->isend && cs->endids.count > 0) { + if (cs->isend && cs->count > 0) { fprintf(f, "\t\t\t\"end_id\": ["); - for (size_t i = 0; i < cs->endids.count; i++) { - fprintf(f, "%u", cs->endids.ids[i]); + for (size_t i = 0; i < cs->count; i++) { + fprintf(f, "%u", cs->ids[i]); - if (i < cs->endids.count - 1) { + if (i < (size_t) cs->count - 1) { fprintf(f, ", "); } } @@ -144,7 +144,7 @@ print_state(FILE *f, fprintf(f, "\t\t\t\"endleaf\": "); if (-1 == print_hook_accept(f, opt, hooks, - cs->endids.ids, cs->endids.count, + cs->ids, cs->count, NULL, NULL)) { return -1; diff --git a/src/libfsm/vm/ir.c b/src/libfsm/vm/ir.c index 608871b61..a32bca463 100644 --- a/src/libfsm/vm/ir.c +++ b/src/libfsm/vm/ir.c @@ -309,7 +309,7 @@ opasm_new(struct dfavm_assembler_ir *a, const struct ret_list *retlist, if (ir_state != NULL) { op->example = ir_state->example; op->ret = ir_state->isend - ? find_ret(retlist, ir_state->endids.ids, ir_state->endids.count) + ? find_ret(retlist, ir_state->ids, ir_state->count) : NULL; } @@ -968,7 +968,7 @@ order_basic_blocks(struct dfavm_assembler_ir *a) /* mark all states as !in_trace */ for (i=0; i < n; i++) { - a->ops[i]->in_trace = 0; + a->ops[i]->in_trace = false; } opp = &a->linked; @@ -987,7 +987,7 @@ order_basic_blocks(struct dfavm_assembler_ir *a) assert(opp != NULL); assert(*opp == NULL); - st->in_trace = 1; + st->in_trace = true; /* look for branches to states not in the trace. * Start at the last branch and work toward the first. diff --git a/src/libfsm/vm/retlist.c b/src/libfsm/vm/retlist.c index 97d4eb3f0..031d1d101 100644 --- a/src/libfsm/vm/retlist.c +++ b/src/libfsm/vm/retlist.c @@ -105,7 +105,7 @@ build_retlist(struct ret_list *list, const struct ir *ir) continue; } - if (!append_ret(list, ir->states[i].endids.ids, ir->states[i].endids.count)) { + if (!append_ret(list, ir->states[i].ids, ir->states[i].count)) { return false; } } diff --git a/src/libfsm/vm/vm.h b/src/libfsm/vm/vm.h index c96bfa84b..540a19d2e 100644 --- a/src/libfsm/vm/vm.h +++ b/src/libfsm/vm/vm.h @@ -79,8 +79,9 @@ struct dfavm_op_ir { const char *example; const struct ret *ret; - uint32_t num_incoming; // number of branches to this instruction - int in_trace; + uint32_t num_incoming:31; // number of branches to this instruction, :31 for packing + unsigned int in_trace:1; + int cmp_arg; enum dfavm_op_cmp cmp; diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 8fd3b6395..5a6b41b24 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -2014,10 +2014,10 @@ analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_p } struct pincer_anchors_env { - int always_consumes_or_anchored_start; - int always_consumes_or_anchored_end; + unsigned int always_consumes_or_anchored_start:1; + unsigned int always_consumes_or_anchored_end:1; - int alloc_fail; + unsigned int alloc_fail:1; size_t used; size_t ceil; struct ast_expr **nodes; From 6a40553b04dc566d80ef87ab9ebb0a9b4dfd55db Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 25 Aug 2024 18:18:15 +0100 Subject: [PATCH 255/373] Guard .tag; it helps keep structs tidy. --- src/libfsm/clone.c | 4 ++++ src/libfsm/consolidate.c | 6 ++++++ src/libfsm/determinise.c | 7 ++++++- src/libfsm/determinise_internal.h | 2 ++ src/libfsm/endids.c | 4 ++++ src/libfsm/epsilons.c | 9 ++++++++- src/libfsm/merge.c | 6 ++++++ 7 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 50f406825..9fd236a4d 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -118,7 +118,9 @@ copy_capture_actions(struct fsm *dst, const struct fsm *src) } struct copy_end_ids_env { +#ifndef NDEBUG char tag; +#endif struct fsm *dst; const struct fsm *src; bool ok; @@ -146,7 +148,9 @@ static int copy_end_ids(struct fsm *dst, const struct fsm *src) { struct copy_end_ids_env env; +#ifndef NDEBUG env.tag = 'c'; /* for clone */ +#endif env.dst = dst; env.src = src; env.ok = true; diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 49b08a821..236a4f6f5 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -36,7 +36,9 @@ struct mapping_closure { }; struct consolidate_copy_capture_actions_env { +#ifndef NDEBUG char tag; +#endif struct fsm *dst; size_t mapping_count; const fsm_state_t *mapping; @@ -201,7 +203,9 @@ consolidate_copy_capture_actions(struct fsm *dst, const struct fsm *src, size_t i; struct consolidate_copy_capture_actions_env env; +#ifndef NDEBUG env.tag = 'C'; +#endif env.dst = dst; env.mapping_count = mapping_count; env.mapping = mapping; @@ -249,7 +253,9 @@ consolidate_end_ids(struct fsm *dst, const struct fsm *src, struct consolidate_end_ids_env env; int ret; +#ifndef NDEBUG env.tag = 'C'; /* for Consolidate */ +#endif env.dst = dst; env.src = src; env.mapping = mapping; diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 805443e37..42992b6bc 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -382,7 +382,12 @@ static int det_copy_capture_actions(struct reverse_mapping *reverse_mappings, struct fsm *dst, struct fsm *src) { - struct det_copy_capture_actions_env env = { 'D', NULL, NULL, 1 }; + struct det_copy_capture_actions_env env = { +#ifndef NDEBUG + 'D', +#endif + NULL, NULL, 1 + }; env.dst = dst; env.reverse_mappings = reverse_mappings; diff --git a/src/libfsm/determinise_internal.h b/src/libfsm/determinise_internal.h index 8628f71b5..cfd4ea663 100644 --- a/src/libfsm/determinise_internal.h +++ b/src/libfsm/determinise_internal.h @@ -83,7 +83,9 @@ struct reverse_mapping { }; struct det_copy_capture_actions_env { +#ifndef NDEBUG char tag; +#endif struct fsm *dst; struct reverse_mapping *reverse_mappings; bool ok; diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 62f279ca3..f77905f67 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -753,7 +753,9 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, } struct carry_env { +#ifndef NDEBUG char tag; +#endif struct fsm *dst; fsm_state_t dst_state; bool ok; @@ -796,7 +798,9 @@ fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, for (state_set_reset(src_set, &it); state_set_next(&it, &s); ) { struct carry_env env; +#ifndef NDEBUG env.tag = 'C'; +#endif env.dst = dst_fsm; env.dst_state = dst_state; env.ok = true; diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index dcd46ddda..825ad9e2a 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -28,7 +28,9 @@ #define DEF_CARRY_ENDIDS_COUNT 2 struct remap_env { +#ifndef NDEBUG char tag; +#endif bool ok; const struct fsm_alloc *alloc; struct state_set **rmap; @@ -183,7 +185,12 @@ remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) struct state_set **rmap; struct state_iter si; fsm_state_t si_s; - struct remap_env env = { 'R', true, NULL, NULL, 0, 0, NULL }; + struct remap_env env = { +#ifndef NDEBUG + 'R', +#endif + true, NULL, NULL, 0, 0, NULL + }; env.alloc = nfa->alloc; /* build a reverse mapping */ diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index b2a556c66..8c972c145 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -26,7 +26,9 @@ #define LOG_MERGE_ENDIDS 0 struct copy_capture_env { +#ifndef NDEBUG char tag; +#endif bool ok; struct fsm *dst; }; @@ -145,7 +147,9 @@ static int copy_capture_actions(struct fsm *dst, struct fsm *src) { struct copy_capture_env env; +#ifndef NDEBUG env.tag = 'C'; +#endif env.dst = dst; env.ok = true; @@ -180,7 +184,9 @@ static int copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) { struct copy_end_ids_env env; +#ifndef NDEBUG env.tag = 'M'; /* for Merge */ +#endif env.dst = dst; env.src = src; env.base_src = base_src; From f1ca8e8c679f57b3877f4e2392891fa791c5a58e Mon Sep 17 00:00:00 2001 From: Kate F Date: Sun, 25 Aug 2024 20:36:33 +0100 Subject: [PATCH 256/373] No need for calloc here. --- src/adt/edgeset.c | 4 ++-- src/adt/idmap.c | 1 + src/adt/internedstateset.c | 6 +++--- src/adt/queue.c | 4 +++- src/libfsm/capture.c | 9 +++++++-- src/libfsm/endids.c | 2 +- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index 8e2554c33..178ddbc2b 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -121,7 +121,7 @@ dump_edge_set(const struct edge_set *set) static struct edge_set * init_empty(const struct fsm_alloc *alloc) { - struct edge_set *set = f_calloc(alloc, 1, sizeof(*set)); + struct edge_set *set = f_malloc(alloc, sizeof(*set)); if (set == NULL) { return NULL; } @@ -719,7 +719,7 @@ edge_set_copy(struct edge_set **dst, const struct fsm_alloc *alloc, return 1; } - set = f_calloc(alloc, 1, sizeof(*set)); + set = f_malloc(alloc, sizeof(*set)); if (set == NULL) { return 0; } diff --git a/src/adt/idmap.c b/src/adt/idmap.c index 208af3c5a..e3facfa49 100644 --- a/src/adt/idmap.c +++ b/src/adt/idmap.c @@ -119,6 +119,7 @@ grow_bucket_values(struct idmap *m, unsigned old_words, unsigned new_words) return 0; } + // TODO: memcpy for (size_t w_i = 0; w_i < old_words; w_i++) { nv[w_i] = b->values[w_i]; } diff --git a/src/adt/internedstateset.c b/src/adt/internedstateset.c index 9daf50ae7..990d9c98f 100644 --- a/src/adt/internedstateset.c +++ b/src/adt/internedstateset.c @@ -77,7 +77,7 @@ interned_state_set_pool_alloc(const struct fsm_alloc *a) fsm_state_t *buf = NULL; uint32_t *buckets = NULL; - res = f_calloc(a, 1, sizeof(*res)); + res = f_malloc(a, sizeof(*res)); if (res == NULL) { goto cleanup; } sets = f_malloc(a, DEF_SETS * sizeof(sets[0])); @@ -93,7 +93,7 @@ interned_state_set_pool_alloc(const struct fsm_alloc *a) buckets[i] = NO_ID; } - struct interned_state_set_pool p = { + *res = (struct interned_state_set_pool) { .alloc = a, .sets = { .ceil = DEF_SETS, @@ -108,7 +108,7 @@ interned_state_set_pool_alloc(const struct fsm_alloc *a) .buckets = buckets, }, }; - memcpy(res, &p, sizeof(p)); + return res; cleanup: diff --git a/src/adt/queue.c b/src/adt/queue.c index 1e5d28e08..9f3507f69 100644 --- a/src/adt/queue.c +++ b/src/adt/queue.c @@ -30,11 +30,13 @@ queue_new(const struct fsm_alloc *a, size_t max_capacity) if (max_capacity == 0) { return NULL; } alloc_size = sizeof(*q) + (max_capacity - 1) * sizeof(q->q[0]); - q = f_calloc(a, 1, alloc_size); + q = f_malloc(a, alloc_size); if (q == NULL) { return NULL; } q->alloc = a; q->capacity = max_capacity; + q->rd = 0; + q->wr = 0; return q; } diff --git a/src/libfsm/capture.c b/src/libfsm/capture.c index 3f750169e..d928a7b77 100644 --- a/src/libfsm/capture.c +++ b/src/libfsm/capture.c @@ -14,11 +14,16 @@ fsm_capture_init(struct fsm *fsm) struct fsm_capture_info *ci = NULL; size_t i; - ci = f_calloc(fsm->alloc, - 1, sizeof(*ci)); + ci = f_malloc(fsm->alloc, sizeof(*ci)); if (ci == NULL) { goto cleanup; } + + ci->max_capture_id = 0; + ci->bucket_count = 0; + ci->buckets_used = 0; + ci->buckets = NULL; + fsm->capture_info = ci; for (i = 0; i < fsm->statealloc; i++) { diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index f77905f67..d889d057b 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -89,7 +89,7 @@ fsm_endid_init(struct fsm *fsm) { struct endid_info_bucket *buckets = NULL; size_t i; - struct endid_info *res = f_calloc(fsm->alloc, 1, sizeof(*res)); + struct endid_info *res = f_malloc(fsm->alloc, sizeof(*res)); if (res == NULL) { return 0; } From 5010a4043e199459860cbc6940f54cfc8aaadf9b Mon Sep 17 00:00:00 2001 From: Kate F Date: Sat, 24 Aug 2024 15:23:48 +0100 Subject: [PATCH 257/373] Stray assertion. This doesn't help for #317, but whatever the solution is there, asserting about it is the wrong thing to do. Spotted by @classabbyamp, thank you --- src/libre/print/abnf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libre/print/abnf.c b/src/libre/print/abnf.c index e8439aa28..a50771ca2 100644 --- a/src/libre/print/abnf.c +++ b/src/libre/print/abnf.c @@ -212,7 +212,6 @@ pp_iter(FILE *f, const struct fsm_options *opt, enum re_flags *re_flags, struct break; case AST_EXPR_SUBTRACT: - assert(!"unimplemented"); pp_atomic(f, opt, re_flags, n->u.subtract.a, n); fprintf(f, " - "); pp_atomic(f, opt, re_flags, n->u.subtract.b, n); From ebbcb356bf7aec23348417e6cbc76a8ad88c814e Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Aug 2024 02:50:00 +0100 Subject: [PATCH 258/373] Update for hooks & options API changes. I have extremely broken the iprange example. I am very unsure what's going on with this program. --- examples/bm/libfsm.c | 4 +- examples/glob/main.c | 2 +- examples/iprange/main.c | 149 +++++++++++++++++++++++++++++++--------- examples/utf8dfa/main.c | 9 ++- examples/words/main.c | 8 +-- 5 files changed, 130 insertions(+), 42 deletions(-) diff --git a/examples/bm/libfsm.c b/examples/bm/libfsm.c index 80b013d63..a7581bd59 100644 --- a/examples/bm/libfsm.c +++ b/examples/bm/libfsm.c @@ -61,7 +61,7 @@ main(int argc, char *argv[]) opt.io = FSM_IO_STR; p = argv[0]; - fsm = re_comp(RE_PCRE, fsm_sgetc, &p, &opt, flags, &e); + fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, flags, &e); if (fsm == NULL) { re_perror(RE_LITERAL, &e, NULL, s); return 1; @@ -80,7 +80,7 @@ main(int argc, char *argv[]) printf("#include \n"); printf("\n"); - fsm_print(stdout, fsm, FSM_PRINT_C); + fsm_print(stdout, fsm, &opt, NULL, FSM_PRINT_C); printf("int\n"); printf("main(void)\n"); diff --git a/examples/glob/main.c b/examples/glob/main.c index a5af0f317..65703352c 100644 --- a/examples/glob/main.c +++ b/examples/glob/main.c @@ -196,7 +196,7 @@ main(int argc, char *argv[]) } if (!quiet) { - fsm_print(stdout, fsm, FSM_PRINT_FSM); + fsm_print(stdout, fsm, NULL, NULL, FSM_PRINT_FSM); } matched = match(fsm, argv[1]); diff --git a/examples/iprange/main.c b/examples/iprange/main.c index 6ce49993b..ba028351b 100644 --- a/examples/iprange/main.c +++ b/examples/iprange/main.c @@ -88,6 +88,24 @@ RB_GENERATE_STATIC(recmap, record, entry, recmap_cmp) static unsigned nrecords; static struct fsm_options opt; +static struct fsm_hooks hooks; + +static struct record * +find_id(unsigned id) +{ + struct record *r; + + /* XXX: this is a crime, we have a tree. + * we should be able to RB_FIND() */ + RB_FOREACH(r, recmap, &recmap) { + if (r->id == id) { + return r; + } + } + + assert(!"unreached"); + abort(); +} static struct record * get_id(char *rec, size_t reclen) @@ -115,7 +133,7 @@ get_id(char *rec, size_t reclen) r->len = reclen; r->id = nrecords++; - r->fsm = fsm_new(&opt); + r->fsm = fsm_new(NULL); if (r->fsm == NULL) { perror("fsm_new"); exit(-1); @@ -133,7 +151,7 @@ get_id(char *rec, size_t reclen) } fsm_setend(r->fsm, r->end, 1); - for (size_t i = 0; i < sizeof r->regs; i++) { + for (size_t i = 0; i < sizeof r->regs / sizeof *r->regs; i++) { r->regs[i].c = '\0'; r->regs[i].s = fsm_none; } @@ -164,7 +182,7 @@ get_id(char *rec, size_t reclen) static void usage(void) { - fprintf(stderr, "ip2fsm -[46] [-f ] -l fmt\n" + fprintf(stderr, "iprange -[46] [-f ] -l fmt\n" "\t-4\t\tIPv4\n" "\t-6\t\tIPv6\n" "\t-f \tuse as input\n" @@ -378,6 +396,9 @@ handle_line(unsigned char *socts, unsigned char *eocts, unsigned noct, } socts[spos] = eocts[spos]; + + // XXX: not sure about this + break; } else { gen_range(r, noct, spos - 1, socts[spos], 255, socts); @@ -422,45 +443,69 @@ important(unsigned n) } static int -leaf(FILE *f, const fsm_end_id_t *ids, size_t count, const void *leaf_opaque) +conflict(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + const char *example, void *hook_opaque) { - const struct record *r; + size_t i; - (void) leaf_opaque; + (void) f; + (void) hook_opaque; + (void) opt; - if (count != 1) { - fprintf(f, "endid conflict\n"); - exit(EXIT_FAILURE); - } + fprintf(stderr, "ambiguous matches for "); - r = (const void *) (intptr_t) ids[0]; /* XXX */ + for (i = 0; i < count; i++) { + const struct record *r; - if (r == NULL) { - fprintf(f, "return -1;"); - return 0; + r = (const void *) (intptr_t) ids[i]; /* XXX */ + + fprintf(stderr, "%s", r->rec); + + if (i + 1 < count) { + fprintf(stderr, ", "); + } + } + + if (example != NULL) { + fprintf(stderr, "; for example on input '%s'", example); } - fprintf(f, "return 0x%u; /* %s */", r->id, r->rec); + fprintf(stderr, "\n"); return 0; } static int -endleaf_dot(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_opaque) +accept_dot(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { - const struct record *r; + fsm_state_t s; assert(f != NULL); - assert(endleaf_opaque == NULL); - (void) endleaf_opaque; + (void) hook_opaque; + + s = * (fsm_state_t *) lang_opaque; fprintf(f, "label = <"); + if (!opt->anonymous_states) { + fprintf(f, "%u", s); + + if (count > 0) { + fprintf(f, "
"); + } + } + for (size_t i = 0; i < count; i++) { - r = (const void *) (intptr_t) ids[i]; /* XXX */ + const struct record *r; + + r = find_id(ids[i]); fprintf(f, "%s", r->rec); /* XXX: escape */ + if (i + 1 < count) { fprintf(f, ", "); } @@ -471,6 +516,35 @@ endleaf_dot(FILE *f, const fsm_end_id_t *ids, size_t count, const void *endleaf_ return 0; } +static int +comment_c(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *hook_opaque) +{ + assert(f != NULL); + + (void) opt; + (void) hook_opaque; + + fprintf(f, "/* "); + + for (size_t i = 0; i < count; i++) { + const struct record *r; + + r = find_id(ids[i]); + + fprintf(f, "%s", r->rec); /* XXX: escape */ + + if (i + 1 < count) { + fprintf(f, ", "); + } + } + + fprintf(f, " */\n"); + + return 0; +} + int main(int argc, char **argv) { @@ -481,12 +555,15 @@ main(int argc, char **argv) int oc = 0; int c; + opt.ambig = AMBIG_ERROR; opt.prefix = NULL; opt.always_hex = 1; opt.anonymous_states = 1; opt.consolidate_edges = 1; opt.case_ranges = 1; + hooks.conflict = conflict; + while (c = getopt(argc, argv, "46f:l:Q"), c != -1) { switch (c) { case '4': ipv = IPV4; break; @@ -531,7 +608,7 @@ main(int argc, char **argv) memset(ones, 0xff, sizeof ones); - fsm = fsm_new(&opt); + fsm = fsm_new(NULL); if (fsm == NULL) { perror("fsm_new"); return -1; @@ -634,23 +711,31 @@ main(int argc, char **argv) struct record *r; RB_FOREACH(r, recmap, &recmap) { + struct fsm_combine_info ci; fsm_state_t start; - if (fsm_minimise(r->fsm) == 0) { + if (!fsm_determinise(r->fsm)) { + perror("fsm_determinse"); + exit(-1); + } + + if (!fsm_minimise(r->fsm)) { perror("fsm_minimise"); exit(-1); } - fsm_setendid(r->fsm, (intptr_t) r); /* XXX */ + fsm_setendid(r->fsm, r->id); (void) fsm_getstart(r->fsm, &start); - fsm = fsm_merge(fsm, r->fsm, NULL); + fsm = fsm_merge(fsm, r->fsm, &ci); if (fsm == NULL) { perror("fsm_merge"); exit(-1); } + (void) ci; + if (!fsm_addedge_epsilon(fsm, fsm_start, start)) { perror("fsm_addedge_epsilon"); exit(-1); @@ -676,7 +761,7 @@ main(int argc, char **argv) tstart = time(NULL); } - if (!fsm_determinise(fsm) == 0) { + if (!fsm_determinise(fsm)) { perror("fsm_determinise"); exit(-1); } @@ -687,15 +772,13 @@ main(int argc, char **argv) } if (oc) { - opt.fragment = 1; - opt.cp = "c"; - opt.leaf = leaf; - opt.leaf_opaque = NULL; - fsm_print(stdout, fsm, FSM_PRINT_C); + opt.fragment = 1; + opt.comments = 1; + hooks.comment = comment_c; + fsm_print(stdout, fsm, &opt, &hooks, FSM_PRINT_C); } else if (odot) { - opt.endleaf = endleaf_dot; - opt.endleaf_opaque = NULL; - fsm_print(stdout, fsm, FSM_PRINT_DOT); + hooks.accept = accept_dot; + fsm_print(stdout, fsm, &opt, &hooks, FSM_PRINT_DOT); } } diff --git a/examples/utf8dfa/main.c b/examples/utf8dfa/main.c index d987575b3..811cd76c0 100644 --- a/examples/utf8dfa/main.c +++ b/examples/utf8dfa/main.c @@ -164,7 +164,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - fsm = fsm_new(&opt); + fsm = fsm_new(NULL); if (fsm == NULL) { perror("fsm_new"); exit(1); @@ -215,12 +215,17 @@ main(int argc, char *argv[]) } } + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(1); + } + if (!fsm_minimise(fsm)) { perror("fsm_minimise"); exit(1); } - fsm_print(stdout, fsm, lang); + fsm_print(stdout, fsm, NULL, NULL, lang); fsm_free(fsm); diff --git a/examples/words/main.c b/examples/words/main.c index 1fe19ddf1..078848b78 100644 --- a/examples/words/main.c +++ b/examples/words/main.c @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } } else { - fsm = fsm_new(&opt); + fsm = fsm_new(NULL); if (fsm == NULL) { perror("fsm_new"); return 1; @@ -127,7 +127,7 @@ int main(int argc, char *argv[]) { struct fsm *r; struct fsm_combine_info ci; - r = re_comp(native ? RE_NATIVE : RE_LITERAL, fsm_sgetc, &p, &opt, 0, &e); + r = re_comp(native ? RE_NATIVE : RE_LITERAL, fsm_sgetc, &p, NULL, 0, &e); if (r == NULL) { re_perror(native ? RE_NATIVE : RE_LITERAL, &e, NULL, s); return 1; @@ -168,7 +168,7 @@ int main(int argc, char *argv[]) { } fsm = re_strings_build(g, - &opt, unanchored ? 0 : (RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT)); + NULL, unanchored ? 0 : (RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT)); if (fsm == NULL) { perror("re_strings_builder_build"); exit(EXIT_FAILURE); @@ -211,7 +211,7 @@ int main(int argc, char *argv[]) { + ((long) post.tv_nsec - (long) pre.tv_nsec) / 1000000; } - fsm_print(stdout, fsm, lang); + fsm_print(stdout, fsm, &opt, NULL, lang); if (timing) { printf("construction, reduction, total: %lu, %lu, %lu\n", ms, mt, ms + mt); From 8f9ea4d13941c09074ed73cf71d02ebadb85f1f1 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Aug 2024 12:01:35 +0100 Subject: [PATCH 259/373] Spelling. Spotted by Dan Kegel, thank you. --- man/rx.1/rx.1.xml | 2 +- src/rx/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/man/rx.1/rx.1.xml b/man/rx.1/rx.1.xml index 5ce8d5b27..a9f1ffd18 100644 --- a/man/rx.1/rx.1.xml +++ b/man/rx.1/rx.1.xml @@ -98,7 +98,7 @@ Description - &rx.1; is a command line too to compile sets of + &rx.1; is a command line tool to compile sets of regular expressions to executable code. Patterns are read from one or more files, with multiple patterns per file. Patterns that &rx.1; declines to handle are written out diff --git a/src/rx/main.c b/src/rx/main.c index 5ed4508b9..0328f900a 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -1041,7 +1041,7 @@ main(int argc, char *argv[]) * On end id numbering: The UI must be one pattern per line, because * it's the only format that doesn't introduce a syntax with escaping. * So that means we need to track the line number associated with each - * regex. And that means the line number is neccessarily the end id. + * regex. And that means the line number is necessarily the end id. * So each pattern carries its associated end id. * * We have four whole FSMs from re_strings(), each potentially contains From 981223fd3674123a7de5976cf873ed3548620288 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 21 May 2024 12:44:19 -0400 Subject: [PATCH 260/373] bitmap.h: Add operations: copy, intersect, union, any, unset.. Most of these are used in fsm_detect_required_characters. Also add #includes for standard headers being used. --- include/adt/bitmap.h | 18 ++++++++++++++++++ src/adt/bitmap.c | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/include/adt/bitmap.h b/include/adt/bitmap.h index ac4a94995..61c4f9a57 100644 --- a/include/adt/bitmap.h +++ b/include/adt/bitmap.h @@ -8,6 +8,9 @@ #define ADT_BITMAP_H #include +#include + +#include #include "print/esc.h" struct fsm_state; @@ -23,6 +26,9 @@ bm_get(const struct bm *bm, size_t i); void bm_set(struct bm *bm, size_t i); +void +bm_unset(struct bm *bm, size_t i); + /* Get a writeable pointer to the Nth word of the char set bitmap, * or NULL if out of bounds. */ uint64_t * @@ -51,5 +57,17 @@ bm_snprint(const struct bm *bm, const struct fsm_options *opt, int boxed, escputc *escputc); +void +bm_copy(struct bm *dst, const struct bm *src); + +void +bm_intersect(struct bm *dst, const struct bm *src); + +void +bm_union(struct bm *dst, const struct bm *src); + +int +bm_any(const struct bm *bm); + #endif diff --git a/src/adt/bitmap.c b/src/adt/bitmap.c index 434dfc222..1fb016083 100644 --- a/src/adt/bitmap.c +++ b/src/adt/bitmap.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,15 @@ bm_set(struct bm *bm, size_t i) u64bitset_set(bm->map, i); } +void +bm_unset(struct bm *bm, size_t i) +{ + assert(bm != NULL); + assert(i <= UCHAR_MAX); + + u64bitset_clear(bm->map, i); +} + uint64_t * bm_nth_word(struct bm *bm, size_t n) { @@ -325,3 +335,34 @@ bm_snprint(const struct bm *bm, const struct fsm_options *opt, return -1; } + +void +bm_copy(struct bm *dst, const struct bm *src) +{ + memcpy(dst, src, sizeof(*src)); +} + +void +bm_intersect(struct bm *dst, const struct bm *src) +{ + for (size_t i = 0; i < sizeof(src->map)/sizeof(src->map[0]); i++) { + dst->map[i] &= src->map[i]; + } +} + +void +bm_union(struct bm *dst, const struct bm *src) +{ + for (size_t i = 0; i < sizeof(src->map)/sizeof(src->map[0]); i++) { + dst->map[i] |= src->map[i]; + } +} + +int +bm_any(const struct bm *bm) +{ + for (size_t i = 0; i < sizeof(bm->map)/sizeof(bm->map[0]); i++) { + if (bm->map[i]) { return 1; } + } + return 0; +} From fa24439b02728ecd76e4ef16b84600642ad3c888 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 21 May 2024 12:50:21 -0400 Subject: [PATCH 261/373] Add fsm_detect_required_characters. This inspects the DFA to determine which characters must appear in any matching input. --- Makefile | 1 + include/fsm/walk.h | 31 ++ src/libfsm/Makefile | 1 + src/libfsm/detect_required.c | 264 ++++++++++++++++++ src/libfsm/libfsm.syms | 1 + tests/detect_required/Makefile | 26 ++ tests/detect_required/detect_required1.c | 32 +++ .../detect_required_step_limit.c | 58 ++++ tests/detect_required/testutil.c | 88 ++++++ tests/detect_required/testutil.h | 21 ++ 10 files changed, 523 insertions(+) create mode 100644 src/libfsm/detect_required.c create mode 100644 tests/detect_required/Makefile create mode 100644 tests/detect_required/detect_required1.c create mode 100644 tests/detect_required/detect_required_step_limit.c create mode 100644 tests/detect_required/testutil.c create mode 100644 tests/detect_required/testutil.h diff --git a/Makefile b/Makefile index 177f8456a..514f80bba 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,7 @@ SUBDIR += tests/intersect SUBDIR += tests/eclosure SUBDIR += tests/equals SUBDIR += tests/subtract +SUBDIR += tests/detect_required SUBDIR += tests/determinise SUBDIR += tests/endids SUBDIR += tests/epsilons diff --git a/include/fsm/walk.h b/include/fsm/walk.h index b433380d5..bc030a017 100644 --- a/include/fsm/walk.h +++ b/include/fsm/walk.h @@ -7,6 +7,8 @@ #ifndef FSM_WALK_H #define FSM_WALK_H +#include + struct fsm; struct fsm_state; @@ -128,5 +130,34 @@ fsm_generate_matches_cb fsm_generate_cb_printf; * to escape all characters or just nonprintable ones. */ fsm_generate_matches_cb fsm_generate_cb_printf_escaped; +/* Walk a DFA and detect which characters MUST appear in the input for a + * match to be possible. For example, if input for the DFA corresponding + * to /^(abc|dbe)$/ does not contain 'b' at all, there's no way it can + * ever match, so executing the regex is unnecessary. This does not detect + * which characters must appear before/after others or how many times, just + * which must be present. + * + * The input must be a DFA. When run with EXPENSIVE_CHECKS this will + * check and return ERROR_MISUSE if it is not, otherwise this is an + * unchecked error. + * + * The bitmap will be cleared before populating. Afterward, + * bm_count(bitmap) will return how many required characters were + * found. + * + * There is an optional step_limit -- if this is reached, then it will + * return FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED and a + * cleared bitmap, because any partial information could still have been + * contradicted later. If the step_limit is 0 it will be ignored. */ +enum fsm_detect_required_characters_res { + FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN, + FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED, + FSM_DETECT_REQUIRED_CHARACTERS_ERROR_MISUSE = -1, + FSM_DETECT_REQUIRED_CHARACTERS_ERROR_ALLOC = -2, +}; +enum fsm_detect_required_characters_res +fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, + struct bm *bitmap); + #endif diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index c1085763e..5e2ed57e3 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -7,6 +7,7 @@ SRC += src/libfsm/complete.c SRC += src/libfsm/consolidate.c SRC += src/libfsm/clone.c SRC += src/libfsm/closure.c +SRC += src/libfsm/detect_required.c SRC += src/libfsm/edge.c SRC += src/libfsm/empty.c SRC += src/libfsm/end.c diff --git a/src/libfsm/detect_required.c b/src/libfsm/detect_required.c new file mode 100644 index 000000000..3701c7150 --- /dev/null +++ b/src/libfsm/detect_required.c @@ -0,0 +1,264 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "internal.h" + +#define LOG_BASE 0 +#define LOG_PROGRESS (LOG_BASE + 0) +#define LOG_STEPS (LOG_BASE + 0) + +/* More than one label */ +#define LABEL_GROUP ((uint16_t)-1) + +struct dr_env { + const struct fsm *dfa; + size_t steps; + + /* Number of times a unique label has been required -- this is a count so that going + * from 0 <-> 1 can set/clear the accumulator, but going from 1 -> 2 etc. does not. */ + size_t counts[256]; + struct bm current; + bool first_end_state; + struct bm overall; + + struct dr_stack { + size_t used; + size_t ceil; + struct stack_frame { + fsm_state_t state; + uint16_t label; /* unique label followed to get here, or LABEL_GROUP */ + struct edge_group_iter iter; + } *frames; + } stack; +}; + +#define DEF_STACK_FRAMES 16 + +/* Check symbols[]: if there's more than one bit set, then set label to + * LABEL_GROUP, otherwise set it to the single bit's character value. + * At least one bit must be set. */ +static void check_symbols(const struct edge_group_iter_info *info, uint16_t *label) +{ + bool any = false; + + for (size_t i = 0; i < 256/64; i++) { + uint64_t w = info->symbols[i]; + if (w == 0) { continue; } + + /* get position of lowest set bit */ + for (size_t b = 0; b < 64; b++) { + const uint64_t bit = 1ULL << b; + if (w & bit) { + if (any) { + *label = LABEL_GROUP; + return; + } + + /* clear it, check if there's anything else set */ + w &= ~bit; + if (w != 0) { + *label = LABEL_GROUP; + return; + } + + *label = 64*i + b; + any = true; + break; + } + } + } + + /* there must be at least one bit set */ + assert(any); +} + +/* Walk a DFA and attempt to detect which characters must appear in any input to match. + * This finds the intersection of characters required on any start->end paths (tracking + * edges with only one label that must be followed by all matches), so it can take + * prohibitively long for large/complex DFAs. */ +enum fsm_detect_required_characters_res +fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, struct bm *bitmap) +{ + assert(dfa != NULL); + assert(bitmap != NULL); + + #if EXPENSIVE_CHECKS + if (!fsm_all(dfa, fsm_isdfa)) { + return FSM_DETECT_REQUIRED_CHARACTERS_ERROR_MISUSE; + } + #endif + + enum fsm_detect_required_characters_res res = FSM_DETECT_REQUIRED_CHARACTERS_ERROR_ALLOC; + + struct dr_env env = { + .dfa = dfa, + .first_end_state = true, + }; + + assert(env.counts[0] == 0); + + const size_t state_count = fsm_countstates(dfa); + fsm_state_t start_state; + if (!fsm_getstart(dfa, &start_state)) { + res = FSM_DETECT_REQUIRED_CHARACTERS_ERROR_MISUSE; + goto cleanup; + } + + #if EXPENSIVE_CHECKS + for (fsm_state_t s = 0; s < state_count; s++) { + assert(!dfa->states[s].visited); + } + #endif + + bm_clear(bitmap); + + /* If the start state is also an end state, then + * it matches the empty string, so we're done. */ + if (fsm_isend(dfa, start_state)) { + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + goto cleanup; + } + + env.stack.frames = f_malloc(dfa->alloc, DEF_STACK_FRAMES * sizeof(env.stack.frames[0])); + if (env.stack.frames == NULL) { goto cleanup; } + env.stack.ceil = DEF_STACK_FRAMES; + + { /* set up start state's stack frame */ + struct stack_frame *sf0 = &env.stack.frames[0]; + sf0->state = start_state; + sf0->label = LABEL_GROUP; + + dfa->states[start_state].visited = true; + + edge_set_group_iter_reset(dfa->states[start_state].edges, + EDGE_GROUP_ITER_ALL, &sf0->iter); + env.stack.used = 1; + } + + while (env.stack.used > 0) { + struct stack_frame *sf = &env.stack.frames[env.stack.used - 1]; + struct edge_group_iter_info info; + env.steps++; + if (LOG_STEPS > 1) { + fprintf(stderr, "-- steps %zu/%zu\n", env.steps, step_limit); + } + if (env.steps == step_limit) { + res = FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED; + goto cleanup; + } + + if (edge_set_group_iter_next(&sf->iter, &info)) { + assert(info.to < state_count); + if (dfa->states[info.to].visited) { + continue; /* skip visited state */ + } + + if (env.stack.used == env.stack.ceil) { /* grow stack */ + const size_t nceil = 2*env.stack.ceil; + assert(nceil > env.stack.ceil); + struct stack_frame *nframes = f_realloc(dfa->alloc, + env.stack.frames, nceil * sizeof(nframes[0])); + if (nframes == NULL) { + return FSM_DETECT_REQUIRED_CHARACTERS_ERROR_ALLOC; + } + + env.stack.frames = nframes; + env.stack.ceil = nceil; + } + + /* enter state */ + dfa->states[info.to].visited = true; + + struct stack_frame *nsf = &env.stack.frames[env.stack.used]; + nsf->state = info.to; + check_symbols(&info, &nsf->label); + + if (nsf->label != LABEL_GROUP) { + size_t offset = (nsf->label & 0xff); + size_t count = ++env.counts[offset]; + if (count == 1) { + bm_set(&env.current, offset); + } + } + + edge_set_group_iter_reset(dfa->states[info.to].edges, + EDGE_GROUP_ITER_ALL, &nsf->iter); + env.stack.used++; + + if (fsm_isend(dfa, info.to)) { + if (env.first_end_state) { + bm_copy(&env.overall, &env.current); + env.first_end_state = false; + } else { /* intersect */ + bm_intersect(&env.overall, &env.current); + } + + if (LOG_PROGRESS) { + fprintf(stderr, "-- current: "); + bm_print(stderr, NULL, &env.current, 0, fsm_escputc); + fprintf(stderr, ", overall: "); + bm_print(stderr, NULL, &env.overall, 0, fsm_escputc); + fprintf(stderr, "\n"); + } + + /* Intersecting with the empty set will always be empty, so + * further exploration is unnecessary. */ + if (!bm_any(&env.overall)) { + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + break; + } + } + + } else { /* done with state */ + /* If this state was reached via a unique label, then + * reduce the count. If the count returns to 0, remove + * it from the constraint set. */ + if (sf->label != LABEL_GROUP) { + size_t offset = (sf->label & 0xff); + size_t count = --env.counts[offset]; + if (count == 0) { + bm_unset(&env.current, offset); + } + } + + /* clear visited */ + dfa->states[sf->state].visited = false; + + env.stack.used--; + } + } + + if (LOG_STEPS) { + fprintf(stderr, "%s: finished in %zu/%zu steps\n", __func__, env.steps, step_limit); + } + + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + bm_copy(bitmap, &env.overall); + +cleanup: + f_free(dfa->alloc, env.stack.frames); + + for (fsm_state_t s = 0; s < state_count; s++) { + dfa->states[s].visited = false; + } + + return res; +} diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 56734e7f2..40670e8fc 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -16,6 +16,7 @@ fsm_reachableall fsm_reachableany fsm_walk_edges fsm_walk_states +fsm_detect_required_characters # fsm_epsilonsonly diff --git a/tests/detect_required/Makefile b/tests/detect_required/Makefile new file mode 100644 index 000000000..34214f07e --- /dev/null +++ b/tests/detect_required/Makefile @@ -0,0 +1,26 @@ +.include "../../share/mk/top.mk" + +TEST.tests/detect_required != ls -1 tests/detect_required/detect_required*.c +TEST_SRCDIR.tests/detect_required = tests/detect_required +TEST_OUTDIR.tests/detect_required = ${BUILD}/tests/detect_required + +.for n in ${TEST.tests/detect_required:T:R:C/^detect_required//} +test:: ${TEST_OUTDIR.tests/detect_required}/res${n} +SRC += ${TEST_SRCDIR.tests/detect_required}/detect_required${n}.c +CFLAGS.${TEST_SRCDIR.tests/detect_required}/detect_required${n}.c = -UNDEBUG + +${TEST_OUTDIR.tests/detect_required}/run${n}: ${TEST_OUTDIR.tests/detect_required}/detect_required${n}.o ${TEST_OUTDIR.tests/detect_required}/testutil.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/detect_required}/run${n} ${TEST_OUTDIR.tests/detect_required}/detect_required${n}.o ${TEST_OUTDIR.tests/detect_required}/testutil.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + +${TEST_OUTDIR.tests/detect_required}/detect_required${n}.o: tests/detect_required/testutil.h + +${TEST_OUTDIR.tests/detect_required}/res${n}: ${TEST_OUTDIR.tests/detect_required}/run${n} + ( ${TEST_OUTDIR.tests/detect_required}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/detect_required}/res${n} + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${TEST_OUTDIR.tests/detect_required}/run${n}: ${BUILD}/lib/${lib:R}.a +.endfor +.endfor + +${TEST_OUTDIR.tests/detect_required}/testutil.o: tests/detect_required/testutil.c + ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/detect_required}/testutil.o tests/detect_required/testutil.c diff --git a/tests/detect_required/detect_required1.c b/tests/detect_required/detect_required1.c new file mode 100644 index 000000000..52bb83477 --- /dev/null +++ b/tests/detect_required/detect_required1.c @@ -0,0 +1,32 @@ +#include "testutil.h" + +const struct testcase tests[] = { + { .regex = "^$", .required = "" }, + { .regex = "^a$", .required = "a" }, + { .regex = "^abcde$", .required = "abcde" }, + { .regex = "^(ab|cd)$", .required = "" }, + { .regex = "^(ab|cd|ef)$", .required = "" }, + { .regex = "^(abc|def)$", .required = "" }, + { .regex = "^(abc|dbf)$", .required = "b" }, + { .regex = "^abc(def)*ghi$", .required = "abcghi" }, + { .regex = "^abc(def)+ghi$", .required = "abcdefghi" }, + { .regex = "^ghi(def)abc$", .required = "abcdefghi" }, +}; + +int main() +{ + const bool first_fail = getenv("FIRST_FAIL") != NULL; + const size_t testcount = sizeof(tests)/sizeof(tests[0]); + + size_t failures = 0; + for (size_t i = 0; i < testcount; i++) { + if (!run_test(&tests[i])) { + failures++; + if (first_fail) { break; } + } + } + + return failures == 0 + ? EXIT_SUCCESS + : EXIT_FAILURE; +} diff --git a/tests/detect_required/detect_required_step_limit.c b/tests/detect_required/detect_required_step_limit.c new file mode 100644 index 000000000..42c1c0f5e --- /dev/null +++ b/tests/detect_required/detect_required_step_limit.c @@ -0,0 +1,58 @@ +#include "testutil.h" + +#include +#include +#include +#include +#include +#include + +int main() +{ + enum re_flags flags = 0; + struct re_err err; + const char *regex = "^abcde$"; + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, ®ex, NULL, flags, &err); + assert(fsm != NULL); + + if (!fsm_determinise(fsm)) { + assert(!"determinise"); + return EXIT_FAILURE; + } + if (!fsm_minimise(fsm)) { + assert(!"minimise"); + return EXIT_FAILURE; + } + + struct bm bitmap; + + /* keep decreasing the step limit until it's hit, and check that + * the bitmap is cleared. */ + bool hit_step_limit = false; + size_t step_limit = 25; + while (!hit_step_limit) { + assert(step_limit > 0); + + const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, &bitmap); + if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { + hit_step_limit = true; + + /* this should not contain any partially complete information */ + for (size_t i = 0; i < 4; i++) { + const uint64_t *w = bm_nth_word(&bitmap, i); + if (*w != 0) { + fprintf(stderr, "-- Test failure: partial information set when step limit reached\n"); + return EXIT_FAILURE; + } + } + } + + step_limit--; + } + assert(hit_step_limit); + printf("-- successfully hit step limit at %zd\n", step_limit); + + fsm_free(fsm); + return EXIT_SUCCESS; +} diff --git a/tests/detect_required/testutil.c b/tests/detect_required/testutil.c new file mode 100644 index 000000000..a9682ee7d --- /dev/null +++ b/tests/detect_required/testutil.c @@ -0,0 +1,88 @@ +#include "testutil.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +bool +run_test(const struct testcase *tc) +{ + bool test_res = false; + + enum re_flags flags = 0; + struct re_err err; + char *regex = (char *)tc->regex; + const char *required = tc->required ? tc->required : ""; + const size_t step_limit = tc->step_limit ? tc->step_limit : DEF_STEP_LIMIT; + + fprintf(stderr, "-- test: regex '%s', required '%s'\n", tc->regex, required); + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, ®ex, NULL, flags, &err); + if (fsm == NULL) { + return false; + } + + if (!fsm_determinise(fsm)) { + assert(!"determinise"); + return false; + } + if (!fsm_minimise(fsm)) { + assert(!"minimise"); + return false; + } + + struct bm bitmap; + bm_clear(&bitmap); + + { + const size_t statecount = fsm_countstates(fsm); + size_t ends = 0; + for (size_t i = 0; i < statecount; i++) { + if (fsm_isend(fsm, i)) { + ends++; + } + } + fprintf(stderr, "-- statecount %zu, %zu ends\n", statecount, ends); + } + + + const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, &bitmap); + if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { + fprintf(stderr, "-- step limit reached, halting\n"); + goto cleanup; + } + assert(res == FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN); + + char buf[257] = {0}; + size_t used = 0; + assert(!bm_get(&bitmap, 0)); /* does not contain 0x00 */ + + int i = 0; + for (;;) { + const size_t next = bm_next(&bitmap, i, 1); + if (next > UCHAR_MAX) { break; } + buf[used++] = (char)next; + i = next; + } + + if (0 != strcmp(required, buf)) { + fprintf(stderr, "Error: mismatch\n"); + fprintf(stderr, "-- expected: [%s]\n", required); + fprintf(stderr, "-- got: [%s]\n", buf); + goto cleanup; + } + + test_res = true; + +cleanup: + fsm_free(fsm); + + return test_res; +} diff --git a/tests/detect_required/testutil.h b/tests/detect_required/testutil.h new file mode 100644 index 000000000..f9378c190 --- /dev/null +++ b/tests/detect_required/testutil.h @@ -0,0 +1,21 @@ +#ifndef TESTUTIL_H +#define TESTUTIL_H + +#include +#include +#include +#include + +#define DEF_STEP_LIMIT 100000 + +struct testcase { + const char *regex; + const char *required; + size_t max_gen_buffer; /* 0: default */ + size_t step_limit; +}; + +bool +run_test(const struct testcase *tc); + +#endif From 6c612491ffdce9eea6042faf4afb7722139460da Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 12 Sep 2024 16:18:07 -0400 Subject: [PATCH 262/373] bugfix: fsm_endid_get should sort with result count, not buffer size. fsm_endid_get's id_buf_count argument is expected to "have enough cells (according to id_buf_count)", but if it has more than enough, stale data can get sorted into the result. Add a test, tests/endids/endids_reused_buffer.c --- src/libfsm/endids.c | 4 ++- tests/endids/endids_reused_buffer.c | 56 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/endids/endids_reused_buffer.c diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index d889d057b..05f49b3d2 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -689,6 +689,7 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, { size_t i; const struct endid_info *ei = NULL; + size_t result_count = 0; uint64_t hash = hash_id(end_state); uint64_t mask; @@ -735,9 +736,10 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, #endif ids[id_i] = b->ids->ids[id_i]; } + result_count = b->ids->count; /* sorting for caller convenience */ - qsort(ids, count, sizeof *ids, cmp_endids); + qsort(ids, result_count, sizeof *ids, cmp_endids); return 1; } else { /* collision */ diff --git a/tests/endids/endids_reused_buffer.c b/tests/endids/endids_reused_buffer.c new file mode 100644 index 000000000..c62eb1dca --- /dev/null +++ b/tests/endids/endids_reused_buffer.c @@ -0,0 +1,56 @@ +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include "endids_utils.h" + +/* Test that providing an overly large endid buffer doesn't clobber the result with stale data. */ +int main(void) +{ + const char *input = "banana turtle"; + + const char *s = input; + struct fsm *fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + int ret; + + ret = fsm_setendid(fsm, (fsm_end_id_t) 12345); + assert(ret == 1); + + ret = fsm_determinise(fsm); + assert(ret == 1); + + ret = fsm_minimise(fsm); + assert(ret == 1); + + fsm_state_t end; + ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + assert(ret == 1); + + const size_t endid_count = fsm_endid_count(fsm, end); + assert(endid_count == 1); + + /* Regression: fsm_endid_get previously qsort'd endids with + * a count based on the buffer size (here ENDID_BUF_CEIL), + * rather than the number of results populated, so if the + * buffer was larger than fsm_endid_count's result stale + * contents could displace the correct results. */ +#define ENDID_BUF_CEIL 10 + fsm_end_id_t endids[ENDID_BUF_CEIL] = { 333 }; + ret = fsm_endid_get(fsm, end, ENDID_BUF_CEIL, endids); + assert(endids[0] == 12345); + + return EXIT_SUCCESS; +} From dec6ec008a80523809889700fd71c21dd4199e00 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 12 Sep 2024 16:46:39 -0400 Subject: [PATCH 263/373] endids_reused_buffer.c: Fix memory leak in test. (free fsm.) --- tests/endids/endids_reused_buffer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/endids/endids_reused_buffer.c b/tests/endids/endids_reused_buffer.c index c62eb1dca..62fe1a5f8 100644 --- a/tests/endids/endids_reused_buffer.c +++ b/tests/endids/endids_reused_buffer.c @@ -52,5 +52,6 @@ int main(void) ret = fsm_endid_get(fsm, end, ENDID_BUF_CEIL, endids); assert(endids[0] == 12345); + fsm_free(fsm); return EXIT_SUCCESS; } From 5705d43fea60e47c90ddffc5f2719c3d3a51a4bb Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Tue, 17 Sep 2024 15:50:04 +0500 Subject: [PATCH 264/373] Update to Unicode 16.0 --- share/ucd/CaseFolding.txt | 35 +- share/ucd/Makefile | 2 +- share/ucd/Scripts.txt | 153 +- share/ucd/UnicodeData.txt | 5203 ++++++++++++++++++- src/libre/class.h | 11 +- src/libre/class/utf8_Arabic.c | 5 +- src/libre/class/utf8_Balinese.c | 2 +- src/libre/class/utf8_Common.c | 21 +- src/libre/class/utf8_Cyrillic.c | 2 +- src/libre/class/utf8_Egyptian_Hieroglyphs.c | 3 +- src/libre/class/utf8_Garay.c | 15 + src/libre/class/utf8_Gurung_Khema.c | 13 + src/libre/class/utf8_Kawi.c | 2 +- src/libre/class/utf8_Khitan_Small_Script.c | 3 +- src/libre/class/utf8_Kirat_Rai.c | 13 + src/libre/class/utf8_L.c | 25 +- src/libre/class/utf8_Latin.c | 4 +- src/libre/class/utf8_Ll.c | 4 + src/libre/class/utf8_Lm.c | 4 + src/libre/class/utf8_Lo.c | 19 +- src/libre/class/utf8_Lu.c | 5 + src/libre/class/utf8_M.c | 15 +- src/libre/class/utf8_Mc.c | 8 + src/libre/class/utf8_Mn.c | 17 +- src/libre/class/utf8_Myanmar.c | 3 +- src/libre/class/utf8_N.c | 7 + src/libre/class/utf8_Nd.c | 7 + src/libre/class/utf8_Ol_Onal.c | 14 + src/libre/class/utf8_P.c | 9 +- src/libre/class/utf8_Pd.c | 1 + src/libre/class/utf8_Po.c | 8 +- src/libre/class/utf8_S.c | 21 +- src/libre/class/utf8_Sm.c | 1 + src/libre/class/utf8_So.c | 20 +- src/libre/class/utf8_Sunuwar.c | 14 + src/libre/class/utf8_Todhri.c | 13 + src/libre/class/utf8_Tulu_Tigalari.c | 23 + src/libre/class/utf8_assigned.c | 62 +- src/libre/class_name.c | 15 +- 39 files changed, 5686 insertions(+), 116 deletions(-) create mode 100644 src/libre/class/utf8_Garay.c create mode 100644 src/libre/class/utf8_Gurung_Khema.c create mode 100644 src/libre/class/utf8_Kirat_Rai.c create mode 100644 src/libre/class/utf8_Ol_Onal.c create mode 100644 src/libre/class/utf8_Sunuwar.c create mode 100644 src/libre/class/utf8_Todhri.c create mode 100644 src/libre/class/utf8_Tulu_Tigalari.c diff --git a/share/ucd/CaseFolding.txt b/share/ucd/CaseFolding.txt index 69c5c64b4..1b7a9c156 100644 --- a/share/ucd/CaseFolding.txt +++ b/share/ucd/CaseFolding.txt @@ -1,8 +1,8 @@ -# CaseFolding-15.1.0.txt -# Date: 2023-05-12, 21:53:10 GMT -# © 2023 Unicode®, Inc. +# CaseFolding-16.0.0.txt +# Date: 2024-04-30, 21:48:11 GMT +# © 2024 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see https://www.unicode.org/terms_of_use.html +# For terms of use and license, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database # For documentation, see https://www.unicode.org/reports/tr44/ @@ -603,6 +603,7 @@ 1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN 1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT 1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1C89; C; 1C8A; # CYRILLIC CAPITAL LETTER TJE 1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN 1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN 1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN @@ -1240,9 +1241,13 @@ A7C5; C; 0282; # LATIN CAPITAL LETTER S WITH HOOK A7C6; C; 1D8E; # LATIN CAPITAL LETTER Z WITH PALATAL HOOK A7C7; C; A7C8; # LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY A7C9; C; A7CA; # LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY +A7CB; C; 0264; # LATIN CAPITAL LETTER RAMS HORN +A7CC; C; A7CD; # LATIN CAPITAL LETTER S WITH DIAGONAL STROKE A7D0; C; A7D1; # LATIN CAPITAL LETTER CLOSED INSULAR G A7D6; C; A7D7; # LATIN CAPITAL LETTER MIDDLE SCOTS S A7D8; C; A7D9; # LATIN CAPITAL LETTER SIGMOID S +A7DA; C; A7DB; # LATIN CAPITAL LETTER LAMBDA +A7DC; C; 019B; # LATIN CAPITAL LETTER LAMBDA WITH STROKE A7F5; C; A7F6; # LATIN CAPITAL LETTER REVERSED HALF H AB70; C; 13A0; # CHEROKEE SMALL LETTER A AB71; C; 13A1; # CHEROKEE SMALL LETTER E @@ -1525,6 +1530,28 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z 10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS 10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN 10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US +10D50; C; 10D70; # GARAY CAPITAL LETTER A +10D51; C; 10D71; # GARAY CAPITAL LETTER CA +10D52; C; 10D72; # GARAY CAPITAL LETTER MA +10D53; C; 10D73; # GARAY CAPITAL LETTER KA +10D54; C; 10D74; # GARAY CAPITAL LETTER BA +10D55; C; 10D75; # GARAY CAPITAL LETTER JA +10D56; C; 10D76; # GARAY CAPITAL LETTER SA +10D57; C; 10D77; # GARAY CAPITAL LETTER WA +10D58; C; 10D78; # GARAY CAPITAL LETTER LA +10D59; C; 10D79; # GARAY CAPITAL LETTER GA +10D5A; C; 10D7A; # GARAY CAPITAL LETTER DA +10D5B; C; 10D7B; # GARAY CAPITAL LETTER XA +10D5C; C; 10D7C; # GARAY CAPITAL LETTER YA +10D5D; C; 10D7D; # GARAY CAPITAL LETTER TA +10D5E; C; 10D7E; # GARAY CAPITAL LETTER RA +10D5F; C; 10D7F; # GARAY CAPITAL LETTER NYA +10D60; C; 10D80; # GARAY CAPITAL LETTER FA +10D61; C; 10D81; # GARAY CAPITAL LETTER NA +10D62; C; 10D82; # GARAY CAPITAL LETTER PA +10D63; C; 10D83; # GARAY CAPITAL LETTER HA +10D64; C; 10D84; # GARAY CAPITAL LETTER OLD KA +10D65; C; 10D85; # GARAY CAPITAL LETTER OLD NA 118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA 118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A 118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI diff --git a/share/ucd/Makefile b/share/ucd/Makefile index c5b9781d0..833910330 100644 --- a/share/ucd/Makefile +++ b/share/ucd/Makefile @@ -1,4 +1,4 @@ -UCD_URL ?= https://www.unicode.org/Public/15.1.0/ucd/ +UCD_URL ?= https://www.unicode.org/Public/16.0.0/ucd/ WGET ?= wget diff --git a/share/ucd/Scripts.txt b/share/ucd/Scripts.txt index 0b3f717cb..443a6d2dd 100644 --- a/share/ucd/Scripts.txt +++ b/share/ucd/Scripts.txt @@ -1,8 +1,8 @@ -# Scripts-15.1.0.txt -# Date: 2023-07-28, 16:01:07 GMT -# © 2023 Unicode®, Inc. +# Scripts-16.0.0.txt +# Date: 2024-04-30, 21:48:40 GMT +# © 2024 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see https://www.unicode.org/terms_of_use.html +# For terms of use and license, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database # For documentation, see https://www.unicode.org/reports/tr44/ @@ -224,7 +224,7 @@ 239B..23B3 ; Common # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM 23B4..23DB ; Common # So [40] TOP SQUARE BRACKET..FUSE 23DC..23E1 ; Common # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET -23E2..2426 ; Common # So [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO +23E2..2429 ; Common # So [72] WHITE TRAPEZIUM..SYMBOL FOR DELETE MEDIUM SHADE FORM 2440..244A ; Common # So [11] OCR HOOK..OCR DOUBLE BACKSLASH 2460..249B ; Common # No [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP 249C..24E9 ; Common # So [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z @@ -398,7 +398,7 @@ 3190..3191 ; Common # So [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK 3192..3195 ; Common # No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK 3196..319F ; Common # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK -31C0..31E3 ; Common # So [36] CJK STROKE T..CJK STROKE Q +31C0..31E5 ; Common # So [38] CJK STROKE T..CJK STROKE SZP 31EF ; Common # So IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION 3220..3229 ; Common # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN 322A..3247 ; Common # So [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO @@ -522,6 +522,9 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 101D0..101FC ; Common # So [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND 102E1..102FB ; Common # No [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED 1BCA0..1BCA3 ; Common # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP +1CC00..1CCEF ; Common # So [240] UP-POINTING GO-KART..OUTLINED LATIN CAPITAL LETTER Z +1CCF0..1CCF9 ; Common # Nd [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE +1CD00..1CEB3 ; Common # So [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET 1CF50..1CFC3 ; Common # So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK 1D000..1D0F5 ; Common # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO 1D100..1D126 ; Common # So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 @@ -614,23 +617,23 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 1F850..1F859 ; Common # So [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW 1F860..1F887 ; Common # So [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW 1F890..1F8AD ; Common # So [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS -1F8B0..1F8B1 ; Common # So [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST +1F8B0..1F8BB ; Common # So [12] ARROW POINTING UPWARDS THEN NORTH WEST..SOUTH WEST ARROW FROM BAR +1F8C0..1F8C1 ; Common # So [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW 1F900..1FA53 ; Common # So [340] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS KNIGHT-BISHOP 1FA60..1FA6D ; Common # So [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER 1FA70..1FA7C ; Common # So [13] BALLET SHOES..CRUTCH -1FA80..1FA88 ; Common # So [9] YO-YO..FLUTE -1FA90..1FABD ; Common # So [46] RINGED PLANET..WING -1FABF..1FAC5 ; Common # So [7] GOOSE..PERSON WITH CROWN -1FACE..1FADB ; Common # So [14] MOOSE..PEA POD -1FAE0..1FAE8 ; Common # So [9] MELTING FACE..SHAKING FACE +1FA80..1FA89 ; Common # So [10] YO-YO..HARP +1FA8F..1FAC6 ; Common # So [56] SHOVEL..FINGERPRINT +1FACE..1FADC ; Common # So [15] MOOSE..ROOT VEGETABLE +1FADF..1FAE9 ; Common # So [11] SPLATTER..FACE WITH BAGS UNDER EYES 1FAF0..1FAF8 ; Common # So [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND 1FB00..1FB92 ; Common # So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK -1FB94..1FBCA ; Common # So [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON +1FB94..1FBEF ; Common # So [92] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE 1FBF0..1FBF9 ; Common # Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE E0001 ; Common # Cf LANGUAGE TAG E0020..E007F ; Common # Cf [96] TAG SPACE..CANCEL TAG -# Total code points: 8306 +# Total code points: 9053 # ================================================ @@ -673,10 +676,10 @@ A770 ; Latin # Lm MODIFIER LETTER US A771..A787 ; Latin # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T A78B..A78E ; Latin # L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT A78F ; Latin # Lo LATIN LETTER SINOLOGICAL DOT -A790..A7CA ; Latin # L& [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY +A790..A7CD ; Latin # L& [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE A7D0..A7D1 ; Latin # L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G A7D3 ; Latin # L& LATIN SMALL LETTER DOUBLE THORN -A7D5..A7D9 ; Latin # L& [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S +A7D5..A7DC ; Latin # L& [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE A7F2..A7F4 ; Latin # Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q A7F5..A7F6 ; Latin # L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H A7F7 ; Latin # Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I @@ -699,7 +702,7 @@ FF41..FF5A ; Latin # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN 1DF0B..1DF1E ; Latin # L& [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL 1DF25..1DF2A ; Latin # L& [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK -# Total code points: 1481 +# Total code points: 1487 # ================================================ @@ -769,7 +772,7 @@ AB65 ; Greek # L& GREEK LETTER SMALL CAPITAL OMEGA 0487 ; Cyrillic # Mn COMBINING CYRILLIC POKRYTIE 0488..0489 ; Cyrillic # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN 048A..052F ; Cyrillic # L& [166] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EL WITH DESCENDER -1C80..1C88 ; Cyrillic # L& [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK +1C80..1C8A ; Cyrillic # L& [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE 1D2B ; Cyrillic # L& CYRILLIC LETTER SMALL CAPITAL EL 1D78 ; Cyrillic # Lm MODIFIER LETTER CYRILLIC EN 2DE0..2DFF ; Cyrillic # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS @@ -788,7 +791,7 @@ FE2E..FE2F ; Cyrillic # Mn [2] COMBINING CYRILLIC TITLO LEFT HALF..COMBININ 1E030..1E06D ; Cyrillic # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE 1E08F ; Cyrillic # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I -# Total code points: 506 +# Total code points: 508 # ================================================ @@ -868,7 +871,7 @@ FB46..FB4F ; Hebrew # Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATU 0888 ; Arabic # Sk ARABIC RAISED ROUND DOT 0889..088E ; Arabic # Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL 0890..0891 ; Arabic # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE -0898..089F ; Arabic # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +0897..089F ; Arabic # Mn [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA 08A0..08C8 ; Arabic # Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF 08C9 ; Arabic # Lm ARABIC SMALL FARSI YEH 08CA..08E1 ; Arabic # Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA @@ -886,7 +889,8 @@ FDFD..FDFF ; Arabic # So [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM. FE70..FE74 ; Arabic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM FE76..FEFC ; Arabic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM 10E60..10E7E ; Arabic # No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS -10EFD..10EFF ; Arabic # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA +10EC2..10EC4 ; Arabic # Lo [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW +10EFC..10EFF ; Arabic # Mn [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA 1EE00..1EE03 ; Arabic # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL 1EE05..1EE1F ; Arabic # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF 1EE21..1EE22 ; Arabic # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM @@ -922,7 +926,7 @@ FE76..FEFC ; Arabic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LA 1EEAB..1EEBB ; Arabic # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN 1EEF0..1EEF1 ; Arabic # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL -# Total code points: 1368 +# Total code points: 1373 # ================================================ @@ -1369,8 +1373,9 @@ AA7B ; Myanmar # Mc MYANMAR SIGN PAO KAREN TONE AA7C ; Myanmar # Mn MYANMAR SIGN TAI LAING TONE-2 AA7D ; Myanmar # Mc MYANMAR SIGN TAI LAING TONE-5 AA7E..AA7F ; Myanmar # Lo [2] MYANMAR LETTER SHWE PALAUNG CHA..MYANMAR LETTER SHWE PALAUNG SHA +116D0..116E3 ; Myanmar # Nd [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE -# Total code points: 223 +# Total code points: 243 # ================================================ @@ -1888,14 +1893,15 @@ A82C ; Syloti_Nagri # Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA 1B42 ; Balinese # Mn BALINESE VOWEL SIGN PEPET 1B43..1B44 ; Balinese # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG 1B45..1B4C ; Balinese # Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA +1B4E..1B4F ; Balinese # Po [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN 1B50..1B59 ; Balinese # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE 1B5A..1B60 ; Balinese # Po [7] BALINESE PANTI..BALINESE PAMENENG 1B61..1B6A ; Balinese # So [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE 1B6B..1B73 ; Balinese # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG 1B74..1B7C ; Balinese # So [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING -1B7D..1B7E ; Balinese # Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1B7D..1B7F ; Balinese # Po [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK -# Total code points: 124 +# Total code points: 127 # ================================================ @@ -2108,8 +2114,9 @@ AADE..AADF ; Tai_Viet # Po [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI 13440 ; Egyptian_Hieroglyphs # Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY 13441..13446 ; Egyptian_Hieroglyphs # Lo [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN 13447..13455 ; Egyptian_Hieroglyphs # Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED +13460..143FA ; Egyptian_Hieroglyphs # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA -# Total code points: 1110 +# Total code points: 5105 # ================================================ @@ -2615,7 +2622,9 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # ================================================ 11700..1171A ; Ahom # Lo [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA -1171D..1171F ; Ahom # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA +1171D ; Ahom # Mn AHOM CONSONANT SIGN MEDIAL LA +1171E ; Ahom # Mc AHOM CONSONANT SIGN MEDIAL RA +1171F ; Ahom # Mn AHOM CONSONANT SIGN MEDIAL LIGATING RA 11720..11721 ; Ahom # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA 11722..11725 ; Ahom # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU 11726 ; Ahom # Mc AHOM VOWEL SIGN E @@ -2949,8 +2958,9 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI 16FE4 ; Khitan_Small_Script # Mn KHITAN SMALL SCRIPT FILLER 18B00..18CD5 ; Khitan_Small_Script # Lo [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18CFF ; Khitan_Small_Script # Lo KHITAN SMALL SCRIPT CHARACTER-18CFF -# Total code points: 471 +# Total code points: 472 # ================================================ @@ -3018,8 +3028,9 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI 11F42 ; Kawi # Mn KAWI CONJOINER 11F43..11F4F ; Kawi # Po [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL 11F50..11F59 ; Kawi # Nd [10] KAWI DIGIT ZERO..KAWI DIGIT NINE +11F5A ; Kawi # Mn KAWI SIGN NUKTA -# Total code points: 86 +# Total code points: 87 # ================================================ @@ -3030,4 +3041,88 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # Total code points: 42 +# ================================================ + +10D40..10D49 ; Garay # Nd [10] GARAY DIGIT ZERO..GARAY DIGIT NINE +10D4A..10D4D ; Garay # Lo [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE +10D4E ; Garay # Lm GARAY VOWEL LENGTH MARK +10D4F ; Garay # Lo GARAY SUKUN +10D50..10D65 ; Garay # L& [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA +10D69..10D6D ; Garay # Mn [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK +10D6E ; Garay # Pd GARAY HYPHEN +10D6F ; Garay # Lm GARAY REDUPLICATION MARK +10D70..10D85 ; Garay # L& [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA +10D8E..10D8F ; Garay # Sm [2] GARAY PLUS SIGN..GARAY MINUS SIGN + +# Total code points: 69 + +# ================================================ + +16100..1611D ; Gurung_Khema # Lo [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA +1611E..16129 ; Gurung_Khema # Mn [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK +1612A..1612C ; Gurung_Khema # Mc [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA +1612D..1612F ; Gurung_Khema # Mn [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA +16130..16139 ; Gurung_Khema # Nd [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE + +# Total code points: 58 + +# ================================================ + +16D40..16D42 ; Kirat_Rai # Lm [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA +16D43..16D6A ; Kirat_Rai # Lo [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU +16D6B..16D6C ; Kirat_Rai # Lm [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT +16D6D..16D6F ; Kirat_Rai # Po [3] KIRAT RAI SIGN YUPI..KIRAT RAI DOUBLE DANDA +16D70..16D79 ; Kirat_Rai # Nd [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE + +# Total code points: 58 + +# ================================================ + +1E5D0..1E5ED ; Ol_Onal # Lo [30] OL ONAL LETTER O..OL ONAL LETTER EG +1E5EE..1E5EF ; Ol_Onal # Mn [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR +1E5F0 ; Ol_Onal # Lo OL ONAL SIGN HODDOND +1E5F1..1E5FA ; Ol_Onal # Nd [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE +1E5FF ; Ol_Onal # Po OL ONAL ABBREVIATION SIGN + +# Total code points: 44 + +# ================================================ + +11BC0..11BE0 ; Sunuwar # Lo [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO +11BE1 ; Sunuwar # Po SUNUWAR SIGN PVO +11BF0..11BF9 ; Sunuwar # Nd [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE + +# Total code points: 44 + +# ================================================ + +105C0..105F3 ; Todhri # Lo [52] TODHRI LETTER A..TODHRI LETTER OO + +# Total code points: 52 + +# ================================================ + +11380..11389 ; Tulu_Tigalari # Lo [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL +1138B ; Tulu_Tigalari # Lo TULU-TIGALARI LETTER EE +1138E ; Tulu_Tigalari # Lo TULU-TIGALARI LETTER AI +11390..113B5 ; Tulu_Tigalari # Lo [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA +113B7 ; Tulu_Tigalari # Lo TULU-TIGALARI SIGN AVAGRAHA +113B8..113BA ; Tulu_Tigalari # Mc [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II +113BB..113C0 ; Tulu_Tigalari # Mn [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL +113C2 ; Tulu_Tigalari # Mc TULU-TIGALARI VOWEL SIGN EE +113C5 ; Tulu_Tigalari # Mc TULU-TIGALARI VOWEL SIGN AI +113C7..113CA ; Tulu_Tigalari # Mc [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA +113CC..113CD ; Tulu_Tigalari # Mc [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA +113CE ; Tulu_Tigalari # Mn TULU-TIGALARI SIGN VIRAMA +113CF ; Tulu_Tigalari # Mc TULU-TIGALARI SIGN LOOPED VIRAMA +113D0 ; Tulu_Tigalari # Mn TULU-TIGALARI CONJOINER +113D1 ; Tulu_Tigalari # Lo TULU-TIGALARI REPHA +113D2 ; Tulu_Tigalari # Mn TULU-TIGALARI GEMINATION MARK +113D3 ; Tulu_Tigalari # Lo TULU-TIGALARI SIGN PLUTA +113D4..113D5 ; Tulu_Tigalari # Po [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA +113D7..113D8 ; Tulu_Tigalari # Po [2] TULU-TIGALARI SIGN OM PUSHPIKA..TULU-TIGALARI SIGN SHRII PUSHPIKA +113E1..113E2 ; Tulu_Tigalari # Mn [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA + +# Total code points: 80 + # EOF diff --git a/share/ucd/UnicodeData.txt b/share/ucd/UnicodeData.txt index bdcc41850..64258a373 100644 --- a/share/ucd/UnicodeData.txt +++ b/share/ucd/UnicodeData.txt @@ -409,7 +409,7 @@ 0198;LATIN CAPITAL LETTER K WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER K HOOK;;;0199; 0199;LATIN SMALL LETTER K WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER K HOOK;;0198;;0198 019A;LATIN SMALL LETTER L WITH BAR;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED L;;023D;;023D -019B;LATIN SMALL LETTER LAMBDA WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED LAMBDA;;;; +019B;LATIN SMALL LETTER LAMBDA WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED LAMBDA;;A7DC;;A7DC 019C;LATIN CAPITAL LETTER TURNED M;Lu;0;L;;;;;N;;;;026F; 019D;LATIN CAPITAL LETTER N WITH LEFT HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER N HOOK;;;0272; 019E;LATIN SMALL LETTER N WITH LONG RIGHT LEG;Ll;0;L;;;;;N;;;0220;;0220 @@ -610,7 +610,7 @@ 0261;LATIN SMALL LETTER SCRIPT G;Ll;0;L;;;;;N;;;A7AC;;A7AC 0262;LATIN LETTER SMALL CAPITAL G;Ll;0;L;;;;;N;;;;; 0263;LATIN SMALL LETTER GAMMA;Ll;0;L;;;;;N;;;0194;;0194 -0264;LATIN SMALL LETTER RAMS HORN;Ll;0;L;;;;;N;LATIN SMALL LETTER BABY GAMMA;;;; +0264;LATIN SMALL LETTER RAMS HORN;Ll;0;L;;;;;N;LATIN SMALL LETTER BABY GAMMA;;A7CB;;A7CB 0265;LATIN SMALL LETTER TURNED H;Ll;0;L;;;;;N;;;A78D;;A78D 0266;LATIN SMALL LETTER H WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER H HOOK;;A7AA;;A7AA 0267;LATIN SMALL LETTER HENG WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER HENG HOOK;;;; @@ -2123,6 +2123,7 @@ 088E;ARABIC VERTICAL TAIL;Lo;0;AL;;;;;N;;;;; 0890;ARABIC POUND MARK ABOVE;Cf;0;AN;;;;;N;;;;; 0891;ARABIC PIASTRE MARK ABOVE;Cf;0;AN;;;;;N;;;;; +0897;ARABIC PEPET;Mn;230;NSM;;;;;N;;;;; 0898;ARABIC SMALL HIGH WORD AL-JUZ;Mn;230;NSM;;;;;N;;;;; 0899;ARABIC SMALL LOW WORD ISHMAAM;Mn;220;NSM;;;;;N;;;;; 089A;ARABIC SMALL LOW WORD IMAALA;Mn;220;NSM;;;;;N;;;;; @@ -6213,6 +6214,8 @@ 1B4A;BALINESE LETTER ZAL SASAK;Lo;0;L;;;;;N;;;;; 1B4B;BALINESE LETTER ASYURA SASAK;Lo;0;L;;;;;N;;;;; 1B4C;BALINESE LETTER ARCHAIC JNYA;Lo;0;L;;;;;N;;;;; +1B4E;BALINESE INVERTED CARIK SIKI;Po;0;L;;;;;N;;;;; +1B4F;BALINESE INVERTED CARIK PAREREN;Po;0;L;;;;;N;;;;; 1B50;BALINESE DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 1B51;BALINESE DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 1B52;BALINESE DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -6260,6 +6263,7 @@ 1B7C;BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING;So;0;L;;;;;N;;;;; 1B7D;BALINESE PANTI LANTANG;Po;0;L;;;;;N;;;;; 1B7E;BALINESE PAMADA LANTANG;Po;0;L;;;;;N;;;;; +1B7F;BALINESE PANTI BAWAK;Po;0;L;;;;;N;;;;; 1B80;SUNDANESE SIGN PANYECEK;Mn;0;NSM;;;;;N;;;;; 1B81;SUNDANESE SIGN PANGLAYAR;Mn;0;NSM;;;;;N;;;;; 1B82;SUNDANESE SIGN PANGWISAD;Mc;0;L;;;;;N;;;;; @@ -6511,6 +6515,8 @@ 1C86;CYRILLIC SMALL LETTER TALL HARD SIGN;Ll;0;L;;;;;N;;;042A;;042A 1C87;CYRILLIC SMALL LETTER TALL YAT;Ll;0;L;;;;;N;;;0462;;0462 1C88;CYRILLIC SMALL LETTER UNBLENDED UK;Ll;0;L;;;;;N;;;A64A;;A64A +1C89;CYRILLIC CAPITAL LETTER TJE;Lu;0;L;;;;;N;;;;1C8A; +1C8A;CYRILLIC SMALL LETTER TJE;Ll;0;L;;;;;N;;;1C89;;1C89 1C90;GEORGIAN MTAVRULI CAPITAL LETTER AN;Lu;0;L;;;;;N;;;;10D0; 1C91;GEORGIAN MTAVRULI CAPITAL LETTER BAN;Lu;0;L;;;;;N;;;;10D1; 1C92;GEORGIAN MTAVRULI CAPITAL LETTER GAN;Lu;0;L;;;;;N;;;;10D2; @@ -7933,7 +7939,7 @@ 226A;MUCH LESS-THAN;Sm;0;ON;;;;;Y;MUCH LESS THAN;;;; 226B;MUCH GREATER-THAN;Sm;0;ON;;;;;Y;MUCH GREATER THAN;;;; 226C;BETWEEN;Sm;0;ON;;;;;N;;;;; -226D;NOT EQUIVALENT TO;Sm;0;ON;224D 0338;;;;N;;;;; +226D;NOT EQUIVALENT TO;Sm;0;ON;224D 0338;;;;Y;;;;; 226E;NOT LESS-THAN;Sm;0;ON;003C 0338;;;;Y;NOT LESS THAN;;;; 226F;NOT GREATER-THAN;Sm;0;ON;003E 0338;;;;Y;NOT GREATER THAN;;;; 2270;NEITHER LESS-THAN NOR EQUAL TO;Sm;0;ON;2264 0338;;;;Y;NEITHER LESS THAN NOR EQUAL TO;;;; @@ -8375,6 +8381,9 @@ 2424;SYMBOL FOR NEWLINE;So;0;ON;;;;;N;GRAPHIC FOR NEWLINE;;;; 2425;SYMBOL FOR DELETE FORM TWO;So;0;ON;;;;;N;;;;; 2426;SYMBOL FOR SUBSTITUTE FORM TWO;So;0;ON;;;;;N;;;;; +2427;SYMBOL FOR DELETE SQUARE CHECKER BOARD FORM;So;0;ON;;;;;N;;;;; +2428;SYMBOL FOR DELETE RECTANGULAR CHECKER BOARD FORM;So;0;ON;;;;;N;;;;; +2429;SYMBOL FOR DELETE MEDIUM SHADE FORM;So;0;ON;;;;;N;;;;; 2440;OCR HOOK;So;0;ON;;;;;N;;;;; 2441;OCR CHAIR;So;0;ON;;;;;N;;;;; 2442;OCR FORK;So;0;ON;;;;;N;;;;; @@ -11709,6 +11718,8 @@ 31E1;CJK STROKE HZZZG;So;0;ON;;;;;N;;;;; 31E2;CJK STROKE PG;So;0;ON;;;;;N;;;;; 31E3;CJK STROKE Q;So;0;ON;;;;;N;;;;; +31E4;CJK STROKE HXG;So;0;ON;;;;;N;;;;; +31E5;CJK STROKE SZP;So;0;ON;;;;;N;;;;; 31EF;IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION;So;0;ON;;;;;N;;;;; 31F0;KATAKANA LETTER SMALL KU;Lo;0;L;;;;;N;;;;; 31F1;KATAKANA LETTER SMALL SI;Lo;0;L;;;;;N;;;;; @@ -14260,6 +14271,9 @@ A7C7;LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY;Lu;0;L;;;;;N;;;;A7C8; A7C8;LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY;Ll;0;L;;;;;N;;;A7C7;;A7C7 A7C9;LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY;Lu;0;L;;;;;N;;;;A7CA; A7CA;LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY;Ll;0;L;;;;;N;;;A7C9;;A7C9 +A7CB;LATIN CAPITAL LETTER RAMS HORN;Lu;0;L;;;;;N;;;;0264; +A7CC;LATIN CAPITAL LETTER S WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A7CD; +A7CD;LATIN SMALL LETTER S WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;A7CC;;A7CC A7D0;LATIN CAPITAL LETTER CLOSED INSULAR G;Lu;0;L;;;;;N;;;;A7D1; A7D1;LATIN SMALL LETTER CLOSED INSULAR G;Ll;0;L;;;;;N;;;A7D0;;A7D0 A7D3;LATIN SMALL LETTER DOUBLE THORN;Ll;0;L;;;;;N;;;;; @@ -14268,6 +14282,9 @@ A7D6;LATIN CAPITAL LETTER MIDDLE SCOTS S;Lu;0;L;;;;;N;;;;A7D7; A7D7;LATIN SMALL LETTER MIDDLE SCOTS S;Ll;0;L;;;;;N;;;A7D6;;A7D6 A7D8;LATIN CAPITAL LETTER SIGMOID S;Lu;0;L;;;;;N;;;;A7D9; A7D9;LATIN SMALL LETTER SIGMOID S;Ll;0;L;;;;;N;;;A7D8;;A7D8 +A7DA;LATIN CAPITAL LETTER LAMBDA;Lu;0;L;;;;;N;;;;A7DB; +A7DB;LATIN SMALL LETTER LAMBDA;Ll;0;L;;;;;N;;;A7DA;;A7DA +A7DC;LATIN CAPITAL LETTER LAMBDA WITH STROKE;Lu;0;L;;;;;N;;;;019B; A7F2;MODIFIER LETTER CAPITAL C;Lm;0;L; 0043;;;;N;;;;; A7F3;MODIFIER LETTER CAPITAL F;Lm;0;L; 0046;;;;N;;;;; A7F4;MODIFIER LETTER CAPITAL Q;Lm;0;L; 0051;;;;N;;;;; @@ -18001,6 +18018,58 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 105B9;VITHKUQI SMALL LETTER XE;Ll;0;L;;;;;N;;;10592;;10592 105BB;VITHKUQI SMALL LETTER Y;Ll;0;L;;;;;N;;;10594;;10594 105BC;VITHKUQI SMALL LETTER ZE;Ll;0;L;;;;;N;;;10595;;10595 +105C0;TODHRI LETTER A;Lo;0;L;;;;;N;;;;; +105C1;TODHRI LETTER AS;Lo;0;L;;;;;N;;;;; +105C2;TODHRI LETTER BA;Lo;0;L;;;;;N;;;;; +105C3;TODHRI LETTER MBA;Lo;0;L;;;;;N;;;;; +105C4;TODHRI LETTER CA;Lo;0;L;;;;;N;;;;; +105C5;TODHRI LETTER CHA;Lo;0;L;;;;;N;;;;; +105C6;TODHRI LETTER DA;Lo;0;L;;;;;N;;;;; +105C7;TODHRI LETTER NDA;Lo;0;L;;;;;N;;;;; +105C8;TODHRI LETTER DHA;Lo;0;L;;;;;N;;;;; +105C9;TODHRI LETTER EI;Lo;0;L;105D2 0307;;;;N;;;;; +105CA;TODHRI LETTER E;Lo;0;L;;;;;N;;;;; +105CB;TODHRI LETTER FA;Lo;0;L;;;;;N;;;;; +105CC;TODHRI LETTER GA;Lo;0;L;;;;;N;;;;; +105CD;TODHRI LETTER NGA;Lo;0;L;;;;;N;;;;; +105CE;TODHRI LETTER GJA;Lo;0;L;;;;;N;;;;; +105CF;TODHRI LETTER NGJA;Lo;0;L;;;;;N;;;;; +105D0;TODHRI LETTER HA;Lo;0;L;;;;;N;;;;; +105D1;TODHRI LETTER HJA;Lo;0;L;;;;;N;;;;; +105D2;TODHRI LETTER I;Lo;0;L;;;;;N;;;;; +105D3;TODHRI LETTER JA;Lo;0;L;;;;;N;;;;; +105D4;TODHRI LETTER KA;Lo;0;L;;;;;N;;;;; +105D5;TODHRI LETTER LA;Lo;0;L;;;;;N;;;;; +105D6;TODHRI LETTER LLA;Lo;0;L;;;;;N;;;;; +105D7;TODHRI LETTER MA;Lo;0;L;;;;;N;;;;; +105D8;TODHRI LETTER NA;Lo;0;L;;;;;N;;;;; +105D9;TODHRI LETTER NJAN;Lo;0;L;;;;;N;;;;; +105DA;TODHRI LETTER O;Lo;0;L;;;;;N;;;;; +105DB;TODHRI LETTER PA;Lo;0;L;;;;;N;;;;; +105DC;TODHRI LETTER QA;Lo;0;L;;;;;N;;;;; +105DD;TODHRI LETTER RA;Lo;0;L;;;;;N;;;;; +105DE;TODHRI LETTER RRA;Lo;0;L;;;;;N;;;;; +105DF;TODHRI LETTER SA;Lo;0;L;;;;;N;;;;; +105E0;TODHRI LETTER SHA;Lo;0;L;;;;;N;;;;; +105E1;TODHRI LETTER SHTA;Lo;0;L;;;;;N;;;;; +105E2;TODHRI LETTER TA;Lo;0;L;;;;;N;;;;; +105E3;TODHRI LETTER THA;Lo;0;L;;;;;N;;;;; +105E4;TODHRI LETTER U;Lo;0;L;105DA 0307;;;;N;;;;; +105E5;TODHRI LETTER VA;Lo;0;L;;;;;N;;;;; +105E6;TODHRI LETTER XA;Lo;0;L;;;;;N;;;;; +105E7;TODHRI LETTER NXA;Lo;0;L;;;;;N;;;;; +105E8;TODHRI LETTER XHA;Lo;0;L;;;;;N;;;;; +105E9;TODHRI LETTER NXHA;Lo;0;L;;;;;N;;;;; +105EA;TODHRI LETTER Y;Lo;0;L;;;;;N;;;;; +105EB;TODHRI LETTER JY;Lo;0;L;;;;;N;;;;; +105EC;TODHRI LETTER ZA;Lo;0;L;;;;;N;;;;; +105ED;TODHRI LETTER ZHA;Lo;0;L;;;;;N;;;;; +105EE;TODHRI LETTER GHA;Lo;0;L;;;;;N;;;;; +105EF;TODHRI LETTER STA;Lo;0;L;;;;;N;;;;; +105F0;TODHRI LETTER SKAN;Lo;0;L;;;;;N;;;;; +105F1;TODHRI LETTER KHA;Lo;0;L;;;;;N;;;;; +105F2;TODHRI LETTER PSA;Lo;0;L;;;;;N;;;;; +105F3;TODHRI LETTER OO;Lo;0;L;;;;;N;;;;; 10600;LINEAR A SIGN AB001;Lo;0;L;;;;;N;;;;; 10601;LINEAR A SIGN AB002;Lo;0;L;;;;;N;;;;; 10602;LINEAR A SIGN AB003;Lo;0;L;;;;;N;;;;; @@ -19322,6 +19391,75 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10D37;HANIFI ROHINGYA DIGIT SEVEN;Nd;0;AN;;7;7;7;N;;;;; 10D38;HANIFI ROHINGYA DIGIT EIGHT;Nd;0;AN;;8;8;8;N;;;;; 10D39;HANIFI ROHINGYA DIGIT NINE;Nd;0;AN;;9;9;9;N;;;;; +10D40;GARAY DIGIT ZERO;Nd;0;AN;;0;0;0;N;;;;; +10D41;GARAY DIGIT ONE;Nd;0;AN;;1;1;1;N;;;;; +10D42;GARAY DIGIT TWO;Nd;0;AN;;2;2;2;N;;;;; +10D43;GARAY DIGIT THREE;Nd;0;AN;;3;3;3;N;;;;; +10D44;GARAY DIGIT FOUR;Nd;0;AN;;4;4;4;N;;;;; +10D45;GARAY DIGIT FIVE;Nd;0;AN;;5;5;5;N;;;;; +10D46;GARAY DIGIT SIX;Nd;0;AN;;6;6;6;N;;;;; +10D47;GARAY DIGIT SEVEN;Nd;0;AN;;7;7;7;N;;;;; +10D48;GARAY DIGIT EIGHT;Nd;0;AN;;8;8;8;N;;;;; +10D49;GARAY DIGIT NINE;Nd;0;AN;;9;9;9;N;;;;; +10D4A;GARAY VOWEL SIGN A;Lo;0;R;;;;;N;;;;; +10D4B;GARAY VOWEL SIGN I;Lo;0;R;;;;;N;;;;; +10D4C;GARAY VOWEL SIGN O;Lo;0;R;;;;;N;;;;; +10D4D;GARAY VOWEL SIGN EE;Lo;0;R;;;;;N;;;;; +10D4E;GARAY VOWEL LENGTH MARK;Lm;0;R;;;;;N;;;;; +10D4F;GARAY SUKUN;Lo;0;R;;;;;N;;;;; +10D50;GARAY CAPITAL LETTER A;Lu;0;R;;;;;N;;;;10D70; +10D51;GARAY CAPITAL LETTER CA;Lu;0;R;;;;;N;;;;10D71; +10D52;GARAY CAPITAL LETTER MA;Lu;0;R;;;;;N;;;;10D72; +10D53;GARAY CAPITAL LETTER KA;Lu;0;R;;;;;N;;;;10D73; +10D54;GARAY CAPITAL LETTER BA;Lu;0;R;;;;;N;;;;10D74; +10D55;GARAY CAPITAL LETTER JA;Lu;0;R;;;;;N;;;;10D75; +10D56;GARAY CAPITAL LETTER SA;Lu;0;R;;;;;N;;;;10D76; +10D57;GARAY CAPITAL LETTER WA;Lu;0;R;;;;;N;;;;10D77; +10D58;GARAY CAPITAL LETTER LA;Lu;0;R;;;;;N;;;;10D78; +10D59;GARAY CAPITAL LETTER GA;Lu;0;R;;;;;N;;;;10D79; +10D5A;GARAY CAPITAL LETTER DA;Lu;0;R;;;;;N;;;;10D7A; +10D5B;GARAY CAPITAL LETTER XA;Lu;0;R;;;;;N;;;;10D7B; +10D5C;GARAY CAPITAL LETTER YA;Lu;0;R;;;;;N;;;;10D7C; +10D5D;GARAY CAPITAL LETTER TA;Lu;0;R;;;;;N;;;;10D7D; +10D5E;GARAY CAPITAL LETTER RA;Lu;0;R;;;;;N;;;;10D7E; +10D5F;GARAY CAPITAL LETTER NYA;Lu;0;R;;;;;N;;;;10D7F; +10D60;GARAY CAPITAL LETTER FA;Lu;0;R;;;;;N;;;;10D80; +10D61;GARAY CAPITAL LETTER NA;Lu;0;R;;;;;N;;;;10D81; +10D62;GARAY CAPITAL LETTER PA;Lu;0;R;;;;;N;;;;10D82; +10D63;GARAY CAPITAL LETTER HA;Lu;0;R;;;;;N;;;;10D83; +10D64;GARAY CAPITAL LETTER OLD KA;Lu;0;R;;;;;N;;;;10D84; +10D65;GARAY CAPITAL LETTER OLD NA;Lu;0;R;;;;;N;;;;10D85; +10D69;GARAY VOWEL SIGN E;Mn;230;NSM;;;;;N;;;;; +10D6A;GARAY CONSONANT GEMINATION MARK;Mn;230;NSM;;;;;N;;;;; +10D6B;GARAY COMBINING DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +10D6C;GARAY COMBINING DOUBLE DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +10D6D;GARAY CONSONANT NASALIZATION MARK;Mn;230;NSM;;;;;N;;;;; +10D6E;GARAY HYPHEN;Pd;0;ON;;;;;N;;;;; +10D6F;GARAY REDUPLICATION MARK;Lm;0;R;;;;;N;;;;; +10D70;GARAY SMALL LETTER A;Ll;0;R;;;;;N;;;10D50;;10D50 +10D71;GARAY SMALL LETTER CA;Ll;0;R;;;;;N;;;10D51;;10D51 +10D72;GARAY SMALL LETTER MA;Ll;0;R;;;;;N;;;10D52;;10D52 +10D73;GARAY SMALL LETTER KA;Ll;0;R;;;;;N;;;10D53;;10D53 +10D74;GARAY SMALL LETTER BA;Ll;0;R;;;;;N;;;10D54;;10D54 +10D75;GARAY SMALL LETTER JA;Ll;0;R;;;;;N;;;10D55;;10D55 +10D76;GARAY SMALL LETTER SA;Ll;0;R;;;;;N;;;10D56;;10D56 +10D77;GARAY SMALL LETTER WA;Ll;0;R;;;;;N;;;10D57;;10D57 +10D78;GARAY SMALL LETTER LA;Ll;0;R;;;;;N;;;10D58;;10D58 +10D79;GARAY SMALL LETTER GA;Ll;0;R;;;;;N;;;10D59;;10D59 +10D7A;GARAY SMALL LETTER DA;Ll;0;R;;;;;N;;;10D5A;;10D5A +10D7B;GARAY SMALL LETTER XA;Ll;0;R;;;;;N;;;10D5B;;10D5B +10D7C;GARAY SMALL LETTER YA;Ll;0;R;;;;;N;;;10D5C;;10D5C +10D7D;GARAY SMALL LETTER TA;Ll;0;R;;;;;N;;;10D5D;;10D5D +10D7E;GARAY SMALL LETTER RA;Ll;0;R;;;;;N;;;10D5E;;10D5E +10D7F;GARAY SMALL LETTER NYA;Ll;0;R;;;;;N;;;10D5F;;10D5F +10D80;GARAY SMALL LETTER FA;Ll;0;R;;;;;N;;;10D60;;10D60 +10D81;GARAY SMALL LETTER NA;Ll;0;R;;;;;N;;;10D61;;10D61 +10D82;GARAY SMALL LETTER PA;Ll;0;R;;;;;N;;;10D62;;10D62 +10D83;GARAY SMALL LETTER HA;Ll;0;R;;;;;N;;;10D63;;10D63 +10D84;GARAY SMALL LETTER OLD KA;Ll;0;R;;;;;N;;;10D64;;10D64 +10D85;GARAY SMALL LETTER OLD NA;Ll;0;R;;;;;N;;;10D65;;10D65 +10D8E;GARAY PLUS SIGN;Sm;0;R;;;;;N;;;;; +10D8F;GARAY MINUS SIGN;Sm;0;R;;;;;N;;;;; 10E60;RUMI DIGIT ONE;No;0;AN;;;1;1;N;;;;; 10E61;RUMI DIGIT TWO;No;0;AN;;;2;2;N;;;;; 10E62;RUMI DIGIT THREE;No;0;AN;;;3;3;N;;;;; @@ -19400,6 +19538,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EC2;ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC3;ARABIC LETTER TAH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC4;ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EFC;ARABIC COMBINING ALEF OVERLAY;Mn;0;NSM;;;;;N;;;;; 10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; 10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; 10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; @@ -20264,6 +20406,86 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11372;COMBINING GRANTHA LETTER NA;Mn;230;NSM;;;;;N;;;;; 11373;COMBINING GRANTHA LETTER VI;Mn;230;NSM;;;;;N;;;;; 11374;COMBINING GRANTHA LETTER PA;Mn;230;NSM;;;;;N;;;;; +11380;TULU-TIGALARI LETTER A;Lo;0;L;;;;;N;;;;; +11381;TULU-TIGALARI LETTER AA;Lo;0;L;;;;;N;;;;; +11382;TULU-TIGALARI LETTER I;Lo;0;L;;;;;N;;;;; +11383;TULU-TIGALARI LETTER II;Lo;0;L;11382 113C9;;;;N;;;;; +11384;TULU-TIGALARI LETTER U;Lo;0;L;;;;;N;;;;; +11385;TULU-TIGALARI LETTER UU;Lo;0;L;11384 113BB;;;;N;;;;; +11386;TULU-TIGALARI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11387;TULU-TIGALARI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11388;TULU-TIGALARI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11389;TULU-TIGALARI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +1138B;TULU-TIGALARI LETTER EE;Lo;0;L;;;;;N;;;;; +1138E;TULU-TIGALARI LETTER AI;Lo;0;L;1138B 113C2;;;;N;;;;; +11390;TULU-TIGALARI LETTER OO;Lo;0;L;;;;;N;;;;; +11391;TULU-TIGALARI LETTER AU;Lo;0;L;11390 113C9;;;;N;;;;; +11392;TULU-TIGALARI LETTER KA;Lo;0;L;;;;;N;;;;; +11393;TULU-TIGALARI LETTER KHA;Lo;0;L;;;;;N;;;;; +11394;TULU-TIGALARI LETTER GA;Lo;0;L;;;;;N;;;;; +11395;TULU-TIGALARI LETTER GHA;Lo;0;L;;;;;N;;;;; +11396;TULU-TIGALARI LETTER NGA;Lo;0;L;;;;;N;;;;; +11397;TULU-TIGALARI LETTER CA;Lo;0;L;;;;;N;;;;; +11398;TULU-TIGALARI LETTER CHA;Lo;0;L;;;;;N;;;;; +11399;TULU-TIGALARI LETTER JA;Lo;0;L;;;;;N;;;;; +1139A;TULU-TIGALARI LETTER JHA;Lo;0;L;;;;;N;;;;; +1139B;TULU-TIGALARI LETTER NYA;Lo;0;L;;;;;N;;;;; +1139C;TULU-TIGALARI LETTER TTA;Lo;0;L;;;;;N;;;;; +1139D;TULU-TIGALARI LETTER TTHA;Lo;0;L;;;;;N;;;;; +1139E;TULU-TIGALARI LETTER DDA;Lo;0;L;;;;;N;;;;; +1139F;TULU-TIGALARI LETTER DDHA;Lo;0;L;;;;;N;;;;; +113A0;TULU-TIGALARI LETTER NNA;Lo;0;L;;;;;N;;;;; +113A1;TULU-TIGALARI LETTER TA;Lo;0;L;;;;;N;;;;; +113A2;TULU-TIGALARI LETTER THA;Lo;0;L;;;;;N;;;;; +113A3;TULU-TIGALARI LETTER DA;Lo;0;L;;;;;N;;;;; +113A4;TULU-TIGALARI LETTER DHA;Lo;0;L;;;;;N;;;;; +113A5;TULU-TIGALARI LETTER NA;Lo;0;L;;;;;N;;;;; +113A6;TULU-TIGALARI LETTER PA;Lo;0;L;;;;;N;;;;; +113A7;TULU-TIGALARI LETTER PHA;Lo;0;L;;;;;N;;;;; +113A8;TULU-TIGALARI LETTER BA;Lo;0;L;;;;;N;;;;; +113A9;TULU-TIGALARI LETTER BHA;Lo;0;L;;;;;N;;;;; +113AA;TULU-TIGALARI LETTER MA;Lo;0;L;;;;;N;;;;; +113AB;TULU-TIGALARI LETTER YA;Lo;0;L;;;;;N;;;;; +113AC;TULU-TIGALARI LETTER RA;Lo;0;L;;;;;N;;;;; +113AD;TULU-TIGALARI LETTER LA;Lo;0;L;;;;;N;;;;; +113AE;TULU-TIGALARI LETTER VA;Lo;0;L;;;;;N;;;;; +113AF;TULU-TIGALARI LETTER SHA;Lo;0;L;;;;;N;;;;; +113B0;TULU-TIGALARI LETTER SSA;Lo;0;L;;;;;N;;;;; +113B1;TULU-TIGALARI LETTER SA;Lo;0;L;;;;;N;;;;; +113B2;TULU-TIGALARI LETTER HA;Lo;0;L;;;;;N;;;;; +113B3;TULU-TIGALARI LETTER LLA;Lo;0;L;;;;;N;;;;; +113B4;TULU-TIGALARI LETTER RRA;Lo;0;L;;;;;N;;;;; +113B5;TULU-TIGALARI LETTER LLLA;Lo;0;L;;;;;N;;;;; +113B7;TULU-TIGALARI SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +113B8;TULU-TIGALARI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +113B9;TULU-TIGALARI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +113BA;TULU-TIGALARI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +113BB;TULU-TIGALARI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +113BC;TULU-TIGALARI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +113BD;TULU-TIGALARI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +113BE;TULU-TIGALARI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +113BF;TULU-TIGALARI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +113C0;TULU-TIGALARI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +113C2;TULU-TIGALARI VOWEL SIGN EE;Mc;0;L;;;;;N;;;;; +113C5;TULU-TIGALARI VOWEL SIGN AI;Mc;0;L;113C2 113C2;;;;N;;;;; +113C7;TULU-TIGALARI VOWEL SIGN OO;Mc;0;L;113C2 113B8;;;;N;;;;; +113C8;TULU-TIGALARI VOWEL SIGN AU;Mc;0;L;113C2 113C9;;;;N;;;;; +113C9;TULU-TIGALARI AU LENGTH MARK;Mc;0;L;;;;;N;;;;; +113CA;TULU-TIGALARI SIGN CANDRA ANUNASIKA;Mc;0;L;;;;;N;;;;; +113CC;TULU-TIGALARI SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +113CD;TULU-TIGALARI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +113CE;TULU-TIGALARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +113CF;TULU-TIGALARI SIGN LOOPED VIRAMA;Mc;9;L;;;;;N;;;;; +113D0;TULU-TIGALARI CONJOINER;Mn;9;NSM;;;;;N;;;;; +113D1;TULU-TIGALARI REPHA;Lo;0;L;;;;;N;;;;; +113D2;TULU-TIGALARI GEMINATION MARK;Mn;0;NSM;;;;;N;;;;; +113D3;TULU-TIGALARI SIGN PLUTA;Lo;0;L;;;;;N;;;;; +113D4;TULU-TIGALARI DANDA;Po;0;L;;;;;N;;;;; +113D5;TULU-TIGALARI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +113D7;TULU-TIGALARI SIGN OM PUSHPIKA;Po;0;L;;;;;N;;;;; +113D8;TULU-TIGALARI SIGN SHRII PUSHPIKA;Po;0;L;;;;;N;;;;; +113E1;TULU-TIGALARI VEDIC TONE SVARITA;Mn;0;NSM;;;;;N;;;;; +113E2;TULU-TIGALARI VEDIC TONE ANUDATTA;Mn;0;NSM;;;;;N;;;;; 11400;NEWA LETTER A;Lo;0;L;;;;;N;;;;; 11401;NEWA LETTER AA;Lo;0;L;;;;;N;;;;; 11402;NEWA LETTER I;Lo;0;L;;;;;N;;;;; @@ -20695,6 +20917,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 116C7;TAKRI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 116C8;TAKRI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 116C9;TAKRI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +116D0;MYANMAR PAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +116D1;MYANMAR PAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +116D2;MYANMAR PAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +116D3;MYANMAR PAO DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +116D4;MYANMAR PAO DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +116D5;MYANMAR PAO DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +116D6;MYANMAR PAO DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +116D7;MYANMAR PAO DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +116D8;MYANMAR PAO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +116D9;MYANMAR PAO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +116DA;MYANMAR EASTERN PWO KAREN DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +116DB;MYANMAR EASTERN PWO KAREN DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +116DC;MYANMAR EASTERN PWO KAREN DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +116DD;MYANMAR EASTERN PWO KAREN DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +116DE;MYANMAR EASTERN PWO KAREN DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +116DF;MYANMAR EASTERN PWO KAREN DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +116E0;MYANMAR EASTERN PWO KAREN DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +116E1;MYANMAR EASTERN PWO KAREN DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +116E2;MYANMAR EASTERN PWO KAREN DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +116E3;MYANMAR EASTERN PWO KAREN DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11700;AHOM LETTER KA;Lo;0;L;;;;;N;;;;; 11701;AHOM LETTER KHA;Lo;0;L;;;;;N;;;;; 11702;AHOM LETTER NGA;Lo;0;L;;;;;N;;;;; @@ -20723,7 +20965,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11719;AHOM LETTER JHA;Lo;0;L;;;;;N;;;;; 1171A;AHOM LETTER ALTERNATE BA;Lo;0;L;;;;;N;;;;; 1171D;AHOM CONSONANT SIGN MEDIAL LA;Mn;0;NSM;;;;;N;;;;; -1171E;AHOM CONSONANT SIGN MEDIAL RA;Mn;0;NSM;;;;;N;;;;; +1171E;AHOM CONSONANT SIGN MEDIAL RA;Mc;0;L;;;;;N;;;;; 1171F;AHOM CONSONANT SIGN MEDIAL LIGATING RA;Mn;0;NSM;;;;;N;;;;; 11720;AHOM VOWEL SIGN A;Mc;0;L;;;;;N;;;;; 11721;AHOM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; @@ -21279,6 +21521,50 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; 11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; 11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; +11BC0;SUNUWAR LETTER DEVI;Lo;0;L;;;;;N;;;;; +11BC1;SUNUWAR LETTER TASLA;Lo;0;L;;;;;N;;;;; +11BC2;SUNUWAR LETTER EKO;Lo;0;L;;;;;N;;;;; +11BC3;SUNUWAR LETTER IMAR;Lo;0;L;;;;;N;;;;; +11BC4;SUNUWAR LETTER REU;Lo;0;L;;;;;N;;;;; +11BC5;SUNUWAR LETTER UTTHI;Lo;0;L;;;;;N;;;;; +11BC6;SUNUWAR LETTER KIK;Lo;0;L;;;;;N;;;;; +11BC7;SUNUWAR LETTER MA;Lo;0;L;;;;;N;;;;; +11BC8;SUNUWAR LETTER APPHO;Lo;0;L;;;;;N;;;;; +11BC9;SUNUWAR LETTER PIP;Lo;0;L;;;;;N;;;;; +11BCA;SUNUWAR LETTER GIL;Lo;0;L;;;;;N;;;;; +11BCB;SUNUWAR LETTER HAMSO;Lo;0;L;;;;;N;;;;; +11BCC;SUNUWAR LETTER CARMI;Lo;0;L;;;;;N;;;;; +11BCD;SUNUWAR LETTER NAH;Lo;0;L;;;;;N;;;;; +11BCE;SUNUWAR LETTER BUR;Lo;0;L;;;;;N;;;;; +11BCF;SUNUWAR LETTER JYAH;Lo;0;L;;;;;N;;;;; +11BD0;SUNUWAR LETTER LOACHA;Lo;0;L;;;;;N;;;;; +11BD1;SUNUWAR LETTER OTTHI;Lo;0;L;;;;;N;;;;; +11BD2;SUNUWAR LETTER SHYELE;Lo;0;L;;;;;N;;;;; +11BD3;SUNUWAR LETTER VARCA;Lo;0;L;;;;;N;;;;; +11BD4;SUNUWAR LETTER YAT;Lo;0;L;;;;;N;;;;; +11BD5;SUNUWAR LETTER AVA;Lo;0;L;;;;;N;;;;; +11BD6;SUNUWAR LETTER AAL;Lo;0;L;;;;;N;;;;; +11BD7;SUNUWAR LETTER DONGA;Lo;0;L;;;;;N;;;;; +11BD8;SUNUWAR LETTER THARI;Lo;0;L;;;;;N;;;;; +11BD9;SUNUWAR LETTER PHAR;Lo;0;L;;;;;N;;;;; +11BDA;SUNUWAR LETTER NGAR;Lo;0;L;;;;;N;;;;; +11BDB;SUNUWAR LETTER KHA;Lo;0;L;;;;;N;;;;; +11BDC;SUNUWAR LETTER SHYER;Lo;0;L;;;;;N;;;;; +11BDD;SUNUWAR LETTER CHELAP;Lo;0;L;;;;;N;;;;; +11BDE;SUNUWAR LETTER TENTU;Lo;0;L;;;;;N;;;;; +11BDF;SUNUWAR LETTER THELE;Lo;0;L;;;;;N;;;;; +11BE0;SUNUWAR LETTER KLOKO;Lo;0;L;;;;;N;;;;; +11BE1;SUNUWAR SIGN PVO;Po;0;L;;;;;N;;;;; +11BF0;SUNUWAR DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11BF1;SUNUWAR DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11BF2;SUNUWAR DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11BF3;SUNUWAR DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11BF4;SUNUWAR DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11BF5;SUNUWAR DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11BF6;SUNUWAR DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11BF7;SUNUWAR DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11BF8;SUNUWAR DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11BF9;SUNUWAR DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21693,6 +21979,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11F5A;KAWI SIGN NUKTA;Mn;0;NSM;;;;;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -24188,6 +24475,4001 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; 13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; 13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; +13460;EGYPTIAN HIEROGLYPH-13460;Lo;0;L;;;;;N;;;;; +13461;EGYPTIAN HIEROGLYPH-13461;Lo;0;L;;;;;N;;;;; +13462;EGYPTIAN HIEROGLYPH-13462;Lo;0;L;;;;;N;;;;; +13463;EGYPTIAN HIEROGLYPH-13463;Lo;0;L;;;;;N;;;;; +13464;EGYPTIAN HIEROGLYPH-13464;Lo;0;L;;;;;N;;;;; +13465;EGYPTIAN HIEROGLYPH-13465;Lo;0;L;;;;;N;;;;; +13466;EGYPTIAN HIEROGLYPH-13466;Lo;0;L;;;;;N;;;;; +13467;EGYPTIAN HIEROGLYPH-13467;Lo;0;L;;;;;N;;;;; +13468;EGYPTIAN HIEROGLYPH-13468;Lo;0;L;;;;;N;;;;; +13469;EGYPTIAN HIEROGLYPH-13469;Lo;0;L;;;;;N;;;;; +1346A;EGYPTIAN HIEROGLYPH-1346A;Lo;0;L;;;;;N;;;;; +1346B;EGYPTIAN HIEROGLYPH-1346B;Lo;0;L;;;;;N;;;;; +1346C;EGYPTIAN HIEROGLYPH-1346C;Lo;0;L;;;;;N;;;;; +1346D;EGYPTIAN HIEROGLYPH-1346D;Lo;0;L;;;;;N;;;;; +1346E;EGYPTIAN HIEROGLYPH-1346E;Lo;0;L;;;;;N;;;;; +1346F;EGYPTIAN HIEROGLYPH-1346F;Lo;0;L;;;;;N;;;;; +13470;EGYPTIAN HIEROGLYPH-13470;Lo;0;L;;;;;N;;;;; +13471;EGYPTIAN HIEROGLYPH-13471;Lo;0;L;;;;;N;;;;; +13472;EGYPTIAN HIEROGLYPH-13472;Lo;0;L;;;;;N;;;;; +13473;EGYPTIAN HIEROGLYPH-13473;Lo;0;L;;;;;N;;;;; +13474;EGYPTIAN HIEROGLYPH-13474;Lo;0;L;;;;;N;;;;; +13475;EGYPTIAN HIEROGLYPH-13475;Lo;0;L;;;;;N;;;;; +13476;EGYPTIAN HIEROGLYPH-13476;Lo;0;L;;;;;N;;;;; +13477;EGYPTIAN HIEROGLYPH-13477;Lo;0;L;;;;;N;;;;; +13478;EGYPTIAN HIEROGLYPH-13478;Lo;0;L;;;;;N;;;;; +13479;EGYPTIAN HIEROGLYPH-13479;Lo;0;L;;;;;N;;;;; +1347A;EGYPTIAN HIEROGLYPH-1347A;Lo;0;L;;;;;N;;;;; +1347B;EGYPTIAN HIEROGLYPH-1347B;Lo;0;L;;;;;N;;;;; +1347C;EGYPTIAN HIEROGLYPH-1347C;Lo;0;L;;;;;N;;;;; +1347D;EGYPTIAN HIEROGLYPH-1347D;Lo;0;L;;;;;N;;;;; +1347E;EGYPTIAN HIEROGLYPH-1347E;Lo;0;L;;;;;N;;;;; +1347F;EGYPTIAN HIEROGLYPH-1347F;Lo;0;L;;;;;N;;;;; +13480;EGYPTIAN HIEROGLYPH-13480;Lo;0;L;;;;;N;;;;; +13481;EGYPTIAN HIEROGLYPH-13481;Lo;0;L;;;;;N;;;;; +13482;EGYPTIAN HIEROGLYPH-13482;Lo;0;L;;;;;N;;;;; +13483;EGYPTIAN HIEROGLYPH-13483;Lo;0;L;;;;;N;;;;; +13484;EGYPTIAN HIEROGLYPH-13484;Lo;0;L;;;;;N;;;;; +13485;EGYPTIAN HIEROGLYPH-13485;Lo;0;L;;;;;N;;;;; +13486;EGYPTIAN HIEROGLYPH-13486;Lo;0;L;;;;;N;;;;; +13487;EGYPTIAN HIEROGLYPH-13487;Lo;0;L;;;;;N;;;;; +13488;EGYPTIAN HIEROGLYPH-13488;Lo;0;L;;;;;N;;;;; +13489;EGYPTIAN HIEROGLYPH-13489;Lo;0;L;;;;;N;;;;; +1348A;EGYPTIAN HIEROGLYPH-1348A;Lo;0;L;;;;;N;;;;; +1348B;EGYPTIAN HIEROGLYPH-1348B;Lo;0;L;;;;;N;;;;; +1348C;EGYPTIAN HIEROGLYPH-1348C;Lo;0;L;;;;;N;;;;; +1348D;EGYPTIAN HIEROGLYPH-1348D;Lo;0;L;;;;;N;;;;; +1348E;EGYPTIAN HIEROGLYPH-1348E;Lo;0;L;;;;;N;;;;; +1348F;EGYPTIAN HIEROGLYPH-1348F;Lo;0;L;;;;;N;;;;; +13490;EGYPTIAN HIEROGLYPH-13490;Lo;0;L;;;;;N;;;;; +13491;EGYPTIAN HIEROGLYPH-13491;Lo;0;L;;;;;N;;;;; +13492;EGYPTIAN HIEROGLYPH-13492;Lo;0;L;;;;;N;;;;; +13493;EGYPTIAN HIEROGLYPH-13493;Lo;0;L;;;;;N;;;;; +13494;EGYPTIAN HIEROGLYPH-13494;Lo;0;L;;;;;N;;;;; +13495;EGYPTIAN HIEROGLYPH-13495;Lo;0;L;;;;;N;;;;; +13496;EGYPTIAN HIEROGLYPH-13496;Lo;0;L;;;;;N;;;;; +13497;EGYPTIAN HIEROGLYPH-13497;Lo;0;L;;;;;N;;;;; +13498;EGYPTIAN HIEROGLYPH-13498;Lo;0;L;;;;;N;;;;; +13499;EGYPTIAN HIEROGLYPH-13499;Lo;0;L;;;;;N;;;;; +1349A;EGYPTIAN HIEROGLYPH-1349A;Lo;0;L;;;;;N;;;;; +1349B;EGYPTIAN HIEROGLYPH-1349B;Lo;0;L;;;;;N;;;;; +1349C;EGYPTIAN HIEROGLYPH-1349C;Lo;0;L;;;;;N;;;;; +1349D;EGYPTIAN HIEROGLYPH-1349D;Lo;0;L;;;;;N;;;;; +1349E;EGYPTIAN HIEROGLYPH-1349E;Lo;0;L;;;;;N;;;;; +1349F;EGYPTIAN HIEROGLYPH-1349F;Lo;0;L;;;;;N;;;;; +134A0;EGYPTIAN HIEROGLYPH-134A0;Lo;0;L;;;;;N;;;;; +134A1;EGYPTIAN HIEROGLYPH-134A1;Lo;0;L;;;;;N;;;;; +134A2;EGYPTIAN HIEROGLYPH-134A2;Lo;0;L;;;;;N;;;;; +134A3;EGYPTIAN HIEROGLYPH-134A3;Lo;0;L;;;;;N;;;;; +134A4;EGYPTIAN HIEROGLYPH-134A4;Lo;0;L;;;;;N;;;;; +134A5;EGYPTIAN HIEROGLYPH-134A5;Lo;0;L;;;;;N;;;;; +134A6;EGYPTIAN HIEROGLYPH-134A6;Lo;0;L;;;;;N;;;;; +134A7;EGYPTIAN HIEROGLYPH-134A7;Lo;0;L;;;;;N;;;;; +134A8;EGYPTIAN HIEROGLYPH-134A8;Lo;0;L;;;;;N;;;;; +134A9;EGYPTIAN HIEROGLYPH-134A9;Lo;0;L;;;;;N;;;;; +134AA;EGYPTIAN HIEROGLYPH-134AA;Lo;0;L;;;;;N;;;;; +134AB;EGYPTIAN HIEROGLYPH-134AB;Lo;0;L;;;;;N;;;;; +134AC;EGYPTIAN HIEROGLYPH-134AC;Lo;0;L;;;;;N;;;;; +134AD;EGYPTIAN HIEROGLYPH-134AD;Lo;0;L;;;;;N;;;;; +134AE;EGYPTIAN HIEROGLYPH-134AE;Lo;0;L;;;;;N;;;;; +134AF;EGYPTIAN HIEROGLYPH-134AF;Lo;0;L;;;;;N;;;;; +134B0;EGYPTIAN HIEROGLYPH-134B0;Lo;0;L;;;;;N;;;;; +134B1;EGYPTIAN HIEROGLYPH-134B1;Lo;0;L;;;;;N;;;;; +134B2;EGYPTIAN HIEROGLYPH-134B2;Lo;0;L;;;;;N;;;;; +134B3;EGYPTIAN HIEROGLYPH-134B3;Lo;0;L;;;;;N;;;;; +134B4;EGYPTIAN HIEROGLYPH-134B4;Lo;0;L;;;;;N;;;;; +134B5;EGYPTIAN HIEROGLYPH-134B5;Lo;0;L;;;;;N;;;;; +134B6;EGYPTIAN HIEROGLYPH-134B6;Lo;0;L;;;;;N;;;;; +134B7;EGYPTIAN HIEROGLYPH-134B7;Lo;0;L;;;;;N;;;;; +134B8;EGYPTIAN HIEROGLYPH-134B8;Lo;0;L;;;;;N;;;;; +134B9;EGYPTIAN HIEROGLYPH-134B9;Lo;0;L;;;;;N;;;;; +134BA;EGYPTIAN HIEROGLYPH-134BA;Lo;0;L;;;;;N;;;;; +134BB;EGYPTIAN HIEROGLYPH-134BB;Lo;0;L;;;;;N;;;;; +134BC;EGYPTIAN HIEROGLYPH-134BC;Lo;0;L;;;;;N;;;;; +134BD;EGYPTIAN HIEROGLYPH-134BD;Lo;0;L;;;;;N;;;;; +134BE;EGYPTIAN HIEROGLYPH-134BE;Lo;0;L;;;;;N;;;;; +134BF;EGYPTIAN HIEROGLYPH-134BF;Lo;0;L;;;;;N;;;;; +134C0;EGYPTIAN HIEROGLYPH-134C0;Lo;0;L;;;;;N;;;;; +134C1;EGYPTIAN HIEROGLYPH-134C1;Lo;0;L;;;;;N;;;;; +134C2;EGYPTIAN HIEROGLYPH-134C2;Lo;0;L;;;;;N;;;;; +134C3;EGYPTIAN HIEROGLYPH-134C3;Lo;0;L;;;;;N;;;;; +134C4;EGYPTIAN HIEROGLYPH-134C4;Lo;0;L;;;;;N;;;;; +134C5;EGYPTIAN HIEROGLYPH-134C5;Lo;0;L;;;;;N;;;;; +134C6;EGYPTIAN HIEROGLYPH-134C6;Lo;0;L;;;;;N;;;;; +134C7;EGYPTIAN HIEROGLYPH-134C7;Lo;0;L;;;;;N;;;;; +134C8;EGYPTIAN HIEROGLYPH-134C8;Lo;0;L;;;;;N;;;;; +134C9;EGYPTIAN HIEROGLYPH-134C9;Lo;0;L;;;;;N;;;;; +134CA;EGYPTIAN HIEROGLYPH-134CA;Lo;0;L;;;;;N;;;;; +134CB;EGYPTIAN HIEROGLYPH-134CB;Lo;0;L;;;;;N;;;;; +134CC;EGYPTIAN HIEROGLYPH-134CC;Lo;0;L;;;;;N;;;;; +134CD;EGYPTIAN HIEROGLYPH-134CD;Lo;0;L;;;;;N;;;;; +134CE;EGYPTIAN HIEROGLYPH-134CE;Lo;0;L;;;;;N;;;;; +134CF;EGYPTIAN HIEROGLYPH-134CF;Lo;0;L;;;;;N;;;;; +134D0;EGYPTIAN HIEROGLYPH-134D0;Lo;0;L;;;;;N;;;;; +134D1;EGYPTIAN HIEROGLYPH-134D1;Lo;0;L;;;;;N;;;;; +134D2;EGYPTIAN HIEROGLYPH-134D2;Lo;0;L;;;;;N;;;;; +134D3;EGYPTIAN HIEROGLYPH-134D3;Lo;0;L;;;;;N;;;;; +134D4;EGYPTIAN HIEROGLYPH-134D4;Lo;0;L;;;;;N;;;;; +134D5;EGYPTIAN HIEROGLYPH-134D5;Lo;0;L;;;;;N;;;;; +134D6;EGYPTIAN HIEROGLYPH-134D6;Lo;0;L;;;;;N;;;;; +134D7;EGYPTIAN HIEROGLYPH-134D7;Lo;0;L;;;;;N;;;;; +134D8;EGYPTIAN HIEROGLYPH-134D8;Lo;0;L;;;;;N;;;;; +134D9;EGYPTIAN HIEROGLYPH-134D9;Lo;0;L;;;;;N;;;;; +134DA;EGYPTIAN HIEROGLYPH-134DA;Lo;0;L;;;;;N;;;;; +134DB;EGYPTIAN HIEROGLYPH-134DB;Lo;0;L;;;;;N;;;;; +134DC;EGYPTIAN HIEROGLYPH-134DC;Lo;0;L;;;;;N;;;;; +134DD;EGYPTIAN HIEROGLYPH-134DD;Lo;0;L;;;;;N;;;;; +134DE;EGYPTIAN HIEROGLYPH-134DE;Lo;0;L;;;;;N;;;;; +134DF;EGYPTIAN HIEROGLYPH-134DF;Lo;0;L;;;;;N;;;;; +134E0;EGYPTIAN HIEROGLYPH-134E0;Lo;0;L;;;;;N;;;;; +134E1;EGYPTIAN HIEROGLYPH-134E1;Lo;0;L;;;;;N;;;;; +134E2;EGYPTIAN HIEROGLYPH-134E2;Lo;0;L;;;;;N;;;;; +134E3;EGYPTIAN HIEROGLYPH-134E3;Lo;0;L;;;;;N;;;;; +134E4;EGYPTIAN HIEROGLYPH-134E4;Lo;0;L;;;;;N;;;;; +134E5;EGYPTIAN HIEROGLYPH-134E5;Lo;0;L;;;;;N;;;;; +134E6;EGYPTIAN HIEROGLYPH-134E6;Lo;0;L;;;;;N;;;;; +134E7;EGYPTIAN HIEROGLYPH-134E7;Lo;0;L;;;;;N;;;;; +134E8;EGYPTIAN HIEROGLYPH-134E8;Lo;0;L;;;;;N;;;;; +134E9;EGYPTIAN HIEROGLYPH-134E9;Lo;0;L;;;;;N;;;;; +134EA;EGYPTIAN HIEROGLYPH-134EA;Lo;0;L;;;;;N;;;;; +134EB;EGYPTIAN HIEROGLYPH-134EB;Lo;0;L;;;;;N;;;;; +134EC;EGYPTIAN HIEROGLYPH-134EC;Lo;0;L;;;;;N;;;;; +134ED;EGYPTIAN HIEROGLYPH-134ED;Lo;0;L;;;;;N;;;;; +134EE;EGYPTIAN HIEROGLYPH-134EE;Lo;0;L;;;;;N;;;;; +134EF;EGYPTIAN HIEROGLYPH-134EF;Lo;0;L;;;;;N;;;;; +134F0;EGYPTIAN HIEROGLYPH-134F0;Lo;0;L;;;;;N;;;;; +134F1;EGYPTIAN HIEROGLYPH-134F1;Lo;0;L;;;;;N;;;;; +134F2;EGYPTIAN HIEROGLYPH-134F2;Lo;0;L;;;;;N;;;;; +134F3;EGYPTIAN HIEROGLYPH-134F3;Lo;0;L;;;;;N;;;;; +134F4;EGYPTIAN HIEROGLYPH-134F4;Lo;0;L;;;;;N;;;;; +134F5;EGYPTIAN HIEROGLYPH-134F5;Lo;0;L;;;;;N;;;;; +134F6;EGYPTIAN HIEROGLYPH-134F6;Lo;0;L;;;;;N;;;;; +134F7;EGYPTIAN HIEROGLYPH-134F7;Lo;0;L;;;;;N;;;;; +134F8;EGYPTIAN HIEROGLYPH-134F8;Lo;0;L;;;;;N;;;;; +134F9;EGYPTIAN HIEROGLYPH-134F9;Lo;0;L;;;;;N;;;;; +134FA;EGYPTIAN HIEROGLYPH-134FA;Lo;0;L;;;;;N;;;;; +134FB;EGYPTIAN HIEROGLYPH-134FB;Lo;0;L;;;;;N;;;;; +134FC;EGYPTIAN HIEROGLYPH-134FC;Lo;0;L;;;;;N;;;;; +134FD;EGYPTIAN HIEROGLYPH-134FD;Lo;0;L;;;;;N;;;;; +134FE;EGYPTIAN HIEROGLYPH-134FE;Lo;0;L;;;;;N;;;;; +134FF;EGYPTIAN HIEROGLYPH-134FF;Lo;0;L;;;;;N;;;;; +13500;EGYPTIAN HIEROGLYPH-13500;Lo;0;L;;;;;N;;;;; +13501;EGYPTIAN HIEROGLYPH-13501;Lo;0;L;;;;;N;;;;; +13502;EGYPTIAN HIEROGLYPH-13502;Lo;0;L;;;;;N;;;;; +13503;EGYPTIAN HIEROGLYPH-13503;Lo;0;L;;;;;N;;;;; +13504;EGYPTIAN HIEROGLYPH-13504;Lo;0;L;;;;;N;;;;; +13505;EGYPTIAN HIEROGLYPH-13505;Lo;0;L;;;;;N;;;;; +13506;EGYPTIAN HIEROGLYPH-13506;Lo;0;L;;;;;N;;;;; +13507;EGYPTIAN HIEROGLYPH-13507;Lo;0;L;;;;;N;;;;; +13508;EGYPTIAN HIEROGLYPH-13508;Lo;0;L;;;;;N;;;;; +13509;EGYPTIAN HIEROGLYPH-13509;Lo;0;L;;;;;N;;;;; +1350A;EGYPTIAN HIEROGLYPH-1350A;Lo;0;L;;;;;N;;;;; +1350B;EGYPTIAN HIEROGLYPH-1350B;Lo;0;L;;;;;N;;;;; +1350C;EGYPTIAN HIEROGLYPH-1350C;Lo;0;L;;;;;N;;;;; +1350D;EGYPTIAN HIEROGLYPH-1350D;Lo;0;L;;;;;N;;;;; +1350E;EGYPTIAN HIEROGLYPH-1350E;Lo;0;L;;;;;N;;;;; +1350F;EGYPTIAN HIEROGLYPH-1350F;Lo;0;L;;;;;N;;;;; +13510;EGYPTIAN HIEROGLYPH-13510;Lo;0;L;;;;;N;;;;; +13511;EGYPTIAN HIEROGLYPH-13511;Lo;0;L;;;;;N;;;;; +13512;EGYPTIAN HIEROGLYPH-13512;Lo;0;L;;;;;N;;;;; +13513;EGYPTIAN HIEROGLYPH-13513;Lo;0;L;;;;;N;;;;; +13514;EGYPTIAN HIEROGLYPH-13514;Lo;0;L;;;;;N;;;;; +13515;EGYPTIAN HIEROGLYPH-13515;Lo;0;L;;;;;N;;;;; +13516;EGYPTIAN HIEROGLYPH-13516;Lo;0;L;;;;;N;;;;; +13517;EGYPTIAN HIEROGLYPH-13517;Lo;0;L;;;;;N;;;;; +13518;EGYPTIAN HIEROGLYPH-13518;Lo;0;L;;;;;N;;;;; +13519;EGYPTIAN HIEROGLYPH-13519;Lo;0;L;;;;;N;;;;; +1351A;EGYPTIAN HIEROGLYPH-1351A;Lo;0;L;;;;;N;;;;; +1351B;EGYPTIAN HIEROGLYPH-1351B;Lo;0;L;;;;;N;;;;; +1351C;EGYPTIAN HIEROGLYPH-1351C;Lo;0;L;;;;;N;;;;; +1351D;EGYPTIAN HIEROGLYPH-1351D;Lo;0;L;;;;;N;;;;; +1351E;EGYPTIAN HIEROGLYPH-1351E;Lo;0;L;;;;;N;;;;; +1351F;EGYPTIAN HIEROGLYPH-1351F;Lo;0;L;;;;;N;;;;; +13520;EGYPTIAN HIEROGLYPH-13520;Lo;0;L;;;;;N;;;;; +13521;EGYPTIAN HIEROGLYPH-13521;Lo;0;L;;;;;N;;;;; +13522;EGYPTIAN HIEROGLYPH-13522;Lo;0;L;;;;;N;;;;; +13523;EGYPTIAN HIEROGLYPH-13523;Lo;0;L;;;;;N;;;;; +13524;EGYPTIAN HIEROGLYPH-13524;Lo;0;L;;;;;N;;;;; +13525;EGYPTIAN HIEROGLYPH-13525;Lo;0;L;;;;;N;;;;; +13526;EGYPTIAN HIEROGLYPH-13526;Lo;0;L;;;;;N;;;;; +13527;EGYPTIAN HIEROGLYPH-13527;Lo;0;L;;;;;N;;;;; +13528;EGYPTIAN HIEROGLYPH-13528;Lo;0;L;;;;;N;;;;; +13529;EGYPTIAN HIEROGLYPH-13529;Lo;0;L;;;;;N;;;;; +1352A;EGYPTIAN HIEROGLYPH-1352A;Lo;0;L;;;;;N;;;;; +1352B;EGYPTIAN HIEROGLYPH-1352B;Lo;0;L;;;;;N;;;;; +1352C;EGYPTIAN HIEROGLYPH-1352C;Lo;0;L;;;;;N;;;;; +1352D;EGYPTIAN HIEROGLYPH-1352D;Lo;0;L;;;;;N;;;;; +1352E;EGYPTIAN HIEROGLYPH-1352E;Lo;0;L;;;;;N;;;;; +1352F;EGYPTIAN HIEROGLYPH-1352F;Lo;0;L;;;;;N;;;;; +13530;EGYPTIAN HIEROGLYPH-13530;Lo;0;L;;;;;N;;;;; +13531;EGYPTIAN HIEROGLYPH-13531;Lo;0;L;;;;;N;;;;; +13532;EGYPTIAN HIEROGLYPH-13532;Lo;0;L;;;;;N;;;;; +13533;EGYPTIAN HIEROGLYPH-13533;Lo;0;L;;;;;N;;;;; +13534;EGYPTIAN HIEROGLYPH-13534;Lo;0;L;;;;;N;;;;; +13535;EGYPTIAN HIEROGLYPH-13535;Lo;0;L;;;;;N;;;;; +13536;EGYPTIAN HIEROGLYPH-13536;Lo;0;L;;;;;N;;;;; +13537;EGYPTIAN HIEROGLYPH-13537;Lo;0;L;;;;;N;;;;; +13538;EGYPTIAN HIEROGLYPH-13538;Lo;0;L;;;;;N;;;;; +13539;EGYPTIAN HIEROGLYPH-13539;Lo;0;L;;;;;N;;;;; +1353A;EGYPTIAN HIEROGLYPH-1353A;Lo;0;L;;;;;N;;;;; +1353B;EGYPTIAN HIEROGLYPH-1353B;Lo;0;L;;;;;N;;;;; +1353C;EGYPTIAN HIEROGLYPH-1353C;Lo;0;L;;;;;N;;;;; +1353D;EGYPTIAN HIEROGLYPH-1353D;Lo;0;L;;;;;N;;;;; +1353E;EGYPTIAN HIEROGLYPH-1353E;Lo;0;L;;;;;N;;;;; +1353F;EGYPTIAN HIEROGLYPH-1353F;Lo;0;L;;;;;N;;;;; +13540;EGYPTIAN HIEROGLYPH-13540;Lo;0;L;;;;;N;;;;; +13541;EGYPTIAN HIEROGLYPH-13541;Lo;0;L;;;;;N;;;;; +13542;EGYPTIAN HIEROGLYPH-13542;Lo;0;L;;;;;N;;;;; +13543;EGYPTIAN HIEROGLYPH-13543;Lo;0;L;;;;;N;;;;; +13544;EGYPTIAN HIEROGLYPH-13544;Lo;0;L;;;;;N;;;;; +13545;EGYPTIAN HIEROGLYPH-13545;Lo;0;L;;;;;N;;;;; +13546;EGYPTIAN HIEROGLYPH-13546;Lo;0;L;;;;;N;;;;; +13547;EGYPTIAN HIEROGLYPH-13547;Lo;0;L;;;;;N;;;;; +13548;EGYPTIAN HIEROGLYPH-13548;Lo;0;L;;;;;N;;;;; +13549;EGYPTIAN HIEROGLYPH-13549;Lo;0;L;;;;;N;;;;; +1354A;EGYPTIAN HIEROGLYPH-1354A;Lo;0;L;;;;;N;;;;; +1354B;EGYPTIAN HIEROGLYPH-1354B;Lo;0;L;;;;;N;;;;; +1354C;EGYPTIAN HIEROGLYPH-1354C;Lo;0;L;;;;;N;;;;; +1354D;EGYPTIAN HIEROGLYPH-1354D;Lo;0;L;;;;;N;;;;; +1354E;EGYPTIAN HIEROGLYPH-1354E;Lo;0;L;;;;;N;;;;; +1354F;EGYPTIAN HIEROGLYPH-1354F;Lo;0;L;;;;;N;;;;; +13550;EGYPTIAN HIEROGLYPH-13550;Lo;0;L;;;;;N;;;;; +13551;EGYPTIAN HIEROGLYPH-13551;Lo;0;L;;;;;N;;;;; +13552;EGYPTIAN HIEROGLYPH-13552;Lo;0;L;;;;;N;;;;; +13553;EGYPTIAN HIEROGLYPH-13553;Lo;0;L;;;;;N;;;;; +13554;EGYPTIAN HIEROGLYPH-13554;Lo;0;L;;;;;N;;;;; +13555;EGYPTIAN HIEROGLYPH-13555;Lo;0;L;;;;;N;;;;; +13556;EGYPTIAN HIEROGLYPH-13556;Lo;0;L;;;;;N;;;;; +13557;EGYPTIAN HIEROGLYPH-13557;Lo;0;L;;;;;N;;;;; +13558;EGYPTIAN HIEROGLYPH-13558;Lo;0;L;;;;;N;;;;; +13559;EGYPTIAN HIEROGLYPH-13559;Lo;0;L;;;;;N;;;;; +1355A;EGYPTIAN HIEROGLYPH-1355A;Lo;0;L;;;;;N;;;;; +1355B;EGYPTIAN HIEROGLYPH-1355B;Lo;0;L;;;;;N;;;;; +1355C;EGYPTIAN HIEROGLYPH-1355C;Lo;0;L;;;;;N;;;;; +1355D;EGYPTIAN HIEROGLYPH-1355D;Lo;0;L;;;;;N;;;;; +1355E;EGYPTIAN HIEROGLYPH-1355E;Lo;0;L;;;;;N;;;;; +1355F;EGYPTIAN HIEROGLYPH-1355F;Lo;0;L;;;;;N;;;;; +13560;EGYPTIAN HIEROGLYPH-13560;Lo;0;L;;;;;N;;;;; +13561;EGYPTIAN HIEROGLYPH-13561;Lo;0;L;;;;;N;;;;; +13562;EGYPTIAN HIEROGLYPH-13562;Lo;0;L;;;;;N;;;;; +13563;EGYPTIAN HIEROGLYPH-13563;Lo;0;L;;;;;N;;;;; +13564;EGYPTIAN HIEROGLYPH-13564;Lo;0;L;;;;;N;;;;; +13565;EGYPTIAN HIEROGLYPH-13565;Lo;0;L;;;;;N;;;;; +13566;EGYPTIAN HIEROGLYPH-13566;Lo;0;L;;;;;N;;;;; +13567;EGYPTIAN HIEROGLYPH-13567;Lo;0;L;;;;;N;;;;; +13568;EGYPTIAN HIEROGLYPH-13568;Lo;0;L;;;;;N;;;;; +13569;EGYPTIAN HIEROGLYPH-13569;Lo;0;L;;;;;N;;;;; +1356A;EGYPTIAN HIEROGLYPH-1356A;Lo;0;L;;;;;N;;;;; +1356B;EGYPTIAN HIEROGLYPH-1356B;Lo;0;L;;;;;N;;;;; +1356C;EGYPTIAN HIEROGLYPH-1356C;Lo;0;L;;;;;N;;;;; +1356D;EGYPTIAN HIEROGLYPH-1356D;Lo;0;L;;;;;N;;;;; +1356E;EGYPTIAN HIEROGLYPH-1356E;Lo;0;L;;;;;N;;;;; +1356F;EGYPTIAN HIEROGLYPH-1356F;Lo;0;L;;;;;N;;;;; +13570;EGYPTIAN HIEROGLYPH-13570;Lo;0;L;;;;;N;;;;; +13571;EGYPTIAN HIEROGLYPH-13571;Lo;0;L;;;;;N;;;;; +13572;EGYPTIAN HIEROGLYPH-13572;Lo;0;L;;;;;N;;;;; +13573;EGYPTIAN HIEROGLYPH-13573;Lo;0;L;;;;;N;;;;; +13574;EGYPTIAN HIEROGLYPH-13574;Lo;0;L;;;;;N;;;;; +13575;EGYPTIAN HIEROGLYPH-13575;Lo;0;L;;;;;N;;;;; +13576;EGYPTIAN HIEROGLYPH-13576;Lo;0;L;;;;;N;;;;; +13577;EGYPTIAN HIEROGLYPH-13577;Lo;0;L;;;;;N;;;;; +13578;EGYPTIAN HIEROGLYPH-13578;Lo;0;L;;;;;N;;;;; +13579;EGYPTIAN HIEROGLYPH-13579;Lo;0;L;;;;;N;;;;; +1357A;EGYPTIAN HIEROGLYPH-1357A;Lo;0;L;;;;;N;;;;; +1357B;EGYPTIAN HIEROGLYPH-1357B;Lo;0;L;;;;;N;;;;; +1357C;EGYPTIAN HIEROGLYPH-1357C;Lo;0;L;;;;;N;;;;; +1357D;EGYPTIAN HIEROGLYPH-1357D;Lo;0;L;;;;;N;;;;; +1357E;EGYPTIAN HIEROGLYPH-1357E;Lo;0;L;;;;;N;;;;; +1357F;EGYPTIAN HIEROGLYPH-1357F;Lo;0;L;;;;;N;;;;; +13580;EGYPTIAN HIEROGLYPH-13580;Lo;0;L;;;;;N;;;;; +13581;EGYPTIAN HIEROGLYPH-13581;Lo;0;L;;;;;N;;;;; +13582;EGYPTIAN HIEROGLYPH-13582;Lo;0;L;;;;;N;;;;; +13583;EGYPTIAN HIEROGLYPH-13583;Lo;0;L;;;;;N;;;;; +13584;EGYPTIAN HIEROGLYPH-13584;Lo;0;L;;;;;N;;;;; +13585;EGYPTIAN HIEROGLYPH-13585;Lo;0;L;;;;;N;;;;; +13586;EGYPTIAN HIEROGLYPH-13586;Lo;0;L;;;;;N;;;;; +13587;EGYPTIAN HIEROGLYPH-13587;Lo;0;L;;;;;N;;;;; +13588;EGYPTIAN HIEROGLYPH-13588;Lo;0;L;;;;;N;;;;; +13589;EGYPTIAN HIEROGLYPH-13589;Lo;0;L;;;;;N;;;;; +1358A;EGYPTIAN HIEROGLYPH-1358A;Lo;0;L;;;;;N;;;;; +1358B;EGYPTIAN HIEROGLYPH-1358B;Lo;0;L;;;;;N;;;;; +1358C;EGYPTIAN HIEROGLYPH-1358C;Lo;0;L;;;;;N;;;;; +1358D;EGYPTIAN HIEROGLYPH-1358D;Lo;0;L;;;;;N;;;;; +1358E;EGYPTIAN HIEROGLYPH-1358E;Lo;0;L;;;;;N;;;;; +1358F;EGYPTIAN HIEROGLYPH-1358F;Lo;0;L;;;;;N;;;;; +13590;EGYPTIAN HIEROGLYPH-13590;Lo;0;L;;;;;N;;;;; +13591;EGYPTIAN HIEROGLYPH-13591;Lo;0;L;;;;;N;;;;; +13592;EGYPTIAN HIEROGLYPH-13592;Lo;0;L;;;;;N;;;;; +13593;EGYPTIAN HIEROGLYPH-13593;Lo;0;L;;;;;N;;;;; +13594;EGYPTIAN HIEROGLYPH-13594;Lo;0;L;;;;;N;;;;; +13595;EGYPTIAN HIEROGLYPH-13595;Lo;0;L;;;;;N;;;;; +13596;EGYPTIAN HIEROGLYPH-13596;Lo;0;L;;;;;N;;;;; +13597;EGYPTIAN HIEROGLYPH-13597;Lo;0;L;;;;;N;;;;; +13598;EGYPTIAN HIEROGLYPH-13598;Lo;0;L;;;;;N;;;;; +13599;EGYPTIAN HIEROGLYPH-13599;Lo;0;L;;;;;N;;;;; +1359A;EGYPTIAN HIEROGLYPH-1359A;Lo;0;L;;;;;N;;;;; +1359B;EGYPTIAN HIEROGLYPH-1359B;Lo;0;L;;;;;N;;;;; +1359C;EGYPTIAN HIEROGLYPH-1359C;Lo;0;L;;;;;N;;;;; +1359D;EGYPTIAN HIEROGLYPH-1359D;Lo;0;L;;;;;N;;;;; +1359E;EGYPTIAN HIEROGLYPH-1359E;Lo;0;L;;;;;N;;;;; +1359F;EGYPTIAN HIEROGLYPH-1359F;Lo;0;L;;;;;N;;;;; +135A0;EGYPTIAN HIEROGLYPH-135A0;Lo;0;L;;;;;N;;;;; +135A1;EGYPTIAN HIEROGLYPH-135A1;Lo;0;L;;;;;N;;;;; +135A2;EGYPTIAN HIEROGLYPH-135A2;Lo;0;L;;;;;N;;;;; +135A3;EGYPTIAN HIEROGLYPH-135A3;Lo;0;L;;;;;N;;;;; +135A4;EGYPTIAN HIEROGLYPH-135A4;Lo;0;L;;;;;N;;;;; +135A5;EGYPTIAN HIEROGLYPH-135A5;Lo;0;L;;;;;N;;;;; +135A6;EGYPTIAN HIEROGLYPH-135A6;Lo;0;L;;;;;N;;;;; +135A7;EGYPTIAN HIEROGLYPH-135A7;Lo;0;L;;;;;N;;;;; +135A8;EGYPTIAN HIEROGLYPH-135A8;Lo;0;L;;;;;N;;;;; +135A9;EGYPTIAN HIEROGLYPH-135A9;Lo;0;L;;;;;N;;;;; +135AA;EGYPTIAN HIEROGLYPH-135AA;Lo;0;L;;;;;N;;;;; +135AB;EGYPTIAN HIEROGLYPH-135AB;Lo;0;L;;;;;N;;;;; +135AC;EGYPTIAN HIEROGLYPH-135AC;Lo;0;L;;;;;N;;;;; +135AD;EGYPTIAN HIEROGLYPH-135AD;Lo;0;L;;;;;N;;;;; +135AE;EGYPTIAN HIEROGLYPH-135AE;Lo;0;L;;;;;N;;;;; +135AF;EGYPTIAN HIEROGLYPH-135AF;Lo;0;L;;;;;N;;;;; +135B0;EGYPTIAN HIEROGLYPH-135B0;Lo;0;L;;;;;N;;;;; +135B1;EGYPTIAN HIEROGLYPH-135B1;Lo;0;L;;;;;N;;;;; +135B2;EGYPTIAN HIEROGLYPH-135B2;Lo;0;L;;;;;N;;;;; +135B3;EGYPTIAN HIEROGLYPH-135B3;Lo;0;L;;;;;N;;;;; +135B4;EGYPTIAN HIEROGLYPH-135B4;Lo;0;L;;;;;N;;;;; +135B5;EGYPTIAN HIEROGLYPH-135B5;Lo;0;L;;;;;N;;;;; +135B6;EGYPTIAN HIEROGLYPH-135B6;Lo;0;L;;;;;N;;;;; +135B7;EGYPTIAN HIEROGLYPH-135B7;Lo;0;L;;;;;N;;;;; +135B8;EGYPTIAN HIEROGLYPH-135B8;Lo;0;L;;;;;N;;;;; +135B9;EGYPTIAN HIEROGLYPH-135B9;Lo;0;L;;;;;N;;;;; +135BA;EGYPTIAN HIEROGLYPH-135BA;Lo;0;L;;;;;N;;;;; +135BB;EGYPTIAN HIEROGLYPH-135BB;Lo;0;L;;;;;N;;;;; +135BC;EGYPTIAN HIEROGLYPH-135BC;Lo;0;L;;;;;N;;;;; +135BD;EGYPTIAN HIEROGLYPH-135BD;Lo;0;L;;;;;N;;;;; +135BE;EGYPTIAN HIEROGLYPH-135BE;Lo;0;L;;;;;N;;;;; +135BF;EGYPTIAN HIEROGLYPH-135BF;Lo;0;L;;;;;N;;;;; +135C0;EGYPTIAN HIEROGLYPH-135C0;Lo;0;L;;;;;N;;;;; +135C1;EGYPTIAN HIEROGLYPH-135C1;Lo;0;L;;;;;N;;;;; +135C2;EGYPTIAN HIEROGLYPH-135C2;Lo;0;L;;;;;N;;;;; +135C3;EGYPTIAN HIEROGLYPH-135C3;Lo;0;L;;;;;N;;;;; +135C4;EGYPTIAN HIEROGLYPH-135C4;Lo;0;L;;;;;N;;;;; +135C5;EGYPTIAN HIEROGLYPH-135C5;Lo;0;L;;;;;N;;;;; +135C6;EGYPTIAN HIEROGLYPH-135C6;Lo;0;L;;;;;N;;;;; +135C7;EGYPTIAN HIEROGLYPH-135C7;Lo;0;L;;;;;N;;;;; +135C8;EGYPTIAN HIEROGLYPH-135C8;Lo;0;L;;;;;N;;;;; +135C9;EGYPTIAN HIEROGLYPH-135C9;Lo;0;L;;;;;N;;;;; +135CA;EGYPTIAN HIEROGLYPH-135CA;Lo;0;L;;;;;N;;;;; +135CB;EGYPTIAN HIEROGLYPH-135CB;Lo;0;L;;;;;N;;;;; +135CC;EGYPTIAN HIEROGLYPH-135CC;Lo;0;L;;;;;N;;;;; +135CD;EGYPTIAN HIEROGLYPH-135CD;Lo;0;L;;;;;N;;;;; +135CE;EGYPTIAN HIEROGLYPH-135CE;Lo;0;L;;;;;N;;;;; +135CF;EGYPTIAN HIEROGLYPH-135CF;Lo;0;L;;;;;N;;;;; +135D0;EGYPTIAN HIEROGLYPH-135D0;Lo;0;L;;;;;N;;;;; +135D1;EGYPTIAN HIEROGLYPH-135D1;Lo;0;L;;;;;N;;;;; +135D2;EGYPTIAN HIEROGLYPH-135D2;Lo;0;L;;;;;N;;;;; +135D3;EGYPTIAN HIEROGLYPH-135D3;Lo;0;L;;;;;N;;;;; +135D4;EGYPTIAN HIEROGLYPH-135D4;Lo;0;L;;;;;N;;;;; +135D5;EGYPTIAN HIEROGLYPH-135D5;Lo;0;L;;;;;N;;;;; +135D6;EGYPTIAN HIEROGLYPH-135D6;Lo;0;L;;;;;N;;;;; +135D7;EGYPTIAN HIEROGLYPH-135D7;Lo;0;L;;;;;N;;;;; +135D8;EGYPTIAN HIEROGLYPH-135D8;Lo;0;L;;;;;N;;;;; +135D9;EGYPTIAN HIEROGLYPH-135D9;Lo;0;L;;;;;N;;;;; +135DA;EGYPTIAN HIEROGLYPH-135DA;Lo;0;L;;;;;N;;;;; +135DB;EGYPTIAN HIEROGLYPH-135DB;Lo;0;L;;;;;N;;;;; +135DC;EGYPTIAN HIEROGLYPH-135DC;Lo;0;L;;;;;N;;;;; +135DD;EGYPTIAN HIEROGLYPH-135DD;Lo;0;L;;;;;N;;;;; +135DE;EGYPTIAN HIEROGLYPH-135DE;Lo;0;L;;;;;N;;;;; +135DF;EGYPTIAN HIEROGLYPH-135DF;Lo;0;L;;;;;N;;;;; +135E0;EGYPTIAN HIEROGLYPH-135E0;Lo;0;L;;;;;N;;;;; +135E1;EGYPTIAN HIEROGLYPH-135E1;Lo;0;L;;;;;N;;;;; +135E2;EGYPTIAN HIEROGLYPH-135E2;Lo;0;L;;;;;N;;;;; +135E3;EGYPTIAN HIEROGLYPH-135E3;Lo;0;L;;;;;N;;;;; +135E4;EGYPTIAN HIEROGLYPH-135E4;Lo;0;L;;;;;N;;;;; +135E5;EGYPTIAN HIEROGLYPH-135E5;Lo;0;L;;;;;N;;;;; +135E6;EGYPTIAN HIEROGLYPH-135E6;Lo;0;L;;;;;N;;;;; +135E7;EGYPTIAN HIEROGLYPH-135E7;Lo;0;L;;;;;N;;;;; +135E8;EGYPTIAN HIEROGLYPH-135E8;Lo;0;L;;;;;N;;;;; +135E9;EGYPTIAN HIEROGLYPH-135E9;Lo;0;L;;;;;N;;;;; +135EA;EGYPTIAN HIEROGLYPH-135EA;Lo;0;L;;;;;N;;;;; +135EB;EGYPTIAN HIEROGLYPH-135EB;Lo;0;L;;;;;N;;;;; +135EC;EGYPTIAN HIEROGLYPH-135EC;Lo;0;L;;;;;N;;;;; +135ED;EGYPTIAN HIEROGLYPH-135ED;Lo;0;L;;;;;N;;;;; +135EE;EGYPTIAN HIEROGLYPH-135EE;Lo;0;L;;;;;N;;;;; +135EF;EGYPTIAN HIEROGLYPH-135EF;Lo;0;L;;;;;N;;;;; +135F0;EGYPTIAN HIEROGLYPH-135F0;Lo;0;L;;;;;N;;;;; +135F1;EGYPTIAN HIEROGLYPH-135F1;Lo;0;L;;;;;N;;;;; +135F2;EGYPTIAN HIEROGLYPH-135F2;Lo;0;L;;;;;N;;;;; +135F3;EGYPTIAN HIEROGLYPH-135F3;Lo;0;L;;;;;N;;;;; +135F4;EGYPTIAN HIEROGLYPH-135F4;Lo;0;L;;;;;N;;;;; +135F5;EGYPTIAN HIEROGLYPH-135F5;Lo;0;L;;;;;N;;;;; +135F6;EGYPTIAN HIEROGLYPH-135F6;Lo;0;L;;;;;N;;;;; +135F7;EGYPTIAN HIEROGLYPH-135F7;Lo;0;L;;;;;N;;;;; +135F8;EGYPTIAN HIEROGLYPH-135F8;Lo;0;L;;;;;N;;;;; +135F9;EGYPTIAN HIEROGLYPH-135F9;Lo;0;L;;;;;N;;;;; +135FA;EGYPTIAN HIEROGLYPH-135FA;Lo;0;L;;;;;N;;;;; +135FB;EGYPTIAN HIEROGLYPH-135FB;Lo;0;L;;;;;N;;;;; +135FC;EGYPTIAN HIEROGLYPH-135FC;Lo;0;L;;;;;N;;;;; +135FD;EGYPTIAN HIEROGLYPH-135FD;Lo;0;L;;;;;N;;;;; +135FE;EGYPTIAN HIEROGLYPH-135FE;Lo;0;L;;;;;N;;;;; +135FF;EGYPTIAN HIEROGLYPH-135FF;Lo;0;L;;;;;N;;;;; +13600;EGYPTIAN HIEROGLYPH-13600;Lo;0;L;;;;;N;;;;; +13601;EGYPTIAN HIEROGLYPH-13601;Lo;0;L;;;;;N;;;;; +13602;EGYPTIAN HIEROGLYPH-13602;Lo;0;L;;;;;N;;;;; +13603;EGYPTIAN HIEROGLYPH-13603;Lo;0;L;;;;;N;;;;; +13604;EGYPTIAN HIEROGLYPH-13604;Lo;0;L;;;;;N;;;;; +13605;EGYPTIAN HIEROGLYPH-13605;Lo;0;L;;;;;N;;;;; +13606;EGYPTIAN HIEROGLYPH-13606;Lo;0;L;;;;;N;;;;; +13607;EGYPTIAN HIEROGLYPH-13607;Lo;0;L;;;;;N;;;;; +13608;EGYPTIAN HIEROGLYPH-13608;Lo;0;L;;;;;N;;;;; +13609;EGYPTIAN HIEROGLYPH-13609;Lo;0;L;;;;;N;;;;; +1360A;EGYPTIAN HIEROGLYPH-1360A;Lo;0;L;;;;;N;;;;; +1360B;EGYPTIAN HIEROGLYPH-1360B;Lo;0;L;;;;;N;;;;; +1360C;EGYPTIAN HIEROGLYPH-1360C;Lo;0;L;;;;;N;;;;; +1360D;EGYPTIAN HIEROGLYPH-1360D;Lo;0;L;;;;;N;;;;; +1360E;EGYPTIAN HIEROGLYPH-1360E;Lo;0;L;;;;;N;;;;; +1360F;EGYPTIAN HIEROGLYPH-1360F;Lo;0;L;;;;;N;;;;; +13610;EGYPTIAN HIEROGLYPH-13610;Lo;0;L;;;;;N;;;;; +13611;EGYPTIAN HIEROGLYPH-13611;Lo;0;L;;;;;N;;;;; +13612;EGYPTIAN HIEROGLYPH-13612;Lo;0;L;;;;;N;;;;; +13613;EGYPTIAN HIEROGLYPH-13613;Lo;0;L;;;;;N;;;;; +13614;EGYPTIAN HIEROGLYPH-13614;Lo;0;L;;;;;N;;;;; +13615;EGYPTIAN HIEROGLYPH-13615;Lo;0;L;;;;;N;;;;; +13616;EGYPTIAN HIEROGLYPH-13616;Lo;0;L;;;;;N;;;;; +13617;EGYPTIAN HIEROGLYPH-13617;Lo;0;L;;;;;N;;;;; +13618;EGYPTIAN HIEROGLYPH-13618;Lo;0;L;;;;;N;;;;; +13619;EGYPTIAN HIEROGLYPH-13619;Lo;0;L;;;;;N;;;;; +1361A;EGYPTIAN HIEROGLYPH-1361A;Lo;0;L;;;;;N;;;;; +1361B;EGYPTIAN HIEROGLYPH-1361B;Lo;0;L;;;;;N;;;;; +1361C;EGYPTIAN HIEROGLYPH-1361C;Lo;0;L;;;;;N;;;;; +1361D;EGYPTIAN HIEROGLYPH-1361D;Lo;0;L;;;;;N;;;;; +1361E;EGYPTIAN HIEROGLYPH-1361E;Lo;0;L;;;;;N;;;;; +1361F;EGYPTIAN HIEROGLYPH-1361F;Lo;0;L;;;;;N;;;;; +13620;EGYPTIAN HIEROGLYPH-13620;Lo;0;L;;;;;N;;;;; +13621;EGYPTIAN HIEROGLYPH-13621;Lo;0;L;;;;;N;;;;; +13622;EGYPTIAN HIEROGLYPH-13622;Lo;0;L;;;;;N;;;;; +13623;EGYPTIAN HIEROGLYPH-13623;Lo;0;L;;;;;N;;;;; +13624;EGYPTIAN HIEROGLYPH-13624;Lo;0;L;;;;;N;;;;; +13625;EGYPTIAN HIEROGLYPH-13625;Lo;0;L;;;;;N;;;;; +13626;EGYPTIAN HIEROGLYPH-13626;Lo;0;L;;;;;N;;;;; +13627;EGYPTIAN HIEROGLYPH-13627;Lo;0;L;;;;;N;;;;; +13628;EGYPTIAN HIEROGLYPH-13628;Lo;0;L;;;;;N;;;;; +13629;EGYPTIAN HIEROGLYPH-13629;Lo;0;L;;;;;N;;;;; +1362A;EGYPTIAN HIEROGLYPH-1362A;Lo;0;L;;;;;N;;;;; +1362B;EGYPTIAN HIEROGLYPH-1362B;Lo;0;L;;;;;N;;;;; +1362C;EGYPTIAN HIEROGLYPH-1362C;Lo;0;L;;;;;N;;;;; +1362D;EGYPTIAN HIEROGLYPH-1362D;Lo;0;L;;;;;N;;;;; +1362E;EGYPTIAN HIEROGLYPH-1362E;Lo;0;L;;;;;N;;;;; +1362F;EGYPTIAN HIEROGLYPH-1362F;Lo;0;L;;;;;N;;;;; +13630;EGYPTIAN HIEROGLYPH-13630;Lo;0;L;;;;;N;;;;; +13631;EGYPTIAN HIEROGLYPH-13631;Lo;0;L;;;;;N;;;;; +13632;EGYPTIAN HIEROGLYPH-13632;Lo;0;L;;;;;N;;;;; +13633;EGYPTIAN HIEROGLYPH-13633;Lo;0;L;;;;;N;;;;; +13634;EGYPTIAN HIEROGLYPH-13634;Lo;0;L;;;;;N;;;;; +13635;EGYPTIAN HIEROGLYPH-13635;Lo;0;L;;;;;N;;;;; +13636;EGYPTIAN HIEROGLYPH-13636;Lo;0;L;;;;;N;;;;; +13637;EGYPTIAN HIEROGLYPH-13637;Lo;0;L;;;;;N;;;;; +13638;EGYPTIAN HIEROGLYPH-13638;Lo;0;L;;;;;N;;;;; +13639;EGYPTIAN HIEROGLYPH-13639;Lo;0;L;;;;;N;;;;; +1363A;EGYPTIAN HIEROGLYPH-1363A;Lo;0;L;;;;;N;;;;; +1363B;EGYPTIAN HIEROGLYPH-1363B;Lo;0;L;;;;;N;;;;; +1363C;EGYPTIAN HIEROGLYPH-1363C;Lo;0;L;;;;;N;;;;; +1363D;EGYPTIAN HIEROGLYPH-1363D;Lo;0;L;;;;;N;;;;; +1363E;EGYPTIAN HIEROGLYPH-1363E;Lo;0;L;;;;;N;;;;; +1363F;EGYPTIAN HIEROGLYPH-1363F;Lo;0;L;;;;;N;;;;; +13640;EGYPTIAN HIEROGLYPH-13640;Lo;0;L;;;;;N;;;;; +13641;EGYPTIAN HIEROGLYPH-13641;Lo;0;L;;;;;N;;;;; +13642;EGYPTIAN HIEROGLYPH-13642;Lo;0;L;;;;;N;;;;; +13643;EGYPTIAN HIEROGLYPH-13643;Lo;0;L;;;;;N;;;;; +13644;EGYPTIAN HIEROGLYPH-13644;Lo;0;L;;;;;N;;;;; +13645;EGYPTIAN HIEROGLYPH-13645;Lo;0;L;;;;;N;;;;; +13646;EGYPTIAN HIEROGLYPH-13646;Lo;0;L;;;;;N;;;;; +13647;EGYPTIAN HIEROGLYPH-13647;Lo;0;L;;;;;N;;;;; +13648;EGYPTIAN HIEROGLYPH-13648;Lo;0;L;;;;;N;;;;; +13649;EGYPTIAN HIEROGLYPH-13649;Lo;0;L;;;;;N;;;;; +1364A;EGYPTIAN HIEROGLYPH-1364A;Lo;0;L;;;;;N;;;;; +1364B;EGYPTIAN HIEROGLYPH-1364B;Lo;0;L;;;;;N;;;;; +1364C;EGYPTIAN HIEROGLYPH-1364C;Lo;0;L;;;;;N;;;;; +1364D;EGYPTIAN HIEROGLYPH-1364D;Lo;0;L;;;;;N;;;;; +1364E;EGYPTIAN HIEROGLYPH-1364E;Lo;0;L;;;;;N;;;;; +1364F;EGYPTIAN HIEROGLYPH-1364F;Lo;0;L;;;;;N;;;;; +13650;EGYPTIAN HIEROGLYPH-13650;Lo;0;L;;;;;N;;;;; +13651;EGYPTIAN HIEROGLYPH-13651;Lo;0;L;;;;;N;;;;; +13652;EGYPTIAN HIEROGLYPH-13652;Lo;0;L;;;;;N;;;;; +13653;EGYPTIAN HIEROGLYPH-13653;Lo;0;L;;;;;N;;;;; +13654;EGYPTIAN HIEROGLYPH-13654;Lo;0;L;;;;;N;;;;; +13655;EGYPTIAN HIEROGLYPH-13655;Lo;0;L;;;;;N;;;;; +13656;EGYPTIAN HIEROGLYPH-13656;Lo;0;L;;;;;N;;;;; +13657;EGYPTIAN HIEROGLYPH-13657;Lo;0;L;;;;;N;;;;; +13658;EGYPTIAN HIEROGLYPH-13658;Lo;0;L;;;;;N;;;;; +13659;EGYPTIAN HIEROGLYPH-13659;Lo;0;L;;;;;N;;;;; +1365A;EGYPTIAN HIEROGLYPH-1365A;Lo;0;L;;;;;N;;;;; +1365B;EGYPTIAN HIEROGLYPH-1365B;Lo;0;L;;;;;N;;;;; +1365C;EGYPTIAN HIEROGLYPH-1365C;Lo;0;L;;;;;N;;;;; +1365D;EGYPTIAN HIEROGLYPH-1365D;Lo;0;L;;;;;N;;;;; +1365E;EGYPTIAN HIEROGLYPH-1365E;Lo;0;L;;;;;N;;;;; +1365F;EGYPTIAN HIEROGLYPH-1365F;Lo;0;L;;;;;N;;;;; +13660;EGYPTIAN HIEROGLYPH-13660;Lo;0;L;;;;;N;;;;; +13661;EGYPTIAN HIEROGLYPH-13661;Lo;0;L;;;;;N;;;;; +13662;EGYPTIAN HIEROGLYPH-13662;Lo;0;L;;;;;N;;;;; +13663;EGYPTIAN HIEROGLYPH-13663;Lo;0;L;;;;;N;;;;; +13664;EGYPTIAN HIEROGLYPH-13664;Lo;0;L;;;;;N;;;;; +13665;EGYPTIAN HIEROGLYPH-13665;Lo;0;L;;;;;N;;;;; +13666;EGYPTIAN HIEROGLYPH-13666;Lo;0;L;;;;;N;;;;; +13667;EGYPTIAN HIEROGLYPH-13667;Lo;0;L;;;;;N;;;;; +13668;EGYPTIAN HIEROGLYPH-13668;Lo;0;L;;;;;N;;;;; +13669;EGYPTIAN HIEROGLYPH-13669;Lo;0;L;;;;;N;;;;; +1366A;EGYPTIAN HIEROGLYPH-1366A;Lo;0;L;;;;;N;;;;; +1366B;EGYPTIAN HIEROGLYPH-1366B;Lo;0;L;;;;;N;;;;; +1366C;EGYPTIAN HIEROGLYPH-1366C;Lo;0;L;;;;;N;;;;; +1366D;EGYPTIAN HIEROGLYPH-1366D;Lo;0;L;;;;;N;;;;; +1366E;EGYPTIAN HIEROGLYPH-1366E;Lo;0;L;;;;;N;;;;; +1366F;EGYPTIAN HIEROGLYPH-1366F;Lo;0;L;;;;;N;;;;; +13670;EGYPTIAN HIEROGLYPH-13670;Lo;0;L;;;;;N;;;;; +13671;EGYPTIAN HIEROGLYPH-13671;Lo;0;L;;;;;N;;;;; +13672;EGYPTIAN HIEROGLYPH-13672;Lo;0;L;;;;;N;;;;; +13673;EGYPTIAN HIEROGLYPH-13673;Lo;0;L;;;;;N;;;;; +13674;EGYPTIAN HIEROGLYPH-13674;Lo;0;L;;;;;N;;;;; +13675;EGYPTIAN HIEROGLYPH-13675;Lo;0;L;;;;;N;;;;; +13676;EGYPTIAN HIEROGLYPH-13676;Lo;0;L;;;;;N;;;;; +13677;EGYPTIAN HIEROGLYPH-13677;Lo;0;L;;;;;N;;;;; +13678;EGYPTIAN HIEROGLYPH-13678;Lo;0;L;;;;;N;;;;; +13679;EGYPTIAN HIEROGLYPH-13679;Lo;0;L;;;;;N;;;;; +1367A;EGYPTIAN HIEROGLYPH-1367A;Lo;0;L;;;;;N;;;;; +1367B;EGYPTIAN HIEROGLYPH-1367B;Lo;0;L;;;;;N;;;;; +1367C;EGYPTIAN HIEROGLYPH-1367C;Lo;0;L;;;;;N;;;;; +1367D;EGYPTIAN HIEROGLYPH-1367D;Lo;0;L;;;;;N;;;;; +1367E;EGYPTIAN HIEROGLYPH-1367E;Lo;0;L;;;;;N;;;;; +1367F;EGYPTIAN HIEROGLYPH-1367F;Lo;0;L;;;;;N;;;;; +13680;EGYPTIAN HIEROGLYPH-13680;Lo;0;L;;;;;N;;;;; +13681;EGYPTIAN HIEROGLYPH-13681;Lo;0;L;;;;;N;;;;; +13682;EGYPTIAN HIEROGLYPH-13682;Lo;0;L;;;;;N;;;;; +13683;EGYPTIAN HIEROGLYPH-13683;Lo;0;L;;;;;N;;;;; +13684;EGYPTIAN HIEROGLYPH-13684;Lo;0;L;;;;;N;;;;; +13685;EGYPTIAN HIEROGLYPH-13685;Lo;0;L;;;;;N;;;;; +13686;EGYPTIAN HIEROGLYPH-13686;Lo;0;L;;;;;N;;;;; +13687;EGYPTIAN HIEROGLYPH-13687;Lo;0;L;;;;;N;;;;; +13688;EGYPTIAN HIEROGLYPH-13688;Lo;0;L;;;;;N;;;;; +13689;EGYPTIAN HIEROGLYPH-13689;Lo;0;L;;;;;N;;;;; +1368A;EGYPTIAN HIEROGLYPH-1368A;Lo;0;L;;;;;N;;;;; +1368B;EGYPTIAN HIEROGLYPH-1368B;Lo;0;L;;;;;N;;;;; +1368C;EGYPTIAN HIEROGLYPH-1368C;Lo;0;L;;;;;N;;;;; +1368D;EGYPTIAN HIEROGLYPH-1368D;Lo;0;L;;;;;N;;;;; +1368E;EGYPTIAN HIEROGLYPH-1368E;Lo;0;L;;;;;N;;;;; +1368F;EGYPTIAN HIEROGLYPH-1368F;Lo;0;L;;;;;N;;;;; +13690;EGYPTIAN HIEROGLYPH-13690;Lo;0;L;;;;;N;;;;; +13691;EGYPTIAN HIEROGLYPH-13691;Lo;0;L;;;;;N;;;;; +13692;EGYPTIAN HIEROGLYPH-13692;Lo;0;L;;;;;N;;;;; +13693;EGYPTIAN HIEROGLYPH-13693;Lo;0;L;;;;;N;;;;; +13694;EGYPTIAN HIEROGLYPH-13694;Lo;0;L;;;;;N;;;;; +13695;EGYPTIAN HIEROGLYPH-13695;Lo;0;L;;;;;N;;;;; +13696;EGYPTIAN HIEROGLYPH-13696;Lo;0;L;;;;;N;;;;; +13697;EGYPTIAN HIEROGLYPH-13697;Lo;0;L;;;;;N;;;;; +13698;EGYPTIAN HIEROGLYPH-13698;Lo;0;L;;;;;N;;;;; +13699;EGYPTIAN HIEROGLYPH-13699;Lo;0;L;;;;;N;;;;; +1369A;EGYPTIAN HIEROGLYPH-1369A;Lo;0;L;;;;;N;;;;; +1369B;EGYPTIAN HIEROGLYPH-1369B;Lo;0;L;;;;;N;;;;; +1369C;EGYPTIAN HIEROGLYPH-1369C;Lo;0;L;;;;;N;;;;; +1369D;EGYPTIAN HIEROGLYPH-1369D;Lo;0;L;;;;;N;;;;; +1369E;EGYPTIAN HIEROGLYPH-1369E;Lo;0;L;;;;;N;;;;; +1369F;EGYPTIAN HIEROGLYPH-1369F;Lo;0;L;;;;;N;;;;; +136A0;EGYPTIAN HIEROGLYPH-136A0;Lo;0;L;;;;;N;;;;; +136A1;EGYPTIAN HIEROGLYPH-136A1;Lo;0;L;;;;;N;;;;; +136A2;EGYPTIAN HIEROGLYPH-136A2;Lo;0;L;;;;;N;;;;; +136A3;EGYPTIAN HIEROGLYPH-136A3;Lo;0;L;;;;;N;;;;; +136A4;EGYPTIAN HIEROGLYPH-136A4;Lo;0;L;;;;;N;;;;; +136A5;EGYPTIAN HIEROGLYPH-136A5;Lo;0;L;;;;;N;;;;; +136A6;EGYPTIAN HIEROGLYPH-136A6;Lo;0;L;;;;;N;;;;; +136A7;EGYPTIAN HIEROGLYPH-136A7;Lo;0;L;;;;;N;;;;; +136A8;EGYPTIAN HIEROGLYPH-136A8;Lo;0;L;;;;;N;;;;; +136A9;EGYPTIAN HIEROGLYPH-136A9;Lo;0;L;;;;;N;;;;; +136AA;EGYPTIAN HIEROGLYPH-136AA;Lo;0;L;;;;;N;;;;; +136AB;EGYPTIAN HIEROGLYPH-136AB;Lo;0;L;;;;;N;;;;; +136AC;EGYPTIAN HIEROGLYPH-136AC;Lo;0;L;;;;;N;;;;; +136AD;EGYPTIAN HIEROGLYPH-136AD;Lo;0;L;;;;;N;;;;; +136AE;EGYPTIAN HIEROGLYPH-136AE;Lo;0;L;;;;;N;;;;; +136AF;EGYPTIAN HIEROGLYPH-136AF;Lo;0;L;;;;;N;;;;; +136B0;EGYPTIAN HIEROGLYPH-136B0;Lo;0;L;;;;;N;;;;; +136B1;EGYPTIAN HIEROGLYPH-136B1;Lo;0;L;;;;;N;;;;; +136B2;EGYPTIAN HIEROGLYPH-136B2;Lo;0;L;;;;;N;;;;; +136B3;EGYPTIAN HIEROGLYPH-136B3;Lo;0;L;;;;;N;;;;; +136B4;EGYPTIAN HIEROGLYPH-136B4;Lo;0;L;;;;;N;;;;; +136B5;EGYPTIAN HIEROGLYPH-136B5;Lo;0;L;;;;;N;;;;; +136B6;EGYPTIAN HIEROGLYPH-136B6;Lo;0;L;;;;;N;;;;; +136B7;EGYPTIAN HIEROGLYPH-136B7;Lo;0;L;;;;;N;;;;; +136B8;EGYPTIAN HIEROGLYPH-136B8;Lo;0;L;;;;;N;;;;; +136B9;EGYPTIAN HIEROGLYPH-136B9;Lo;0;L;;;;;N;;;;; +136BA;EGYPTIAN HIEROGLYPH-136BA;Lo;0;L;;;;;N;;;;; +136BB;EGYPTIAN HIEROGLYPH-136BB;Lo;0;L;;;;;N;;;;; +136BC;EGYPTIAN HIEROGLYPH-136BC;Lo;0;L;;;;;N;;;;; +136BD;EGYPTIAN HIEROGLYPH-136BD;Lo;0;L;;;;;N;;;;; +136BE;EGYPTIAN HIEROGLYPH-136BE;Lo;0;L;;;;;N;;;;; +136BF;EGYPTIAN HIEROGLYPH-136BF;Lo;0;L;;;;;N;;;;; +136C0;EGYPTIAN HIEROGLYPH-136C0;Lo;0;L;;;;;N;;;;; +136C1;EGYPTIAN HIEROGLYPH-136C1;Lo;0;L;;;;;N;;;;; +136C2;EGYPTIAN HIEROGLYPH-136C2;Lo;0;L;;;;;N;;;;; +136C3;EGYPTIAN HIEROGLYPH-136C3;Lo;0;L;;;;;N;;;;; +136C4;EGYPTIAN HIEROGLYPH-136C4;Lo;0;L;;;;;N;;;;; +136C5;EGYPTIAN HIEROGLYPH-136C5;Lo;0;L;;;;;N;;;;; +136C6;EGYPTIAN HIEROGLYPH-136C6;Lo;0;L;;;;;N;;;;; +136C7;EGYPTIAN HIEROGLYPH-136C7;Lo;0;L;;;;;N;;;;; +136C8;EGYPTIAN HIEROGLYPH-136C8;Lo;0;L;;;;;N;;;;; +136C9;EGYPTIAN HIEROGLYPH-136C9;Lo;0;L;;;;;N;;;;; +136CA;EGYPTIAN HIEROGLYPH-136CA;Lo;0;L;;;;;N;;;;; +136CB;EGYPTIAN HIEROGLYPH-136CB;Lo;0;L;;;;;N;;;;; +136CC;EGYPTIAN HIEROGLYPH-136CC;Lo;0;L;;;;;N;;;;; +136CD;EGYPTIAN HIEROGLYPH-136CD;Lo;0;L;;;;;N;;;;; +136CE;EGYPTIAN HIEROGLYPH-136CE;Lo;0;L;;;;;N;;;;; +136CF;EGYPTIAN HIEROGLYPH-136CF;Lo;0;L;;;;;N;;;;; +136D0;EGYPTIAN HIEROGLYPH-136D0;Lo;0;L;;;;;N;;;;; +136D1;EGYPTIAN HIEROGLYPH-136D1;Lo;0;L;;;;;N;;;;; +136D2;EGYPTIAN HIEROGLYPH-136D2;Lo;0;L;;;;;N;;;;; +136D3;EGYPTIAN HIEROGLYPH-136D3;Lo;0;L;;;;;N;;;;; +136D4;EGYPTIAN HIEROGLYPH-136D4;Lo;0;L;;;;;N;;;;; +136D5;EGYPTIAN HIEROGLYPH-136D5;Lo;0;L;;;;;N;;;;; +136D6;EGYPTIAN HIEROGLYPH-136D6;Lo;0;L;;;;;N;;;;; +136D7;EGYPTIAN HIEROGLYPH-136D7;Lo;0;L;;;;;N;;;;; +136D8;EGYPTIAN HIEROGLYPH-136D8;Lo;0;L;;;;;N;;;;; +136D9;EGYPTIAN HIEROGLYPH-136D9;Lo;0;L;;;;;N;;;;; +136DA;EGYPTIAN HIEROGLYPH-136DA;Lo;0;L;;;;;N;;;;; +136DB;EGYPTIAN HIEROGLYPH-136DB;Lo;0;L;;;;;N;;;;; +136DC;EGYPTIAN HIEROGLYPH-136DC;Lo;0;L;;;;;N;;;;; +136DD;EGYPTIAN HIEROGLYPH-136DD;Lo;0;L;;;;;N;;;;; +136DE;EGYPTIAN HIEROGLYPH-136DE;Lo;0;L;;;;;N;;;;; +136DF;EGYPTIAN HIEROGLYPH-136DF;Lo;0;L;;;;;N;;;;; +136E0;EGYPTIAN HIEROGLYPH-136E0;Lo;0;L;;;;;N;;;;; +136E1;EGYPTIAN HIEROGLYPH-136E1;Lo;0;L;;;;;N;;;;; +136E2;EGYPTIAN HIEROGLYPH-136E2;Lo;0;L;;;;;N;;;;; +136E3;EGYPTIAN HIEROGLYPH-136E3;Lo;0;L;;;;;N;;;;; +136E4;EGYPTIAN HIEROGLYPH-136E4;Lo;0;L;;;;;N;;;;; +136E5;EGYPTIAN HIEROGLYPH-136E5;Lo;0;L;;;;;N;;;;; +136E6;EGYPTIAN HIEROGLYPH-136E6;Lo;0;L;;;;;N;;;;; +136E7;EGYPTIAN HIEROGLYPH-136E7;Lo;0;L;;;;;N;;;;; +136E8;EGYPTIAN HIEROGLYPH-136E8;Lo;0;L;;;;;N;;;;; +136E9;EGYPTIAN HIEROGLYPH-136E9;Lo;0;L;;;;;N;;;;; +136EA;EGYPTIAN HIEROGLYPH-136EA;Lo;0;L;;;;;N;;;;; +136EB;EGYPTIAN HIEROGLYPH-136EB;Lo;0;L;;;;;N;;;;; +136EC;EGYPTIAN HIEROGLYPH-136EC;Lo;0;L;;;;;N;;;;; +136ED;EGYPTIAN HIEROGLYPH-136ED;Lo;0;L;;;;;N;;;;; +136EE;EGYPTIAN HIEROGLYPH-136EE;Lo;0;L;;;;;N;;;;; +136EF;EGYPTIAN HIEROGLYPH-136EF;Lo;0;L;;;;;N;;;;; +136F0;EGYPTIAN HIEROGLYPH-136F0;Lo;0;L;;;;;N;;;;; +136F1;EGYPTIAN HIEROGLYPH-136F1;Lo;0;L;;;;;N;;;;; +136F2;EGYPTIAN HIEROGLYPH-136F2;Lo;0;L;;;;;N;;;;; +136F3;EGYPTIAN HIEROGLYPH-136F3;Lo;0;L;;;;;N;;;;; +136F4;EGYPTIAN HIEROGLYPH-136F4;Lo;0;L;;;;;N;;;;; +136F5;EGYPTIAN HIEROGLYPH-136F5;Lo;0;L;;;;;N;;;;; +136F6;EGYPTIAN HIEROGLYPH-136F6;Lo;0;L;;;;;N;;;;; +136F7;EGYPTIAN HIEROGLYPH-136F7;Lo;0;L;;;;;N;;;;; +136F8;EGYPTIAN HIEROGLYPH-136F8;Lo;0;L;;;;;N;;;;; +136F9;EGYPTIAN HIEROGLYPH-136F9;Lo;0;L;;;;;N;;;;; +136FA;EGYPTIAN HIEROGLYPH-136FA;Lo;0;L;;;;;N;;;;; +136FB;EGYPTIAN HIEROGLYPH-136FB;Lo;0;L;;;;;N;;;;; +136FC;EGYPTIAN HIEROGLYPH-136FC;Lo;0;L;;;;;N;;;;; +136FD;EGYPTIAN HIEROGLYPH-136FD;Lo;0;L;;;;;N;;;;; +136FE;EGYPTIAN HIEROGLYPH-136FE;Lo;0;L;;;;;N;;;;; +136FF;EGYPTIAN HIEROGLYPH-136FF;Lo;0;L;;;;;N;;;;; +13700;EGYPTIAN HIEROGLYPH-13700;Lo;0;L;;;;;N;;;;; +13701;EGYPTIAN HIEROGLYPH-13701;Lo;0;L;;;;;N;;;;; +13702;EGYPTIAN HIEROGLYPH-13702;Lo;0;L;;;;;N;;;;; +13703;EGYPTIAN HIEROGLYPH-13703;Lo;0;L;;;;;N;;;;; +13704;EGYPTIAN HIEROGLYPH-13704;Lo;0;L;;;;;N;;;;; +13705;EGYPTIAN HIEROGLYPH-13705;Lo;0;L;;;;;N;;;;; +13706;EGYPTIAN HIEROGLYPH-13706;Lo;0;L;;;;;N;;;;; +13707;EGYPTIAN HIEROGLYPH-13707;Lo;0;L;;;;;N;;;;; +13708;EGYPTIAN HIEROGLYPH-13708;Lo;0;L;;;;;N;;;;; +13709;EGYPTIAN HIEROGLYPH-13709;Lo;0;L;;;;;N;;;;; +1370A;EGYPTIAN HIEROGLYPH-1370A;Lo;0;L;;;;;N;;;;; +1370B;EGYPTIAN HIEROGLYPH-1370B;Lo;0;L;;;;;N;;;;; +1370C;EGYPTIAN HIEROGLYPH-1370C;Lo;0;L;;;;;N;;;;; +1370D;EGYPTIAN HIEROGLYPH-1370D;Lo;0;L;;;;;N;;;;; +1370E;EGYPTIAN HIEROGLYPH-1370E;Lo;0;L;;;;;N;;;;; +1370F;EGYPTIAN HIEROGLYPH-1370F;Lo;0;L;;;;;N;;;;; +13710;EGYPTIAN HIEROGLYPH-13710;Lo;0;L;;;;;N;;;;; +13711;EGYPTIAN HIEROGLYPH-13711;Lo;0;L;;;;;N;;;;; +13712;EGYPTIAN HIEROGLYPH-13712;Lo;0;L;;;;;N;;;;; +13713;EGYPTIAN HIEROGLYPH-13713;Lo;0;L;;;;;N;;;;; +13714;EGYPTIAN HIEROGLYPH-13714;Lo;0;L;;;;;N;;;;; +13715;EGYPTIAN HIEROGLYPH-13715;Lo;0;L;;;;;N;;;;; +13716;EGYPTIAN HIEROGLYPH-13716;Lo;0;L;;;;;N;;;;; +13717;EGYPTIAN HIEROGLYPH-13717;Lo;0;L;;;;;N;;;;; +13718;EGYPTIAN HIEROGLYPH-13718;Lo;0;L;;;;;N;;;;; +13719;EGYPTIAN HIEROGLYPH-13719;Lo;0;L;;;;;N;;;;; +1371A;EGYPTIAN HIEROGLYPH-1371A;Lo;0;L;;;;;N;;;;; +1371B;EGYPTIAN HIEROGLYPH-1371B;Lo;0;L;;;;;N;;;;; +1371C;EGYPTIAN HIEROGLYPH-1371C;Lo;0;L;;;;;N;;;;; +1371D;EGYPTIAN HIEROGLYPH-1371D;Lo;0;L;;;;;N;;;;; +1371E;EGYPTIAN HIEROGLYPH-1371E;Lo;0;L;;;;;N;;;;; +1371F;EGYPTIAN HIEROGLYPH-1371F;Lo;0;L;;;;;N;;;;; +13720;EGYPTIAN HIEROGLYPH-13720;Lo;0;L;;;;;N;;;;; +13721;EGYPTIAN HIEROGLYPH-13721;Lo;0;L;;;;;N;;;;; +13722;EGYPTIAN HIEROGLYPH-13722;Lo;0;L;;;;;N;;;;; +13723;EGYPTIAN HIEROGLYPH-13723;Lo;0;L;;;;;N;;;;; +13724;EGYPTIAN HIEROGLYPH-13724;Lo;0;L;;;;;N;;;;; +13725;EGYPTIAN HIEROGLYPH-13725;Lo;0;L;;;;;N;;;;; +13726;EGYPTIAN HIEROGLYPH-13726;Lo;0;L;;;;;N;;;;; +13727;EGYPTIAN HIEROGLYPH-13727;Lo;0;L;;;;;N;;;;; +13728;EGYPTIAN HIEROGLYPH-13728;Lo;0;L;;;;;N;;;;; +13729;EGYPTIAN HIEROGLYPH-13729;Lo;0;L;;;;;N;;;;; +1372A;EGYPTIAN HIEROGLYPH-1372A;Lo;0;L;;;;;N;;;;; +1372B;EGYPTIAN HIEROGLYPH-1372B;Lo;0;L;;;;;N;;;;; +1372C;EGYPTIAN HIEROGLYPH-1372C;Lo;0;L;;;;;N;;;;; +1372D;EGYPTIAN HIEROGLYPH-1372D;Lo;0;L;;;;;N;;;;; +1372E;EGYPTIAN HIEROGLYPH-1372E;Lo;0;L;;;;;N;;;;; +1372F;EGYPTIAN HIEROGLYPH-1372F;Lo;0;L;;;;;N;;;;; +13730;EGYPTIAN HIEROGLYPH-13730;Lo;0;L;;;;;N;;;;; +13731;EGYPTIAN HIEROGLYPH-13731;Lo;0;L;;;;;N;;;;; +13732;EGYPTIAN HIEROGLYPH-13732;Lo;0;L;;;;;N;;;;; +13733;EGYPTIAN HIEROGLYPH-13733;Lo;0;L;;;;;N;;;;; +13734;EGYPTIAN HIEROGLYPH-13734;Lo;0;L;;;;;N;;;;; +13735;EGYPTIAN HIEROGLYPH-13735;Lo;0;L;;;;;N;;;;; +13736;EGYPTIAN HIEROGLYPH-13736;Lo;0;L;;;;;N;;;;; +13737;EGYPTIAN HIEROGLYPH-13737;Lo;0;L;;;;;N;;;;; +13738;EGYPTIAN HIEROGLYPH-13738;Lo;0;L;;;;;N;;;;; +13739;EGYPTIAN HIEROGLYPH-13739;Lo;0;L;;;;;N;;;;; +1373A;EGYPTIAN HIEROGLYPH-1373A;Lo;0;L;;;;;N;;;;; +1373B;EGYPTIAN HIEROGLYPH-1373B;Lo;0;L;;;;;N;;;;; +1373C;EGYPTIAN HIEROGLYPH-1373C;Lo;0;L;;;;;N;;;;; +1373D;EGYPTIAN HIEROGLYPH-1373D;Lo;0;L;;;;;N;;;;; +1373E;EGYPTIAN HIEROGLYPH-1373E;Lo;0;L;;;;;N;;;;; +1373F;EGYPTIAN HIEROGLYPH-1373F;Lo;0;L;;;;;N;;;;; +13740;EGYPTIAN HIEROGLYPH-13740;Lo;0;L;;;;;N;;;;; +13741;EGYPTIAN HIEROGLYPH-13741;Lo;0;L;;;;;N;;;;; +13742;EGYPTIAN HIEROGLYPH-13742;Lo;0;L;;;;;N;;;;; +13743;EGYPTIAN HIEROGLYPH-13743;Lo;0;L;;;;;N;;;;; +13744;EGYPTIAN HIEROGLYPH-13744;Lo;0;L;;;;;N;;;;; +13745;EGYPTIAN HIEROGLYPH-13745;Lo;0;L;;;;;N;;;;; +13746;EGYPTIAN HIEROGLYPH-13746;Lo;0;L;;;;;N;;;;; +13747;EGYPTIAN HIEROGLYPH-13747;Lo;0;L;;;;;N;;;;; +13748;EGYPTIAN HIEROGLYPH-13748;Lo;0;L;;;;;N;;;;; +13749;EGYPTIAN HIEROGLYPH-13749;Lo;0;L;;;;;N;;;;; +1374A;EGYPTIAN HIEROGLYPH-1374A;Lo;0;L;;;;;N;;;;; +1374B;EGYPTIAN HIEROGLYPH-1374B;Lo;0;L;;;;;N;;;;; +1374C;EGYPTIAN HIEROGLYPH-1374C;Lo;0;L;;;;;N;;;;; +1374D;EGYPTIAN HIEROGLYPH-1374D;Lo;0;L;;;;;N;;;;; +1374E;EGYPTIAN HIEROGLYPH-1374E;Lo;0;L;;;;;N;;;;; +1374F;EGYPTIAN HIEROGLYPH-1374F;Lo;0;L;;;;;N;;;;; +13750;EGYPTIAN HIEROGLYPH-13750;Lo;0;L;;;;;N;;;;; +13751;EGYPTIAN HIEROGLYPH-13751;Lo;0;L;;;;;N;;;;; +13752;EGYPTIAN HIEROGLYPH-13752;Lo;0;L;;;;;N;;;;; +13753;EGYPTIAN HIEROGLYPH-13753;Lo;0;L;;;;;N;;;;; +13754;EGYPTIAN HIEROGLYPH-13754;Lo;0;L;;;;;N;;;;; +13755;EGYPTIAN HIEROGLYPH-13755;Lo;0;L;;;;;N;;;;; +13756;EGYPTIAN HIEROGLYPH-13756;Lo;0;L;;;;;N;;;;; +13757;EGYPTIAN HIEROGLYPH-13757;Lo;0;L;;;;;N;;;;; +13758;EGYPTIAN HIEROGLYPH-13758;Lo;0;L;;;;;N;;;;; +13759;EGYPTIAN HIEROGLYPH-13759;Lo;0;L;;;;;N;;;;; +1375A;EGYPTIAN HIEROGLYPH-1375A;Lo;0;L;;;;;N;;;;; +1375B;EGYPTIAN HIEROGLYPH-1375B;Lo;0;L;;;;;N;;;;; +1375C;EGYPTIAN HIEROGLYPH-1375C;Lo;0;L;;;;;N;;;;; +1375D;EGYPTIAN HIEROGLYPH-1375D;Lo;0;L;;;;;N;;;;; +1375E;EGYPTIAN HIEROGLYPH-1375E;Lo;0;L;;;;;N;;;;; +1375F;EGYPTIAN HIEROGLYPH-1375F;Lo;0;L;;;;;N;;;;; +13760;EGYPTIAN HIEROGLYPH-13760;Lo;0;L;;;;;N;;;;; +13761;EGYPTIAN HIEROGLYPH-13761;Lo;0;L;;;;;N;;;;; +13762;EGYPTIAN HIEROGLYPH-13762;Lo;0;L;;;;;N;;;;; +13763;EGYPTIAN HIEROGLYPH-13763;Lo;0;L;;;;;N;;;;; +13764;EGYPTIAN HIEROGLYPH-13764;Lo;0;L;;;;;N;;;;; +13765;EGYPTIAN HIEROGLYPH-13765;Lo;0;L;;;;;N;;;;; +13766;EGYPTIAN HIEROGLYPH-13766;Lo;0;L;;;;;N;;;;; +13767;EGYPTIAN HIEROGLYPH-13767;Lo;0;L;;;;;N;;;;; +13768;EGYPTIAN HIEROGLYPH-13768;Lo;0;L;;;;;N;;;;; +13769;EGYPTIAN HIEROGLYPH-13769;Lo;0;L;;;;;N;;;;; +1376A;EGYPTIAN HIEROGLYPH-1376A;Lo;0;L;;;;;N;;;;; +1376B;EGYPTIAN HIEROGLYPH-1376B;Lo;0;L;;;;;N;;;;; +1376C;EGYPTIAN HIEROGLYPH-1376C;Lo;0;L;;;;;N;;;;; +1376D;EGYPTIAN HIEROGLYPH-1376D;Lo;0;L;;;;;N;;;;; +1376E;EGYPTIAN HIEROGLYPH-1376E;Lo;0;L;;;;;N;;;;; +1376F;EGYPTIAN HIEROGLYPH-1376F;Lo;0;L;;;;;N;;;;; +13770;EGYPTIAN HIEROGLYPH-13770;Lo;0;L;;;;;N;;;;; +13771;EGYPTIAN HIEROGLYPH-13771;Lo;0;L;;;;;N;;;;; +13772;EGYPTIAN HIEROGLYPH-13772;Lo;0;L;;;;;N;;;;; +13773;EGYPTIAN HIEROGLYPH-13773;Lo;0;L;;;;;N;;;;; +13774;EGYPTIAN HIEROGLYPH-13774;Lo;0;L;;;;;N;;;;; +13775;EGYPTIAN HIEROGLYPH-13775;Lo;0;L;;;;;N;;;;; +13776;EGYPTIAN HIEROGLYPH-13776;Lo;0;L;;;;;N;;;;; +13777;EGYPTIAN HIEROGLYPH-13777;Lo;0;L;;;;;N;;;;; +13778;EGYPTIAN HIEROGLYPH-13778;Lo;0;L;;;;;N;;;;; +13779;EGYPTIAN HIEROGLYPH-13779;Lo;0;L;;;;;N;;;;; +1377A;EGYPTIAN HIEROGLYPH-1377A;Lo;0;L;;;;;N;;;;; +1377B;EGYPTIAN HIEROGLYPH-1377B;Lo;0;L;;;;;N;;;;; +1377C;EGYPTIAN HIEROGLYPH-1377C;Lo;0;L;;;;;N;;;;; +1377D;EGYPTIAN HIEROGLYPH-1377D;Lo;0;L;;;;;N;;;;; +1377E;EGYPTIAN HIEROGLYPH-1377E;Lo;0;L;;;;;N;;;;; +1377F;EGYPTIAN HIEROGLYPH-1377F;Lo;0;L;;;;;N;;;;; +13780;EGYPTIAN HIEROGLYPH-13780;Lo;0;L;;;;;N;;;;; +13781;EGYPTIAN HIEROGLYPH-13781;Lo;0;L;;;;;N;;;;; +13782;EGYPTIAN HIEROGLYPH-13782;Lo;0;L;;;;;N;;;;; +13783;EGYPTIAN HIEROGLYPH-13783;Lo;0;L;;;;;N;;;;; +13784;EGYPTIAN HIEROGLYPH-13784;Lo;0;L;;;;;N;;;;; +13785;EGYPTIAN HIEROGLYPH-13785;Lo;0;L;;;;;N;;;;; +13786;EGYPTIAN HIEROGLYPH-13786;Lo;0;L;;;;;N;;;;; +13787;EGYPTIAN HIEROGLYPH-13787;Lo;0;L;;;;;N;;;;; +13788;EGYPTIAN HIEROGLYPH-13788;Lo;0;L;;;;;N;;;;; +13789;EGYPTIAN HIEROGLYPH-13789;Lo;0;L;;;;;N;;;;; +1378A;EGYPTIAN HIEROGLYPH-1378A;Lo;0;L;;;;;N;;;;; +1378B;EGYPTIAN HIEROGLYPH-1378B;Lo;0;L;;;;;N;;;;; +1378C;EGYPTIAN HIEROGLYPH-1378C;Lo;0;L;;;;;N;;;;; +1378D;EGYPTIAN HIEROGLYPH-1378D;Lo;0;L;;;;;N;;;;; +1378E;EGYPTIAN HIEROGLYPH-1378E;Lo;0;L;;;;;N;;;;; +1378F;EGYPTIAN HIEROGLYPH-1378F;Lo;0;L;;;;;N;;;;; +13790;EGYPTIAN HIEROGLYPH-13790;Lo;0;L;;;;;N;;;;; +13791;EGYPTIAN HIEROGLYPH-13791;Lo;0;L;;;;;N;;;;; +13792;EGYPTIAN HIEROGLYPH-13792;Lo;0;L;;;;;N;;;;; +13793;EGYPTIAN HIEROGLYPH-13793;Lo;0;L;;;;;N;;;;; +13794;EGYPTIAN HIEROGLYPH-13794;Lo;0;L;;;;;N;;;;; +13795;EGYPTIAN HIEROGLYPH-13795;Lo;0;L;;;;;N;;;;; +13796;EGYPTIAN HIEROGLYPH-13796;Lo;0;L;;;;;N;;;;; +13797;EGYPTIAN HIEROGLYPH-13797;Lo;0;L;;;;;N;;;;; +13798;EGYPTIAN HIEROGLYPH-13798;Lo;0;L;;;;;N;;;;; +13799;EGYPTIAN HIEROGLYPH-13799;Lo;0;L;;;;;N;;;;; +1379A;EGYPTIAN HIEROGLYPH-1379A;Lo;0;L;;;;;N;;;;; +1379B;EGYPTIAN HIEROGLYPH-1379B;Lo;0;L;;;;;N;;;;; +1379C;EGYPTIAN HIEROGLYPH-1379C;Lo;0;L;;;;;N;;;;; +1379D;EGYPTIAN HIEROGLYPH-1379D;Lo;0;L;;;;;N;;;;; +1379E;EGYPTIAN HIEROGLYPH-1379E;Lo;0;L;;;;;N;;;;; +1379F;EGYPTIAN HIEROGLYPH-1379F;Lo;0;L;;;;;N;;;;; +137A0;EGYPTIAN HIEROGLYPH-137A0;Lo;0;L;;;;;N;;;;; +137A1;EGYPTIAN HIEROGLYPH-137A1;Lo;0;L;;;;;N;;;;; +137A2;EGYPTIAN HIEROGLYPH-137A2;Lo;0;L;;;;;N;;;;; +137A3;EGYPTIAN HIEROGLYPH-137A3;Lo;0;L;;;;;N;;;;; +137A4;EGYPTIAN HIEROGLYPH-137A4;Lo;0;L;;;;;N;;;;; +137A5;EGYPTIAN HIEROGLYPH-137A5;Lo;0;L;;;;;N;;;;; +137A6;EGYPTIAN HIEROGLYPH-137A6;Lo;0;L;;;;;N;;;;; +137A7;EGYPTIAN HIEROGLYPH-137A7;Lo;0;L;;;;;N;;;;; +137A8;EGYPTIAN HIEROGLYPH-137A8;Lo;0;L;;;;;N;;;;; +137A9;EGYPTIAN HIEROGLYPH-137A9;Lo;0;L;;;;;N;;;;; +137AA;EGYPTIAN HIEROGLYPH-137AA;Lo;0;L;;;;;N;;;;; +137AB;EGYPTIAN HIEROGLYPH-137AB;Lo;0;L;;;;;N;;;;; +137AC;EGYPTIAN HIEROGLYPH-137AC;Lo;0;L;;;;;N;;;;; +137AD;EGYPTIAN HIEROGLYPH-137AD;Lo;0;L;;;;;N;;;;; +137AE;EGYPTIAN HIEROGLYPH-137AE;Lo;0;L;;;;;N;;;;; +137AF;EGYPTIAN HIEROGLYPH-137AF;Lo;0;L;;;;;N;;;;; +137B0;EGYPTIAN HIEROGLYPH-137B0;Lo;0;L;;;;;N;;;;; +137B1;EGYPTIAN HIEROGLYPH-137B1;Lo;0;L;;;;;N;;;;; +137B2;EGYPTIAN HIEROGLYPH-137B2;Lo;0;L;;;;;N;;;;; +137B3;EGYPTIAN HIEROGLYPH-137B3;Lo;0;L;;;;;N;;;;; +137B4;EGYPTIAN HIEROGLYPH-137B4;Lo;0;L;;;;;N;;;;; +137B5;EGYPTIAN HIEROGLYPH-137B5;Lo;0;L;;;;;N;;;;; +137B6;EGYPTIAN HIEROGLYPH-137B6;Lo;0;L;;;;;N;;;;; +137B7;EGYPTIAN HIEROGLYPH-137B7;Lo;0;L;;;;;N;;;;; +137B8;EGYPTIAN HIEROGLYPH-137B8;Lo;0;L;;;;;N;;;;; +137B9;EGYPTIAN HIEROGLYPH-137B9;Lo;0;L;;;;;N;;;;; +137BA;EGYPTIAN HIEROGLYPH-137BA;Lo;0;L;;;;;N;;;;; +137BB;EGYPTIAN HIEROGLYPH-137BB;Lo;0;L;;;;;N;;;;; +137BC;EGYPTIAN HIEROGLYPH-137BC;Lo;0;L;;;;;N;;;;; +137BD;EGYPTIAN HIEROGLYPH-137BD;Lo;0;L;;;;;N;;;;; +137BE;EGYPTIAN HIEROGLYPH-137BE;Lo;0;L;;;;;N;;;;; +137BF;EGYPTIAN HIEROGLYPH-137BF;Lo;0;L;;;;;N;;;;; +137C0;EGYPTIAN HIEROGLYPH-137C0;Lo;0;L;;;;;N;;;;; +137C1;EGYPTIAN HIEROGLYPH-137C1;Lo;0;L;;;;;N;;;;; +137C2;EGYPTIAN HIEROGLYPH-137C2;Lo;0;L;;;;;N;;;;; +137C3;EGYPTIAN HIEROGLYPH-137C3;Lo;0;L;;;;;N;;;;; +137C4;EGYPTIAN HIEROGLYPH-137C4;Lo;0;L;;;;;N;;;;; +137C5;EGYPTIAN HIEROGLYPH-137C5;Lo;0;L;;;;;N;;;;; +137C6;EGYPTIAN HIEROGLYPH-137C6;Lo;0;L;;;;;N;;;;; +137C7;EGYPTIAN HIEROGLYPH-137C7;Lo;0;L;;;;;N;;;;; +137C8;EGYPTIAN HIEROGLYPH-137C8;Lo;0;L;;;;;N;;;;; +137C9;EGYPTIAN HIEROGLYPH-137C9;Lo;0;L;;;;;N;;;;; +137CA;EGYPTIAN HIEROGLYPH-137CA;Lo;0;L;;;;;N;;;;; +137CB;EGYPTIAN HIEROGLYPH-137CB;Lo;0;L;;;;;N;;;;; +137CC;EGYPTIAN HIEROGLYPH-137CC;Lo;0;L;;;;;N;;;;; +137CD;EGYPTIAN HIEROGLYPH-137CD;Lo;0;L;;;;;N;;;;; +137CE;EGYPTIAN HIEROGLYPH-137CE;Lo;0;L;;;;;N;;;;; +137CF;EGYPTIAN HIEROGLYPH-137CF;Lo;0;L;;;;;N;;;;; +137D0;EGYPTIAN HIEROGLYPH-137D0;Lo;0;L;;;;;N;;;;; +137D1;EGYPTIAN HIEROGLYPH-137D1;Lo;0;L;;;;;N;;;;; +137D2;EGYPTIAN HIEROGLYPH-137D2;Lo;0;L;;;;;N;;;;; +137D3;EGYPTIAN HIEROGLYPH-137D3;Lo;0;L;;;;;N;;;;; +137D4;EGYPTIAN HIEROGLYPH-137D4;Lo;0;L;;;;;N;;;;; +137D5;EGYPTIAN HIEROGLYPH-137D5;Lo;0;L;;;;;N;;;;; +137D6;EGYPTIAN HIEROGLYPH-137D6;Lo;0;L;;;;;N;;;;; +137D7;EGYPTIAN HIEROGLYPH-137D7;Lo;0;L;;;;;N;;;;; +137D8;EGYPTIAN HIEROGLYPH-137D8;Lo;0;L;;;;;N;;;;; +137D9;EGYPTIAN HIEROGLYPH-137D9;Lo;0;L;;;;;N;;;;; +137DA;EGYPTIAN HIEROGLYPH-137DA;Lo;0;L;;;;;N;;;;; +137DB;EGYPTIAN HIEROGLYPH-137DB;Lo;0;L;;;;;N;;;;; +137DC;EGYPTIAN HIEROGLYPH-137DC;Lo;0;L;;;;;N;;;;; +137DD;EGYPTIAN HIEROGLYPH-137DD;Lo;0;L;;;;;N;;;;; +137DE;EGYPTIAN HIEROGLYPH-137DE;Lo;0;L;;;;;N;;;;; +137DF;EGYPTIAN HIEROGLYPH-137DF;Lo;0;L;;;;;N;;;;; +137E0;EGYPTIAN HIEROGLYPH-137E0;Lo;0;L;;;;;N;;;;; +137E1;EGYPTIAN HIEROGLYPH-137E1;Lo;0;L;;;;;N;;;;; +137E2;EGYPTIAN HIEROGLYPH-137E2;Lo;0;L;;;;;N;;;;; +137E3;EGYPTIAN HIEROGLYPH-137E3;Lo;0;L;;;;;N;;;;; +137E4;EGYPTIAN HIEROGLYPH-137E4;Lo;0;L;;;;;N;;;;; +137E5;EGYPTIAN HIEROGLYPH-137E5;Lo;0;L;;;;;N;;;;; +137E6;EGYPTIAN HIEROGLYPH-137E6;Lo;0;L;;;;;N;;;;; +137E7;EGYPTIAN HIEROGLYPH-137E7;Lo;0;L;;;;;N;;;;; +137E8;EGYPTIAN HIEROGLYPH-137E8;Lo;0;L;;;;;N;;;;; +137E9;EGYPTIAN HIEROGLYPH-137E9;Lo;0;L;;;;;N;;;;; +137EA;EGYPTIAN HIEROGLYPH-137EA;Lo;0;L;;;;;N;;;;; +137EB;EGYPTIAN HIEROGLYPH-137EB;Lo;0;L;;;;;N;;;;; +137EC;EGYPTIAN HIEROGLYPH-137EC;Lo;0;L;;;;;N;;;;; +137ED;EGYPTIAN HIEROGLYPH-137ED;Lo;0;L;;;;;N;;;;; +137EE;EGYPTIAN HIEROGLYPH-137EE;Lo;0;L;;;;;N;;;;; +137EF;EGYPTIAN HIEROGLYPH-137EF;Lo;0;L;;;;;N;;;;; +137F0;EGYPTIAN HIEROGLYPH-137F0;Lo;0;L;;;;;N;;;;; +137F1;EGYPTIAN HIEROGLYPH-137F1;Lo;0;L;;;;;N;;;;; +137F2;EGYPTIAN HIEROGLYPH-137F2;Lo;0;L;;;;;N;;;;; +137F3;EGYPTIAN HIEROGLYPH-137F3;Lo;0;L;;;;;N;;;;; +137F4;EGYPTIAN HIEROGLYPH-137F4;Lo;0;L;;;;;N;;;;; +137F5;EGYPTIAN HIEROGLYPH-137F5;Lo;0;L;;;;;N;;;;; +137F6;EGYPTIAN HIEROGLYPH-137F6;Lo;0;L;;;;;N;;;;; +137F7;EGYPTIAN HIEROGLYPH-137F7;Lo;0;L;;;;;N;;;;; +137F8;EGYPTIAN HIEROGLYPH-137F8;Lo;0;L;;;;;N;;;;; +137F9;EGYPTIAN HIEROGLYPH-137F9;Lo;0;L;;;;;N;;;;; +137FA;EGYPTIAN HIEROGLYPH-137FA;Lo;0;L;;;;;N;;;;; +137FB;EGYPTIAN HIEROGLYPH-137FB;Lo;0;L;;;;;N;;;;; +137FC;EGYPTIAN HIEROGLYPH-137FC;Lo;0;L;;;;;N;;;;; +137FD;EGYPTIAN HIEROGLYPH-137FD;Lo;0;L;;;;;N;;;;; +137FE;EGYPTIAN HIEROGLYPH-137FE;Lo;0;L;;;;;N;;;;; +137FF;EGYPTIAN HIEROGLYPH-137FF;Lo;0;L;;;;;N;;;;; +13800;EGYPTIAN HIEROGLYPH-13800;Lo;0;L;;;;;N;;;;; +13801;EGYPTIAN HIEROGLYPH-13801;Lo;0;L;;;;;N;;;;; +13802;EGYPTIAN HIEROGLYPH-13802;Lo;0;L;;;;;N;;;;; +13803;EGYPTIAN HIEROGLYPH-13803;Lo;0;L;;;;;N;;;;; +13804;EGYPTIAN HIEROGLYPH-13804;Lo;0;L;;;;;N;;;;; +13805;EGYPTIAN HIEROGLYPH-13805;Lo;0;L;;;;;N;;;;; +13806;EGYPTIAN HIEROGLYPH-13806;Lo;0;L;;;;;N;;;;; +13807;EGYPTIAN HIEROGLYPH-13807;Lo;0;L;;;;;N;;;;; +13808;EGYPTIAN HIEROGLYPH-13808;Lo;0;L;;;;;N;;;;; +13809;EGYPTIAN HIEROGLYPH-13809;Lo;0;L;;;;;N;;;;; +1380A;EGYPTIAN HIEROGLYPH-1380A;Lo;0;L;;;;;N;;;;; +1380B;EGYPTIAN HIEROGLYPH-1380B;Lo;0;L;;;;;N;;;;; +1380C;EGYPTIAN HIEROGLYPH-1380C;Lo;0;L;;;;;N;;;;; +1380D;EGYPTIAN HIEROGLYPH-1380D;Lo;0;L;;;;;N;;;;; +1380E;EGYPTIAN HIEROGLYPH-1380E;Lo;0;L;;;;;N;;;;; +1380F;EGYPTIAN HIEROGLYPH-1380F;Lo;0;L;;;;;N;;;;; +13810;EGYPTIAN HIEROGLYPH-13810;Lo;0;L;;;;;N;;;;; +13811;EGYPTIAN HIEROGLYPH-13811;Lo;0;L;;;;;N;;;;; +13812;EGYPTIAN HIEROGLYPH-13812;Lo;0;L;;;;;N;;;;; +13813;EGYPTIAN HIEROGLYPH-13813;Lo;0;L;;;;;N;;;;; +13814;EGYPTIAN HIEROGLYPH-13814;Lo;0;L;;;;;N;;;;; +13815;EGYPTIAN HIEROGLYPH-13815;Lo;0;L;;;;;N;;;;; +13816;EGYPTIAN HIEROGLYPH-13816;Lo;0;L;;;;;N;;;;; +13817;EGYPTIAN HIEROGLYPH-13817;Lo;0;L;;;;;N;;;;; +13818;EGYPTIAN HIEROGLYPH-13818;Lo;0;L;;;;;N;;;;; +13819;EGYPTIAN HIEROGLYPH-13819;Lo;0;L;;;;;N;;;;; +1381A;EGYPTIAN HIEROGLYPH-1381A;Lo;0;L;;;;;N;;;;; +1381B;EGYPTIAN HIEROGLYPH-1381B;Lo;0;L;;;;;N;;;;; +1381C;EGYPTIAN HIEROGLYPH-1381C;Lo;0;L;;;;;N;;;;; +1381D;EGYPTIAN HIEROGLYPH-1381D;Lo;0;L;;;;;N;;;;; +1381E;EGYPTIAN HIEROGLYPH-1381E;Lo;0;L;;;;;N;;;;; +1381F;EGYPTIAN HIEROGLYPH-1381F;Lo;0;L;;;;;N;;;;; +13820;EGYPTIAN HIEROGLYPH-13820;Lo;0;L;;;;;N;;;;; +13821;EGYPTIAN HIEROGLYPH-13821;Lo;0;L;;;;;N;;;;; +13822;EGYPTIAN HIEROGLYPH-13822;Lo;0;L;;;;;N;;;;; +13823;EGYPTIAN HIEROGLYPH-13823;Lo;0;L;;;;;N;;;;; +13824;EGYPTIAN HIEROGLYPH-13824;Lo;0;L;;;;;N;;;;; +13825;EGYPTIAN HIEROGLYPH-13825;Lo;0;L;;;;;N;;;;; +13826;EGYPTIAN HIEROGLYPH-13826;Lo;0;L;;;;;N;;;;; +13827;EGYPTIAN HIEROGLYPH-13827;Lo;0;L;;;;;N;;;;; +13828;EGYPTIAN HIEROGLYPH-13828;Lo;0;L;;;;;N;;;;; +13829;EGYPTIAN HIEROGLYPH-13829;Lo;0;L;;;;;N;;;;; +1382A;EGYPTIAN HIEROGLYPH-1382A;Lo;0;L;;;;;N;;;;; +1382B;EGYPTIAN HIEROGLYPH-1382B;Lo;0;L;;;;;N;;;;; +1382C;EGYPTIAN HIEROGLYPH-1382C;Lo;0;L;;;;;N;;;;; +1382D;EGYPTIAN HIEROGLYPH-1382D;Lo;0;L;;;;;N;;;;; +1382E;EGYPTIAN HIEROGLYPH-1382E;Lo;0;L;;;;;N;;;;; +1382F;EGYPTIAN HIEROGLYPH-1382F;Lo;0;L;;;;;N;;;;; +13830;EGYPTIAN HIEROGLYPH-13830;Lo;0;L;;;;;N;;;;; +13831;EGYPTIAN HIEROGLYPH-13831;Lo;0;L;;;;;N;;;;; +13832;EGYPTIAN HIEROGLYPH-13832;Lo;0;L;;;;;N;;;;; +13833;EGYPTIAN HIEROGLYPH-13833;Lo;0;L;;;;;N;;;;; +13834;EGYPTIAN HIEROGLYPH-13834;Lo;0;L;;;;;N;;;;; +13835;EGYPTIAN HIEROGLYPH-13835;Lo;0;L;;;;;N;;;;; +13836;EGYPTIAN HIEROGLYPH-13836;Lo;0;L;;;;;N;;;;; +13837;EGYPTIAN HIEROGLYPH-13837;Lo;0;L;;;;;N;;;;; +13838;EGYPTIAN HIEROGLYPH-13838;Lo;0;L;;;;;N;;;;; +13839;EGYPTIAN HIEROGLYPH-13839;Lo;0;L;;;;;N;;;;; +1383A;EGYPTIAN HIEROGLYPH-1383A;Lo;0;L;;;;;N;;;;; +1383B;EGYPTIAN HIEROGLYPH-1383B;Lo;0;L;;;;;N;;;;; +1383C;EGYPTIAN HIEROGLYPH-1383C;Lo;0;L;;;;;N;;;;; +1383D;EGYPTIAN HIEROGLYPH-1383D;Lo;0;L;;;;;N;;;;; +1383E;EGYPTIAN HIEROGLYPH-1383E;Lo;0;L;;;;;N;;;;; +1383F;EGYPTIAN HIEROGLYPH-1383F;Lo;0;L;;;;;N;;;;; +13840;EGYPTIAN HIEROGLYPH-13840;Lo;0;L;;;;;N;;;;; +13841;EGYPTIAN HIEROGLYPH-13841;Lo;0;L;;;;;N;;;;; +13842;EGYPTIAN HIEROGLYPH-13842;Lo;0;L;;;;;N;;;;; +13843;EGYPTIAN HIEROGLYPH-13843;Lo;0;L;;;;;N;;;;; +13844;EGYPTIAN HIEROGLYPH-13844;Lo;0;L;;;;;N;;;;; +13845;EGYPTIAN HIEROGLYPH-13845;Lo;0;L;;;;;N;;;;; +13846;EGYPTIAN HIEROGLYPH-13846;Lo;0;L;;;;;N;;;;; +13847;EGYPTIAN HIEROGLYPH-13847;Lo;0;L;;;;;N;;;;; +13848;EGYPTIAN HIEROGLYPH-13848;Lo;0;L;;;;;N;;;;; +13849;EGYPTIAN HIEROGLYPH-13849;Lo;0;L;;;;;N;;;;; +1384A;EGYPTIAN HIEROGLYPH-1384A;Lo;0;L;;;;;N;;;;; +1384B;EGYPTIAN HIEROGLYPH-1384B;Lo;0;L;;;;;N;;;;; +1384C;EGYPTIAN HIEROGLYPH-1384C;Lo;0;L;;;;;N;;;;; +1384D;EGYPTIAN HIEROGLYPH-1384D;Lo;0;L;;;;;N;;;;; +1384E;EGYPTIAN HIEROGLYPH-1384E;Lo;0;L;;;;;N;;;;; +1384F;EGYPTIAN HIEROGLYPH-1384F;Lo;0;L;;;;;N;;;;; +13850;EGYPTIAN HIEROGLYPH-13850;Lo;0;L;;;;;N;;;;; +13851;EGYPTIAN HIEROGLYPH-13851;Lo;0;L;;;;;N;;;;; +13852;EGYPTIAN HIEROGLYPH-13852;Lo;0;L;;;;;N;;;;; +13853;EGYPTIAN HIEROGLYPH-13853;Lo;0;L;;;;;N;;;;; +13854;EGYPTIAN HIEROGLYPH-13854;Lo;0;L;;;;;N;;;;; +13855;EGYPTIAN HIEROGLYPH-13855;Lo;0;L;;;;;N;;;;; +13856;EGYPTIAN HIEROGLYPH-13856;Lo;0;L;;;;;N;;;;; +13857;EGYPTIAN HIEROGLYPH-13857;Lo;0;L;;;;;N;;;;; +13858;EGYPTIAN HIEROGLYPH-13858;Lo;0;L;;;;;N;;;;; +13859;EGYPTIAN HIEROGLYPH-13859;Lo;0;L;;;;;N;;;;; +1385A;EGYPTIAN HIEROGLYPH-1385A;Lo;0;L;;;;;N;;;;; +1385B;EGYPTIAN HIEROGLYPH-1385B;Lo;0;L;;;;;N;;;;; +1385C;EGYPTIAN HIEROGLYPH-1385C;Lo;0;L;;;;;N;;;;; +1385D;EGYPTIAN HIEROGLYPH-1385D;Lo;0;L;;;;;N;;;;; +1385E;EGYPTIAN HIEROGLYPH-1385E;Lo;0;L;;;;;N;;;;; +1385F;EGYPTIAN HIEROGLYPH-1385F;Lo;0;L;;;;;N;;;;; +13860;EGYPTIAN HIEROGLYPH-13860;Lo;0;L;;;;;N;;;;; +13861;EGYPTIAN HIEROGLYPH-13861;Lo;0;L;;;;;N;;;;; +13862;EGYPTIAN HIEROGLYPH-13862;Lo;0;L;;;;;N;;;;; +13863;EGYPTIAN HIEROGLYPH-13863;Lo;0;L;;;;;N;;;;; +13864;EGYPTIAN HIEROGLYPH-13864;Lo;0;L;;;;;N;;;;; +13865;EGYPTIAN HIEROGLYPH-13865;Lo;0;L;;;;;N;;;;; +13866;EGYPTIAN HIEROGLYPH-13866;Lo;0;L;;;;;N;;;;; +13867;EGYPTIAN HIEROGLYPH-13867;Lo;0;L;;;;;N;;;;; +13868;EGYPTIAN HIEROGLYPH-13868;Lo;0;L;;;;;N;;;;; +13869;EGYPTIAN HIEROGLYPH-13869;Lo;0;L;;;;;N;;;;; +1386A;EGYPTIAN HIEROGLYPH-1386A;Lo;0;L;;;;;N;;;;; +1386B;EGYPTIAN HIEROGLYPH-1386B;Lo;0;L;;;;;N;;;;; +1386C;EGYPTIAN HIEROGLYPH-1386C;Lo;0;L;;;;;N;;;;; +1386D;EGYPTIAN HIEROGLYPH-1386D;Lo;0;L;;;;;N;;;;; +1386E;EGYPTIAN HIEROGLYPH-1386E;Lo;0;L;;;;;N;;;;; +1386F;EGYPTIAN HIEROGLYPH-1386F;Lo;0;L;;;;;N;;;;; +13870;EGYPTIAN HIEROGLYPH-13870;Lo;0;L;;;;;N;;;;; +13871;EGYPTIAN HIEROGLYPH-13871;Lo;0;L;;;;;N;;;;; +13872;EGYPTIAN HIEROGLYPH-13872;Lo;0;L;;;;;N;;;;; +13873;EGYPTIAN HIEROGLYPH-13873;Lo;0;L;;;;;N;;;;; +13874;EGYPTIAN HIEROGLYPH-13874;Lo;0;L;;;;;N;;;;; +13875;EGYPTIAN HIEROGLYPH-13875;Lo;0;L;;;;;N;;;;; +13876;EGYPTIAN HIEROGLYPH-13876;Lo;0;L;;;;;N;;;;; +13877;EGYPTIAN HIEROGLYPH-13877;Lo;0;L;;;;;N;;;;; +13878;EGYPTIAN HIEROGLYPH-13878;Lo;0;L;;;;;N;;;;; +13879;EGYPTIAN HIEROGLYPH-13879;Lo;0;L;;;;;N;;;;; +1387A;EGYPTIAN HIEROGLYPH-1387A;Lo;0;L;;;;;N;;;;; +1387B;EGYPTIAN HIEROGLYPH-1387B;Lo;0;L;;;;;N;;;;; +1387C;EGYPTIAN HIEROGLYPH-1387C;Lo;0;L;;;;;N;;;;; +1387D;EGYPTIAN HIEROGLYPH-1387D;Lo;0;L;;;;;N;;;;; +1387E;EGYPTIAN HIEROGLYPH-1387E;Lo;0;L;;;;;N;;;;; +1387F;EGYPTIAN HIEROGLYPH-1387F;Lo;0;L;;;;;N;;;;; +13880;EGYPTIAN HIEROGLYPH-13880;Lo;0;L;;;;;N;;;;; +13881;EGYPTIAN HIEROGLYPH-13881;Lo;0;L;;;;;N;;;;; +13882;EGYPTIAN HIEROGLYPH-13882;Lo;0;L;;;;;N;;;;; +13883;EGYPTIAN HIEROGLYPH-13883;Lo;0;L;;;;;N;;;;; +13884;EGYPTIAN HIEROGLYPH-13884;Lo;0;L;;;;;N;;;;; +13885;EGYPTIAN HIEROGLYPH-13885;Lo;0;L;;;;;N;;;;; +13886;EGYPTIAN HIEROGLYPH-13886;Lo;0;L;;;;;N;;;;; +13887;EGYPTIAN HIEROGLYPH-13887;Lo;0;L;;;;;N;;;;; +13888;EGYPTIAN HIEROGLYPH-13888;Lo;0;L;;;;;N;;;;; +13889;EGYPTIAN HIEROGLYPH-13889;Lo;0;L;;;;;N;;;;; +1388A;EGYPTIAN HIEROGLYPH-1388A;Lo;0;L;;;;;N;;;;; +1388B;EGYPTIAN HIEROGLYPH-1388B;Lo;0;L;;;;;N;;;;; +1388C;EGYPTIAN HIEROGLYPH-1388C;Lo;0;L;;;;;N;;;;; +1388D;EGYPTIAN HIEROGLYPH-1388D;Lo;0;L;;;;;N;;;;; +1388E;EGYPTIAN HIEROGLYPH-1388E;Lo;0;L;;;;;N;;;;; +1388F;EGYPTIAN HIEROGLYPH-1388F;Lo;0;L;;;;;N;;;;; +13890;EGYPTIAN HIEROGLYPH-13890;Lo;0;L;;;;;N;;;;; +13891;EGYPTIAN HIEROGLYPH-13891;Lo;0;L;;;;;N;;;;; +13892;EGYPTIAN HIEROGLYPH-13892;Lo;0;L;;;;;N;;;;; +13893;EGYPTIAN HIEROGLYPH-13893;Lo;0;L;;;;;N;;;;; +13894;EGYPTIAN HIEROGLYPH-13894;Lo;0;L;;;;;N;;;;; +13895;EGYPTIAN HIEROGLYPH-13895;Lo;0;L;;;;;N;;;;; +13896;EGYPTIAN HIEROGLYPH-13896;Lo;0;L;;;;;N;;;;; +13897;EGYPTIAN HIEROGLYPH-13897;Lo;0;L;;;;;N;;;;; +13898;EGYPTIAN HIEROGLYPH-13898;Lo;0;L;;;;;N;;;;; +13899;EGYPTIAN HIEROGLYPH-13899;Lo;0;L;;;;;N;;;;; +1389A;EGYPTIAN HIEROGLYPH-1389A;Lo;0;L;;;;;N;;;;; +1389B;EGYPTIAN HIEROGLYPH-1389B;Lo;0;L;;;;;N;;;;; +1389C;EGYPTIAN HIEROGLYPH-1389C;Lo;0;L;;;;;N;;;;; +1389D;EGYPTIAN HIEROGLYPH-1389D;Lo;0;L;;;;;N;;;;; +1389E;EGYPTIAN HIEROGLYPH-1389E;Lo;0;L;;;;;N;;;;; +1389F;EGYPTIAN HIEROGLYPH-1389F;Lo;0;L;;;;;N;;;;; +138A0;EGYPTIAN HIEROGLYPH-138A0;Lo;0;L;;;;;N;;;;; +138A1;EGYPTIAN HIEROGLYPH-138A1;Lo;0;L;;;;;N;;;;; +138A2;EGYPTIAN HIEROGLYPH-138A2;Lo;0;L;;;;;N;;;;; +138A3;EGYPTIAN HIEROGLYPH-138A3;Lo;0;L;;;;;N;;;;; +138A4;EGYPTIAN HIEROGLYPH-138A4;Lo;0;L;;;;;N;;;;; +138A5;EGYPTIAN HIEROGLYPH-138A5;Lo;0;L;;;;;N;;;;; +138A6;EGYPTIAN HIEROGLYPH-138A6;Lo;0;L;;;;;N;;;;; +138A7;EGYPTIAN HIEROGLYPH-138A7;Lo;0;L;;;;;N;;;;; +138A8;EGYPTIAN HIEROGLYPH-138A8;Lo;0;L;;;;;N;;;;; +138A9;EGYPTIAN HIEROGLYPH-138A9;Lo;0;L;;;;;N;;;;; +138AA;EGYPTIAN HIEROGLYPH-138AA;Lo;0;L;;;;;N;;;;; +138AB;EGYPTIAN HIEROGLYPH-138AB;Lo;0;L;;;;;N;;;;; +138AC;EGYPTIAN HIEROGLYPH-138AC;Lo;0;L;;;;;N;;;;; +138AD;EGYPTIAN HIEROGLYPH-138AD;Lo;0;L;;;;;N;;;;; +138AE;EGYPTIAN HIEROGLYPH-138AE;Lo;0;L;;;;;N;;;;; +138AF;EGYPTIAN HIEROGLYPH-138AF;Lo;0;L;;;;;N;;;;; +138B0;EGYPTIAN HIEROGLYPH-138B0;Lo;0;L;;;;;N;;;;; +138B1;EGYPTIAN HIEROGLYPH-138B1;Lo;0;L;;;;;N;;;;; +138B2;EGYPTIAN HIEROGLYPH-138B2;Lo;0;L;;;;;N;;;;; +138B3;EGYPTIAN HIEROGLYPH-138B3;Lo;0;L;;;;;N;;;;; +138B4;EGYPTIAN HIEROGLYPH-138B4;Lo;0;L;;;;;N;;;;; +138B5;EGYPTIAN HIEROGLYPH-138B5;Lo;0;L;;;;;N;;;;; +138B6;EGYPTIAN HIEROGLYPH-138B6;Lo;0;L;;;;;N;;;;; +138B7;EGYPTIAN HIEROGLYPH-138B7;Lo;0;L;;;;;N;;;;; +138B8;EGYPTIAN HIEROGLYPH-138B8;Lo;0;L;;;;;N;;;;; +138B9;EGYPTIAN HIEROGLYPH-138B9;Lo;0;L;;;;;N;;;;; +138BA;EGYPTIAN HIEROGLYPH-138BA;Lo;0;L;;;;;N;;;;; +138BB;EGYPTIAN HIEROGLYPH-138BB;Lo;0;L;;;;;N;;;;; +138BC;EGYPTIAN HIEROGLYPH-138BC;Lo;0;L;;;;;N;;;;; +138BD;EGYPTIAN HIEROGLYPH-138BD;Lo;0;L;;;;;N;;;;; +138BE;EGYPTIAN HIEROGLYPH-138BE;Lo;0;L;;;;;N;;;;; +138BF;EGYPTIAN HIEROGLYPH-138BF;Lo;0;L;;;;;N;;;;; +138C0;EGYPTIAN HIEROGLYPH-138C0;Lo;0;L;;;;;N;;;;; +138C1;EGYPTIAN HIEROGLYPH-138C1;Lo;0;L;;;;;N;;;;; +138C2;EGYPTIAN HIEROGLYPH-138C2;Lo;0;L;;;;;N;;;;; +138C3;EGYPTIAN HIEROGLYPH-138C3;Lo;0;L;;;;;N;;;;; +138C4;EGYPTIAN HIEROGLYPH-138C4;Lo;0;L;;;;;N;;;;; +138C5;EGYPTIAN HIEROGLYPH-138C5;Lo;0;L;;;;;N;;;;; +138C6;EGYPTIAN HIEROGLYPH-138C6;Lo;0;L;;;;;N;;;;; +138C7;EGYPTIAN HIEROGLYPH-138C7;Lo;0;L;;;;;N;;;;; +138C8;EGYPTIAN HIEROGLYPH-138C8;Lo;0;L;;;;;N;;;;; +138C9;EGYPTIAN HIEROGLYPH-138C9;Lo;0;L;;;;;N;;;;; +138CA;EGYPTIAN HIEROGLYPH-138CA;Lo;0;L;;;;;N;;;;; +138CB;EGYPTIAN HIEROGLYPH-138CB;Lo;0;L;;;;;N;;;;; +138CC;EGYPTIAN HIEROGLYPH-138CC;Lo;0;L;;;;;N;;;;; +138CD;EGYPTIAN HIEROGLYPH-138CD;Lo;0;L;;;;;N;;;;; +138CE;EGYPTIAN HIEROGLYPH-138CE;Lo;0;L;;;;;N;;;;; +138CF;EGYPTIAN HIEROGLYPH-138CF;Lo;0;L;;;;;N;;;;; +138D0;EGYPTIAN HIEROGLYPH-138D0;Lo;0;L;;;;;N;;;;; +138D1;EGYPTIAN HIEROGLYPH-138D1;Lo;0;L;;;;;N;;;;; +138D2;EGYPTIAN HIEROGLYPH-138D2;Lo;0;L;;;;;N;;;;; +138D3;EGYPTIAN HIEROGLYPH-138D3;Lo;0;L;;;;;N;;;;; +138D4;EGYPTIAN HIEROGLYPH-138D4;Lo;0;L;;;;;N;;;;; +138D5;EGYPTIAN HIEROGLYPH-138D5;Lo;0;L;;;;;N;;;;; +138D6;EGYPTIAN HIEROGLYPH-138D6;Lo;0;L;;;;;N;;;;; +138D7;EGYPTIAN HIEROGLYPH-138D7;Lo;0;L;;;;;N;;;;; +138D8;EGYPTIAN HIEROGLYPH-138D8;Lo;0;L;;;;;N;;;;; +138D9;EGYPTIAN HIEROGLYPH-138D9;Lo;0;L;;;;;N;;;;; +138DA;EGYPTIAN HIEROGLYPH-138DA;Lo;0;L;;;;;N;;;;; +138DB;EGYPTIAN HIEROGLYPH-138DB;Lo;0;L;;;;;N;;;;; +138DC;EGYPTIAN HIEROGLYPH-138DC;Lo;0;L;;;;;N;;;;; +138DD;EGYPTIAN HIEROGLYPH-138DD;Lo;0;L;;;;;N;;;;; +138DE;EGYPTIAN HIEROGLYPH-138DE;Lo;0;L;;;;;N;;;;; +138DF;EGYPTIAN HIEROGLYPH-138DF;Lo;0;L;;;;;N;;;;; +138E0;EGYPTIAN HIEROGLYPH-138E0;Lo;0;L;;;;;N;;;;; +138E1;EGYPTIAN HIEROGLYPH-138E1;Lo;0;L;;;;;N;;;;; +138E2;EGYPTIAN HIEROGLYPH-138E2;Lo;0;L;;;;;N;;;;; +138E3;EGYPTIAN HIEROGLYPH-138E3;Lo;0;L;;;;;N;;;;; +138E4;EGYPTIAN HIEROGLYPH-138E4;Lo;0;L;;;;;N;;;;; +138E5;EGYPTIAN HIEROGLYPH-138E5;Lo;0;L;;;;;N;;;;; +138E6;EGYPTIAN HIEROGLYPH-138E6;Lo;0;L;;;;;N;;;;; +138E7;EGYPTIAN HIEROGLYPH-138E7;Lo;0;L;;;;;N;;;;; +138E8;EGYPTIAN HIEROGLYPH-138E8;Lo;0;L;;;;;N;;;;; +138E9;EGYPTIAN HIEROGLYPH-138E9;Lo;0;L;;;;;N;;;;; +138EA;EGYPTIAN HIEROGLYPH-138EA;Lo;0;L;;;;;N;;;;; +138EB;EGYPTIAN HIEROGLYPH-138EB;Lo;0;L;;;;;N;;;;; +138EC;EGYPTIAN HIEROGLYPH-138EC;Lo;0;L;;;;;N;;;;; +138ED;EGYPTIAN HIEROGLYPH-138ED;Lo;0;L;;;;;N;;;;; +138EE;EGYPTIAN HIEROGLYPH-138EE;Lo;0;L;;;;;N;;;;; +138EF;EGYPTIAN HIEROGLYPH-138EF;Lo;0;L;;;;;N;;;;; +138F0;EGYPTIAN HIEROGLYPH-138F0;Lo;0;L;;;;;N;;;;; +138F1;EGYPTIAN HIEROGLYPH-138F1;Lo;0;L;;;;;N;;;;; +138F2;EGYPTIAN HIEROGLYPH-138F2;Lo;0;L;;;;;N;;;;; +138F3;EGYPTIAN HIEROGLYPH-138F3;Lo;0;L;;;;;N;;;;; +138F4;EGYPTIAN HIEROGLYPH-138F4;Lo;0;L;;;;;N;;;;; +138F5;EGYPTIAN HIEROGLYPH-138F5;Lo;0;L;;;;;N;;;;; +138F6;EGYPTIAN HIEROGLYPH-138F6;Lo;0;L;;;;;N;;;;; +138F7;EGYPTIAN HIEROGLYPH-138F7;Lo;0;L;;;;;N;;;;; +138F8;EGYPTIAN HIEROGLYPH-138F8;Lo;0;L;;;;;N;;;;; +138F9;EGYPTIAN HIEROGLYPH-138F9;Lo;0;L;;;;;N;;;;; +138FA;EGYPTIAN HIEROGLYPH-138FA;Lo;0;L;;;;;N;;;;; +138FB;EGYPTIAN HIEROGLYPH-138FB;Lo;0;L;;;;;N;;;;; +138FC;EGYPTIAN HIEROGLYPH-138FC;Lo;0;L;;;;;N;;;;; +138FD;EGYPTIAN HIEROGLYPH-138FD;Lo;0;L;;;;;N;;;;; +138FE;EGYPTIAN HIEROGLYPH-138FE;Lo;0;L;;;;;N;;;;; +138FF;EGYPTIAN HIEROGLYPH-138FF;Lo;0;L;;;;;N;;;;; +13900;EGYPTIAN HIEROGLYPH-13900;Lo;0;L;;;;;N;;;;; +13901;EGYPTIAN HIEROGLYPH-13901;Lo;0;L;;;;;N;;;;; +13902;EGYPTIAN HIEROGLYPH-13902;Lo;0;L;;;;;N;;;;; +13903;EGYPTIAN HIEROGLYPH-13903;Lo;0;L;;;;;N;;;;; +13904;EGYPTIAN HIEROGLYPH-13904;Lo;0;L;;;;;N;;;;; +13905;EGYPTIAN HIEROGLYPH-13905;Lo;0;L;;;;;N;;;;; +13906;EGYPTIAN HIEROGLYPH-13906;Lo;0;L;;;;;N;;;;; +13907;EGYPTIAN HIEROGLYPH-13907;Lo;0;L;;;;;N;;;;; +13908;EGYPTIAN HIEROGLYPH-13908;Lo;0;L;;;;;N;;;;; +13909;EGYPTIAN HIEROGLYPH-13909;Lo;0;L;;;;;N;;;;; +1390A;EGYPTIAN HIEROGLYPH-1390A;Lo;0;L;;;;;N;;;;; +1390B;EGYPTIAN HIEROGLYPH-1390B;Lo;0;L;;;;;N;;;;; +1390C;EGYPTIAN HIEROGLYPH-1390C;Lo;0;L;;;;;N;;;;; +1390D;EGYPTIAN HIEROGLYPH-1390D;Lo;0;L;;;;;N;;;;; +1390E;EGYPTIAN HIEROGLYPH-1390E;Lo;0;L;;;;;N;;;;; +1390F;EGYPTIAN HIEROGLYPH-1390F;Lo;0;L;;;;;N;;;;; +13910;EGYPTIAN HIEROGLYPH-13910;Lo;0;L;;;;;N;;;;; +13911;EGYPTIAN HIEROGLYPH-13911;Lo;0;L;;;;;N;;;;; +13912;EGYPTIAN HIEROGLYPH-13912;Lo;0;L;;;;;N;;;;; +13913;EGYPTIAN HIEROGLYPH-13913;Lo;0;L;;;;;N;;;;; +13914;EGYPTIAN HIEROGLYPH-13914;Lo;0;L;;;;;N;;;;; +13915;EGYPTIAN HIEROGLYPH-13915;Lo;0;L;;;;;N;;;;; +13916;EGYPTIAN HIEROGLYPH-13916;Lo;0;L;;;;;N;;;;; +13917;EGYPTIAN HIEROGLYPH-13917;Lo;0;L;;;;;N;;;;; +13918;EGYPTIAN HIEROGLYPH-13918;Lo;0;L;;;;;N;;;;; +13919;EGYPTIAN HIEROGLYPH-13919;Lo;0;L;;;;;N;;;;; +1391A;EGYPTIAN HIEROGLYPH-1391A;Lo;0;L;;;;;N;;;;; +1391B;EGYPTIAN HIEROGLYPH-1391B;Lo;0;L;;;;;N;;;;; +1391C;EGYPTIAN HIEROGLYPH-1391C;Lo;0;L;;;;;N;;;;; +1391D;EGYPTIAN HIEROGLYPH-1391D;Lo;0;L;;;;;N;;;;; +1391E;EGYPTIAN HIEROGLYPH-1391E;Lo;0;L;;;;;N;;;;; +1391F;EGYPTIAN HIEROGLYPH-1391F;Lo;0;L;;;;;N;;;;; +13920;EGYPTIAN HIEROGLYPH-13920;Lo;0;L;;;;;N;;;;; +13921;EGYPTIAN HIEROGLYPH-13921;Lo;0;L;;;;;N;;;;; +13922;EGYPTIAN HIEROGLYPH-13922;Lo;0;L;;;;;N;;;;; +13923;EGYPTIAN HIEROGLYPH-13923;Lo;0;L;;;;;N;;;;; +13924;EGYPTIAN HIEROGLYPH-13924;Lo;0;L;;;;;N;;;;; +13925;EGYPTIAN HIEROGLYPH-13925;Lo;0;L;;;;;N;;;;; +13926;EGYPTIAN HIEROGLYPH-13926;Lo;0;L;;;;;N;;;;; +13927;EGYPTIAN HIEROGLYPH-13927;Lo;0;L;;;;;N;;;;; +13928;EGYPTIAN HIEROGLYPH-13928;Lo;0;L;;;;;N;;;;; +13929;EGYPTIAN HIEROGLYPH-13929;Lo;0;L;;;;;N;;;;; +1392A;EGYPTIAN HIEROGLYPH-1392A;Lo;0;L;;;;;N;;;;; +1392B;EGYPTIAN HIEROGLYPH-1392B;Lo;0;L;;;;;N;;;;; +1392C;EGYPTIAN HIEROGLYPH-1392C;Lo;0;L;;;;;N;;;;; +1392D;EGYPTIAN HIEROGLYPH-1392D;Lo;0;L;;;;;N;;;;; +1392E;EGYPTIAN HIEROGLYPH-1392E;Lo;0;L;;;;;N;;;;; +1392F;EGYPTIAN HIEROGLYPH-1392F;Lo;0;L;;;;;N;;;;; +13930;EGYPTIAN HIEROGLYPH-13930;Lo;0;L;;;;;N;;;;; +13931;EGYPTIAN HIEROGLYPH-13931;Lo;0;L;;;;;N;;;;; +13932;EGYPTIAN HIEROGLYPH-13932;Lo;0;L;;;;;N;;;;; +13933;EGYPTIAN HIEROGLYPH-13933;Lo;0;L;;;;;N;;;;; +13934;EGYPTIAN HIEROGLYPH-13934;Lo;0;L;;;;;N;;;;; +13935;EGYPTIAN HIEROGLYPH-13935;Lo;0;L;;;;;N;;;;; +13936;EGYPTIAN HIEROGLYPH-13936;Lo;0;L;;;;;N;;;;; +13937;EGYPTIAN HIEROGLYPH-13937;Lo;0;L;;;;;N;;;;; +13938;EGYPTIAN HIEROGLYPH-13938;Lo;0;L;;;;;N;;;;; +13939;EGYPTIAN HIEROGLYPH-13939;Lo;0;L;;;;;N;;;;; +1393A;EGYPTIAN HIEROGLYPH-1393A;Lo;0;L;;;;;N;;;;; +1393B;EGYPTIAN HIEROGLYPH-1393B;Lo;0;L;;;;;N;;;;; +1393C;EGYPTIAN HIEROGLYPH-1393C;Lo;0;L;;;;;N;;;;; +1393D;EGYPTIAN HIEROGLYPH-1393D;Lo;0;L;;;;;N;;;;; +1393E;EGYPTIAN HIEROGLYPH-1393E;Lo;0;L;;;;;N;;;;; +1393F;EGYPTIAN HIEROGLYPH-1393F;Lo;0;L;;;;;N;;;;; +13940;EGYPTIAN HIEROGLYPH-13940;Lo;0;L;;;;;N;;;;; +13941;EGYPTIAN HIEROGLYPH-13941;Lo;0;L;;;;;N;;;;; +13942;EGYPTIAN HIEROGLYPH-13942;Lo;0;L;;;;;N;;;;; +13943;EGYPTIAN HIEROGLYPH-13943;Lo;0;L;;;;;N;;;;; +13944;EGYPTIAN HIEROGLYPH-13944;Lo;0;L;;;;;N;;;;; +13945;EGYPTIAN HIEROGLYPH-13945;Lo;0;L;;;;;N;;;;; +13946;EGYPTIAN HIEROGLYPH-13946;Lo;0;L;;;;;N;;;;; +13947;EGYPTIAN HIEROGLYPH-13947;Lo;0;L;;;;;N;;;;; +13948;EGYPTIAN HIEROGLYPH-13948;Lo;0;L;;;;;N;;;;; +13949;EGYPTIAN HIEROGLYPH-13949;Lo;0;L;;;;;N;;;;; +1394A;EGYPTIAN HIEROGLYPH-1394A;Lo;0;L;;;;;N;;;;; +1394B;EGYPTIAN HIEROGLYPH-1394B;Lo;0;L;;;;;N;;;;; +1394C;EGYPTIAN HIEROGLYPH-1394C;Lo;0;L;;;;;N;;;;; +1394D;EGYPTIAN HIEROGLYPH-1394D;Lo;0;L;;;;;N;;;;; +1394E;EGYPTIAN HIEROGLYPH-1394E;Lo;0;L;;;;;N;;;;; +1394F;EGYPTIAN HIEROGLYPH-1394F;Lo;0;L;;;;;N;;;;; +13950;EGYPTIAN HIEROGLYPH-13950;Lo;0;L;;;;;N;;;;; +13951;EGYPTIAN HIEROGLYPH-13951;Lo;0;L;;;;;N;;;;; +13952;EGYPTIAN HIEROGLYPH-13952;Lo;0;L;;;;;N;;;;; +13953;EGYPTIAN HIEROGLYPH-13953;Lo;0;L;;;;;N;;;;; +13954;EGYPTIAN HIEROGLYPH-13954;Lo;0;L;;;;;N;;;;; +13955;EGYPTIAN HIEROGLYPH-13955;Lo;0;L;;;;;N;;;;; +13956;EGYPTIAN HIEROGLYPH-13956;Lo;0;L;;;;;N;;;;; +13957;EGYPTIAN HIEROGLYPH-13957;Lo;0;L;;;;;N;;;;; +13958;EGYPTIAN HIEROGLYPH-13958;Lo;0;L;;;;;N;;;;; +13959;EGYPTIAN HIEROGLYPH-13959;Lo;0;L;;;;;N;;;;; +1395A;EGYPTIAN HIEROGLYPH-1395A;Lo;0;L;;;;;N;;;;; +1395B;EGYPTIAN HIEROGLYPH-1395B;Lo;0;L;;;;;N;;;;; +1395C;EGYPTIAN HIEROGLYPH-1395C;Lo;0;L;;;;;N;;;;; +1395D;EGYPTIAN HIEROGLYPH-1395D;Lo;0;L;;;;;N;;;;; +1395E;EGYPTIAN HIEROGLYPH-1395E;Lo;0;L;;;;;N;;;;; +1395F;EGYPTIAN HIEROGLYPH-1395F;Lo;0;L;;;;;N;;;;; +13960;EGYPTIAN HIEROGLYPH-13960;Lo;0;L;;;;;N;;;;; +13961;EGYPTIAN HIEROGLYPH-13961;Lo;0;L;;;;;N;;;;; +13962;EGYPTIAN HIEROGLYPH-13962;Lo;0;L;;;;;N;;;;; +13963;EGYPTIAN HIEROGLYPH-13963;Lo;0;L;;;;;N;;;;; +13964;EGYPTIAN HIEROGLYPH-13964;Lo;0;L;;;;;N;;;;; +13965;EGYPTIAN HIEROGLYPH-13965;Lo;0;L;;;;;N;;;;; +13966;EGYPTIAN HIEROGLYPH-13966;Lo;0;L;;;;;N;;;;; +13967;EGYPTIAN HIEROGLYPH-13967;Lo;0;L;;;;;N;;;;; +13968;EGYPTIAN HIEROGLYPH-13968;Lo;0;L;;;;;N;;;;; +13969;EGYPTIAN HIEROGLYPH-13969;Lo;0;L;;;;;N;;;;; +1396A;EGYPTIAN HIEROGLYPH-1396A;Lo;0;L;;;;;N;;;;; +1396B;EGYPTIAN HIEROGLYPH-1396B;Lo;0;L;;;;;N;;;;; +1396C;EGYPTIAN HIEROGLYPH-1396C;Lo;0;L;;;;;N;;;;; +1396D;EGYPTIAN HIEROGLYPH-1396D;Lo;0;L;;;;;N;;;;; +1396E;EGYPTIAN HIEROGLYPH-1396E;Lo;0;L;;;;;N;;;;; +1396F;EGYPTIAN HIEROGLYPH-1396F;Lo;0;L;;;;;N;;;;; +13970;EGYPTIAN HIEROGLYPH-13970;Lo;0;L;;;;;N;;;;; +13971;EGYPTIAN HIEROGLYPH-13971;Lo;0;L;;;;;N;;;;; +13972;EGYPTIAN HIEROGLYPH-13972;Lo;0;L;;;;;N;;;;; +13973;EGYPTIAN HIEROGLYPH-13973;Lo;0;L;;;;;N;;;;; +13974;EGYPTIAN HIEROGLYPH-13974;Lo;0;L;;;;;N;;;;; +13975;EGYPTIAN HIEROGLYPH-13975;Lo;0;L;;;;;N;;;;; +13976;EGYPTIAN HIEROGLYPH-13976;Lo;0;L;;;;;N;;;;; +13977;EGYPTIAN HIEROGLYPH-13977;Lo;0;L;;;;;N;;;;; +13978;EGYPTIAN HIEROGLYPH-13978;Lo;0;L;;;;;N;;;;; +13979;EGYPTIAN HIEROGLYPH-13979;Lo;0;L;;;;;N;;;;; +1397A;EGYPTIAN HIEROGLYPH-1397A;Lo;0;L;;;;;N;;;;; +1397B;EGYPTIAN HIEROGLYPH-1397B;Lo;0;L;;;;;N;;;;; +1397C;EGYPTIAN HIEROGLYPH-1397C;Lo;0;L;;;;;N;;;;; +1397D;EGYPTIAN HIEROGLYPH-1397D;Lo;0;L;;;;;N;;;;; +1397E;EGYPTIAN HIEROGLYPH-1397E;Lo;0;L;;;;;N;;;;; +1397F;EGYPTIAN HIEROGLYPH-1397F;Lo;0;L;;;;;N;;;;; +13980;EGYPTIAN HIEROGLYPH-13980;Lo;0;L;;;;;N;;;;; +13981;EGYPTIAN HIEROGLYPH-13981;Lo;0;L;;;;;N;;;;; +13982;EGYPTIAN HIEROGLYPH-13982;Lo;0;L;;;;;N;;;;; +13983;EGYPTIAN HIEROGLYPH-13983;Lo;0;L;;;;;N;;;;; +13984;EGYPTIAN HIEROGLYPH-13984;Lo;0;L;;;;;N;;;;; +13985;EGYPTIAN HIEROGLYPH-13985;Lo;0;L;;;;;N;;;;; +13986;EGYPTIAN HIEROGLYPH-13986;Lo;0;L;;;;;N;;;;; +13987;EGYPTIAN HIEROGLYPH-13987;Lo;0;L;;;;;N;;;;; +13988;EGYPTIAN HIEROGLYPH-13988;Lo;0;L;;;;;N;;;;; +13989;EGYPTIAN HIEROGLYPH-13989;Lo;0;L;;;;;N;;;;; +1398A;EGYPTIAN HIEROGLYPH-1398A;Lo;0;L;;;;;N;;;;; +1398B;EGYPTIAN HIEROGLYPH-1398B;Lo;0;L;;;;;N;;;;; +1398C;EGYPTIAN HIEROGLYPH-1398C;Lo;0;L;;;;;N;;;;; +1398D;EGYPTIAN HIEROGLYPH-1398D;Lo;0;L;;;;;N;;;;; +1398E;EGYPTIAN HIEROGLYPH-1398E;Lo;0;L;;;;;N;;;;; +1398F;EGYPTIAN HIEROGLYPH-1398F;Lo;0;L;;;;;N;;;;; +13990;EGYPTIAN HIEROGLYPH-13990;Lo;0;L;;;;;N;;;;; +13991;EGYPTIAN HIEROGLYPH-13991;Lo;0;L;;;;;N;;;;; +13992;EGYPTIAN HIEROGLYPH-13992;Lo;0;L;;;;;N;;;;; +13993;EGYPTIAN HIEROGLYPH-13993;Lo;0;L;;;;;N;;;;; +13994;EGYPTIAN HIEROGLYPH-13994;Lo;0;L;;;;;N;;;;; +13995;EGYPTIAN HIEROGLYPH-13995;Lo;0;L;;;;;N;;;;; +13996;EGYPTIAN HIEROGLYPH-13996;Lo;0;L;;;;;N;;;;; +13997;EGYPTIAN HIEROGLYPH-13997;Lo;0;L;;;;;N;;;;; +13998;EGYPTIAN HIEROGLYPH-13998;Lo;0;L;;;;;N;;;;; +13999;EGYPTIAN HIEROGLYPH-13999;Lo;0;L;;;;;N;;;;; +1399A;EGYPTIAN HIEROGLYPH-1399A;Lo;0;L;;;;;N;;;;; +1399B;EGYPTIAN HIEROGLYPH-1399B;Lo;0;L;;;;;N;;;;; +1399C;EGYPTIAN HIEROGLYPH-1399C;Lo;0;L;;;;;N;;;;; +1399D;EGYPTIAN HIEROGLYPH-1399D;Lo;0;L;;;;;N;;;;; +1399E;EGYPTIAN HIEROGLYPH-1399E;Lo;0;L;;;;;N;;;;; +1399F;EGYPTIAN HIEROGLYPH-1399F;Lo;0;L;;;;;N;;;;; +139A0;EGYPTIAN HIEROGLYPH-139A0;Lo;0;L;;;;;N;;;;; +139A1;EGYPTIAN HIEROGLYPH-139A1;Lo;0;L;;;;;N;;;;; +139A2;EGYPTIAN HIEROGLYPH-139A2;Lo;0;L;;;;;N;;;;; +139A3;EGYPTIAN HIEROGLYPH-139A3;Lo;0;L;;;;;N;;;;; +139A4;EGYPTIAN HIEROGLYPH-139A4;Lo;0;L;;;;;N;;;;; +139A5;EGYPTIAN HIEROGLYPH-139A5;Lo;0;L;;;;;N;;;;; +139A6;EGYPTIAN HIEROGLYPH-139A6;Lo;0;L;;;;;N;;;;; +139A7;EGYPTIAN HIEROGLYPH-139A7;Lo;0;L;;;;;N;;;;; +139A8;EGYPTIAN HIEROGLYPH-139A8;Lo;0;L;;;;;N;;;;; +139A9;EGYPTIAN HIEROGLYPH-139A9;Lo;0;L;;;;;N;;;;; +139AA;EGYPTIAN HIEROGLYPH-139AA;Lo;0;L;;;;;N;;;;; +139AB;EGYPTIAN HIEROGLYPH-139AB;Lo;0;L;;;;;N;;;;; +139AC;EGYPTIAN HIEROGLYPH-139AC;Lo;0;L;;;;;N;;;;; +139AD;EGYPTIAN HIEROGLYPH-139AD;Lo;0;L;;;;;N;;;;; +139AE;EGYPTIAN HIEROGLYPH-139AE;Lo;0;L;;;;;N;;;;; +139AF;EGYPTIAN HIEROGLYPH-139AF;Lo;0;L;;;;;N;;;;; +139B0;EGYPTIAN HIEROGLYPH-139B0;Lo;0;L;;;;;N;;;;; +139B1;EGYPTIAN HIEROGLYPH-139B1;Lo;0;L;;;;;N;;;;; +139B2;EGYPTIAN HIEROGLYPH-139B2;Lo;0;L;;;;;N;;;;; +139B3;EGYPTIAN HIEROGLYPH-139B3;Lo;0;L;;;;;N;;;;; +139B4;EGYPTIAN HIEROGLYPH-139B4;Lo;0;L;;;;;N;;;;; +139B5;EGYPTIAN HIEROGLYPH-139B5;Lo;0;L;;;;;N;;;;; +139B6;EGYPTIAN HIEROGLYPH-139B6;Lo;0;L;;;;;N;;;;; +139B7;EGYPTIAN HIEROGLYPH-139B7;Lo;0;L;;;;;N;;;;; +139B8;EGYPTIAN HIEROGLYPH-139B8;Lo;0;L;;;;;N;;;;; +139B9;EGYPTIAN HIEROGLYPH-139B9;Lo;0;L;;;;;N;;;;; +139BA;EGYPTIAN HIEROGLYPH-139BA;Lo;0;L;;;;;N;;;;; +139BB;EGYPTIAN HIEROGLYPH-139BB;Lo;0;L;;;;;N;;;;; +139BC;EGYPTIAN HIEROGLYPH-139BC;Lo;0;L;;;;;N;;;;; +139BD;EGYPTIAN HIEROGLYPH-139BD;Lo;0;L;;;;;N;;;;; +139BE;EGYPTIAN HIEROGLYPH-139BE;Lo;0;L;;;;;N;;;;; +139BF;EGYPTIAN HIEROGLYPH-139BF;Lo;0;L;;;;;N;;;;; +139C0;EGYPTIAN HIEROGLYPH-139C0;Lo;0;L;;;;;N;;;;; +139C1;EGYPTIAN HIEROGLYPH-139C1;Lo;0;L;;;;;N;;;;; +139C2;EGYPTIAN HIEROGLYPH-139C2;Lo;0;L;;;;;N;;;;; +139C3;EGYPTIAN HIEROGLYPH-139C3;Lo;0;L;;;;;N;;;;; +139C4;EGYPTIAN HIEROGLYPH-139C4;Lo;0;L;;;;;N;;;;; +139C5;EGYPTIAN HIEROGLYPH-139C5;Lo;0;L;;;;;N;;;;; +139C6;EGYPTIAN HIEROGLYPH-139C6;Lo;0;L;;;;;N;;;;; +139C7;EGYPTIAN HIEROGLYPH-139C7;Lo;0;L;;;;;N;;;;; +139C8;EGYPTIAN HIEROGLYPH-139C8;Lo;0;L;;;;;N;;;;; +139C9;EGYPTIAN HIEROGLYPH-139C9;Lo;0;L;;;;;N;;;;; +139CA;EGYPTIAN HIEROGLYPH-139CA;Lo;0;L;;;;;N;;;;; +139CB;EGYPTIAN HIEROGLYPH-139CB;Lo;0;L;;;;;N;;;;; +139CC;EGYPTIAN HIEROGLYPH-139CC;Lo;0;L;;;;;N;;;;; +139CD;EGYPTIAN HIEROGLYPH-139CD;Lo;0;L;;;;;N;;;;; +139CE;EGYPTIAN HIEROGLYPH-139CE;Lo;0;L;;;;;N;;;;; +139CF;EGYPTIAN HIEROGLYPH-139CF;Lo;0;L;;;;;N;;;;; +139D0;EGYPTIAN HIEROGLYPH-139D0;Lo;0;L;;;;;N;;;;; +139D1;EGYPTIAN HIEROGLYPH-139D1;Lo;0;L;;;;;N;;;;; +139D2;EGYPTIAN HIEROGLYPH-139D2;Lo;0;L;;;;;N;;;;; +139D3;EGYPTIAN HIEROGLYPH-139D3;Lo;0;L;;;;;N;;;;; +139D4;EGYPTIAN HIEROGLYPH-139D4;Lo;0;L;;;;;N;;;;; +139D5;EGYPTIAN HIEROGLYPH-139D5;Lo;0;L;;;;;N;;;;; +139D6;EGYPTIAN HIEROGLYPH-139D6;Lo;0;L;;;;;N;;;;; +139D7;EGYPTIAN HIEROGLYPH-139D7;Lo;0;L;;;;;N;;;;; +139D8;EGYPTIAN HIEROGLYPH-139D8;Lo;0;L;;;;;N;;;;; +139D9;EGYPTIAN HIEROGLYPH-139D9;Lo;0;L;;;;;N;;;;; +139DA;EGYPTIAN HIEROGLYPH-139DA;Lo;0;L;;;;;N;;;;; +139DB;EGYPTIAN HIEROGLYPH-139DB;Lo;0;L;;;;;N;;;;; +139DC;EGYPTIAN HIEROGLYPH-139DC;Lo;0;L;;;;;N;;;;; +139DD;EGYPTIAN HIEROGLYPH-139DD;Lo;0;L;;;;;N;;;;; +139DE;EGYPTIAN HIEROGLYPH-139DE;Lo;0;L;;;;;N;;;;; +139DF;EGYPTIAN HIEROGLYPH-139DF;Lo;0;L;;;;;N;;;;; +139E0;EGYPTIAN HIEROGLYPH-139E0;Lo;0;L;;;;;N;;;;; +139E1;EGYPTIAN HIEROGLYPH-139E1;Lo;0;L;;;;;N;;;;; +139E2;EGYPTIAN HIEROGLYPH-139E2;Lo;0;L;;;;;N;;;;; +139E3;EGYPTIAN HIEROGLYPH-139E3;Lo;0;L;;;;;N;;;;; +139E4;EGYPTIAN HIEROGLYPH-139E4;Lo;0;L;;;;;N;;;;; +139E5;EGYPTIAN HIEROGLYPH-139E5;Lo;0;L;;;;;N;;;;; +139E6;EGYPTIAN HIEROGLYPH-139E6;Lo;0;L;;;;;N;;;;; +139E7;EGYPTIAN HIEROGLYPH-139E7;Lo;0;L;;;;;N;;;;; +139E8;EGYPTIAN HIEROGLYPH-139E8;Lo;0;L;;;;;N;;;;; +139E9;EGYPTIAN HIEROGLYPH-139E9;Lo;0;L;;;;;N;;;;; +139EA;EGYPTIAN HIEROGLYPH-139EA;Lo;0;L;;;;;N;;;;; +139EB;EGYPTIAN HIEROGLYPH-139EB;Lo;0;L;;;;;N;;;;; +139EC;EGYPTIAN HIEROGLYPH-139EC;Lo;0;L;;;;;N;;;;; +139ED;EGYPTIAN HIEROGLYPH-139ED;Lo;0;L;;;;;N;;;;; +139EE;EGYPTIAN HIEROGLYPH-139EE;Lo;0;L;;;;;N;;;;; +139EF;EGYPTIAN HIEROGLYPH-139EF;Lo;0;L;;;;;N;;;;; +139F0;EGYPTIAN HIEROGLYPH-139F0;Lo;0;L;;;;;N;;;;; +139F1;EGYPTIAN HIEROGLYPH-139F1;Lo;0;L;;;;;N;;;;; +139F2;EGYPTIAN HIEROGLYPH-139F2;Lo;0;L;;;;;N;;;;; +139F3;EGYPTIAN HIEROGLYPH-139F3;Lo;0;L;;;;;N;;;;; +139F4;EGYPTIAN HIEROGLYPH-139F4;Lo;0;L;;;;;N;;;;; +139F5;EGYPTIAN HIEROGLYPH-139F5;Lo;0;L;;;;;N;;;;; +139F6;EGYPTIAN HIEROGLYPH-139F6;Lo;0;L;;;;;N;;;;; +139F7;EGYPTIAN HIEROGLYPH-139F7;Lo;0;L;;;;;N;;;;; +139F8;EGYPTIAN HIEROGLYPH-139F8;Lo;0;L;;;;;N;;;;; +139F9;EGYPTIAN HIEROGLYPH-139F9;Lo;0;L;;;;;N;;;;; +139FA;EGYPTIAN HIEROGLYPH-139FA;Lo;0;L;;;;;N;;;;; +139FB;EGYPTIAN HIEROGLYPH-139FB;Lo;0;L;;;;;N;;;;; +139FC;EGYPTIAN HIEROGLYPH-139FC;Lo;0;L;;;;;N;;;;; +139FD;EGYPTIAN HIEROGLYPH-139FD;Lo;0;L;;;;;N;;;;; +139FE;EGYPTIAN HIEROGLYPH-139FE;Lo;0;L;;;;;N;;;;; +139FF;EGYPTIAN HIEROGLYPH-139FF;Lo;0;L;;;;;N;;;;; +13A00;EGYPTIAN HIEROGLYPH-13A00;Lo;0;L;;;;;N;;;;; +13A01;EGYPTIAN HIEROGLYPH-13A01;Lo;0;L;;;;;N;;;;; +13A02;EGYPTIAN HIEROGLYPH-13A02;Lo;0;L;;;;;N;;;;; +13A03;EGYPTIAN HIEROGLYPH-13A03;Lo;0;L;;;;;N;;;;; +13A04;EGYPTIAN HIEROGLYPH-13A04;Lo;0;L;;;;;N;;;;; +13A05;EGYPTIAN HIEROGLYPH-13A05;Lo;0;L;;;;;N;;;;; +13A06;EGYPTIAN HIEROGLYPH-13A06;Lo;0;L;;;;;N;;;;; +13A07;EGYPTIAN HIEROGLYPH-13A07;Lo;0;L;;;;;N;;;;; +13A08;EGYPTIAN HIEROGLYPH-13A08;Lo;0;L;;;;;N;;;;; +13A09;EGYPTIAN HIEROGLYPH-13A09;Lo;0;L;;;;;N;;;;; +13A0A;EGYPTIAN HIEROGLYPH-13A0A;Lo;0;L;;;;;N;;;;; +13A0B;EGYPTIAN HIEROGLYPH-13A0B;Lo;0;L;;;;;N;;;;; +13A0C;EGYPTIAN HIEROGLYPH-13A0C;Lo;0;L;;;;;N;;;;; +13A0D;EGYPTIAN HIEROGLYPH-13A0D;Lo;0;L;;;;;N;;;;; +13A0E;EGYPTIAN HIEROGLYPH-13A0E;Lo;0;L;;;;;N;;;;; +13A0F;EGYPTIAN HIEROGLYPH-13A0F;Lo;0;L;;;;;N;;;;; +13A10;EGYPTIAN HIEROGLYPH-13A10;Lo;0;L;;;;;N;;;;; +13A11;EGYPTIAN HIEROGLYPH-13A11;Lo;0;L;;;;;N;;;;; +13A12;EGYPTIAN HIEROGLYPH-13A12;Lo;0;L;;;;;N;;;;; +13A13;EGYPTIAN HIEROGLYPH-13A13;Lo;0;L;;;;;N;;;;; +13A14;EGYPTIAN HIEROGLYPH-13A14;Lo;0;L;;;;;N;;;;; +13A15;EGYPTIAN HIEROGLYPH-13A15;Lo;0;L;;;;;N;;;;; +13A16;EGYPTIAN HIEROGLYPH-13A16;Lo;0;L;;;;;N;;;;; +13A17;EGYPTIAN HIEROGLYPH-13A17;Lo;0;L;;;;;N;;;;; +13A18;EGYPTIAN HIEROGLYPH-13A18;Lo;0;L;;;;;N;;;;; +13A19;EGYPTIAN HIEROGLYPH-13A19;Lo;0;L;;;;;N;;;;; +13A1A;EGYPTIAN HIEROGLYPH-13A1A;Lo;0;L;;;;;N;;;;; +13A1B;EGYPTIAN HIEROGLYPH-13A1B;Lo;0;L;;;;;N;;;;; +13A1C;EGYPTIAN HIEROGLYPH-13A1C;Lo;0;L;;;;;N;;;;; +13A1D;EGYPTIAN HIEROGLYPH-13A1D;Lo;0;L;;;;;N;;;;; +13A1E;EGYPTIAN HIEROGLYPH-13A1E;Lo;0;L;;;;;N;;;;; +13A1F;EGYPTIAN HIEROGLYPH-13A1F;Lo;0;L;;;;;N;;;;; +13A20;EGYPTIAN HIEROGLYPH-13A20;Lo;0;L;;;;;N;;;;; +13A21;EGYPTIAN HIEROGLYPH-13A21;Lo;0;L;;;;;N;;;;; +13A22;EGYPTIAN HIEROGLYPH-13A22;Lo;0;L;;;;;N;;;;; +13A23;EGYPTIAN HIEROGLYPH-13A23;Lo;0;L;;;;;N;;;;; +13A24;EGYPTIAN HIEROGLYPH-13A24;Lo;0;L;;;;;N;;;;; +13A25;EGYPTIAN HIEROGLYPH-13A25;Lo;0;L;;;;;N;;;;; +13A26;EGYPTIAN HIEROGLYPH-13A26;Lo;0;L;;;;;N;;;;; +13A27;EGYPTIAN HIEROGLYPH-13A27;Lo;0;L;;;;;N;;;;; +13A28;EGYPTIAN HIEROGLYPH-13A28;Lo;0;L;;;;;N;;;;; +13A29;EGYPTIAN HIEROGLYPH-13A29;Lo;0;L;;;;;N;;;;; +13A2A;EGYPTIAN HIEROGLYPH-13A2A;Lo;0;L;;;;;N;;;;; +13A2B;EGYPTIAN HIEROGLYPH-13A2B;Lo;0;L;;;;;N;;;;; +13A2C;EGYPTIAN HIEROGLYPH-13A2C;Lo;0;L;;;;;N;;;;; +13A2D;EGYPTIAN HIEROGLYPH-13A2D;Lo;0;L;;;;;N;;;;; +13A2E;EGYPTIAN HIEROGLYPH-13A2E;Lo;0;L;;;;;N;;;;; +13A2F;EGYPTIAN HIEROGLYPH-13A2F;Lo;0;L;;;;;N;;;;; +13A30;EGYPTIAN HIEROGLYPH-13A30;Lo;0;L;;;;;N;;;;; +13A31;EGYPTIAN HIEROGLYPH-13A31;Lo;0;L;;;;;N;;;;; +13A32;EGYPTIAN HIEROGLYPH-13A32;Lo;0;L;;;;;N;;;;; +13A33;EGYPTIAN HIEROGLYPH-13A33;Lo;0;L;;;;;N;;;;; +13A34;EGYPTIAN HIEROGLYPH-13A34;Lo;0;L;;;;;N;;;;; +13A35;EGYPTIAN HIEROGLYPH-13A35;Lo;0;L;;;;;N;;;;; +13A36;EGYPTIAN HIEROGLYPH-13A36;Lo;0;L;;;;;N;;;;; +13A37;EGYPTIAN HIEROGLYPH-13A37;Lo;0;L;;;;;N;;;;; +13A38;EGYPTIAN HIEROGLYPH-13A38;Lo;0;L;;;;;N;;;;; +13A39;EGYPTIAN HIEROGLYPH-13A39;Lo;0;L;;;;;N;;;;; +13A3A;EGYPTIAN HIEROGLYPH-13A3A;Lo;0;L;;;;;N;;;;; +13A3B;EGYPTIAN HIEROGLYPH-13A3B;Lo;0;L;;;;;N;;;;; +13A3C;EGYPTIAN HIEROGLYPH-13A3C;Lo;0;L;;;;;N;;;;; +13A3D;EGYPTIAN HIEROGLYPH-13A3D;Lo;0;L;;;;;N;;;;; +13A3E;EGYPTIAN HIEROGLYPH-13A3E;Lo;0;L;;;;;N;;;;; +13A3F;EGYPTIAN HIEROGLYPH-13A3F;Lo;0;L;;;;;N;;;;; +13A40;EGYPTIAN HIEROGLYPH-13A40;Lo;0;L;;;;;N;;;;; +13A41;EGYPTIAN HIEROGLYPH-13A41;Lo;0;L;;;;;N;;;;; +13A42;EGYPTIAN HIEROGLYPH-13A42;Lo;0;L;;;;;N;;;;; +13A43;EGYPTIAN HIEROGLYPH-13A43;Lo;0;L;;;;;N;;;;; +13A44;EGYPTIAN HIEROGLYPH-13A44;Lo;0;L;;;;;N;;;;; +13A45;EGYPTIAN HIEROGLYPH-13A45;Lo;0;L;;;;;N;;;;; +13A46;EGYPTIAN HIEROGLYPH-13A46;Lo;0;L;;;;;N;;;;; +13A47;EGYPTIAN HIEROGLYPH-13A47;Lo;0;L;;;;;N;;;;; +13A48;EGYPTIAN HIEROGLYPH-13A48;Lo;0;L;;;;;N;;;;; +13A49;EGYPTIAN HIEROGLYPH-13A49;Lo;0;L;;;;;N;;;;; +13A4A;EGYPTIAN HIEROGLYPH-13A4A;Lo;0;L;;;;;N;;;;; +13A4B;EGYPTIAN HIEROGLYPH-13A4B;Lo;0;L;;;;;N;;;;; +13A4C;EGYPTIAN HIEROGLYPH-13A4C;Lo;0;L;;;;;N;;;;; +13A4D;EGYPTIAN HIEROGLYPH-13A4D;Lo;0;L;;;;;N;;;;; +13A4E;EGYPTIAN HIEROGLYPH-13A4E;Lo;0;L;;;;;N;;;;; +13A4F;EGYPTIAN HIEROGLYPH-13A4F;Lo;0;L;;;;;N;;;;; +13A50;EGYPTIAN HIEROGLYPH-13A50;Lo;0;L;;;;;N;;;;; +13A51;EGYPTIAN HIEROGLYPH-13A51;Lo;0;L;;;;;N;;;;; +13A52;EGYPTIAN HIEROGLYPH-13A52;Lo;0;L;;;;;N;;;;; +13A53;EGYPTIAN HIEROGLYPH-13A53;Lo;0;L;;;;;N;;;;; +13A54;EGYPTIAN HIEROGLYPH-13A54;Lo;0;L;;;;;N;;;;; +13A55;EGYPTIAN HIEROGLYPH-13A55;Lo;0;L;;;;;N;;;;; +13A56;EGYPTIAN HIEROGLYPH-13A56;Lo;0;L;;;;;N;;;;; +13A57;EGYPTIAN HIEROGLYPH-13A57;Lo;0;L;;;;;N;;;;; +13A58;EGYPTIAN HIEROGLYPH-13A58;Lo;0;L;;;;;N;;;;; +13A59;EGYPTIAN HIEROGLYPH-13A59;Lo;0;L;;;;;N;;;;; +13A5A;EGYPTIAN HIEROGLYPH-13A5A;Lo;0;L;;;;;N;;;;; +13A5B;EGYPTIAN HIEROGLYPH-13A5B;Lo;0;L;;;;;N;;;;; +13A5C;EGYPTIAN HIEROGLYPH-13A5C;Lo;0;L;;;;;N;;;;; +13A5D;EGYPTIAN HIEROGLYPH-13A5D;Lo;0;L;;;;;N;;;;; +13A5E;EGYPTIAN HIEROGLYPH-13A5E;Lo;0;L;;;;;N;;;;; +13A5F;EGYPTIAN HIEROGLYPH-13A5F;Lo;0;L;;;;;N;;;;; +13A60;EGYPTIAN HIEROGLYPH-13A60;Lo;0;L;;;;;N;;;;; +13A61;EGYPTIAN HIEROGLYPH-13A61;Lo;0;L;;;;;N;;;;; +13A62;EGYPTIAN HIEROGLYPH-13A62;Lo;0;L;;;;;N;;;;; +13A63;EGYPTIAN HIEROGLYPH-13A63;Lo;0;L;;;;;N;;;;; +13A64;EGYPTIAN HIEROGLYPH-13A64;Lo;0;L;;;;;N;;;;; +13A65;EGYPTIAN HIEROGLYPH-13A65;Lo;0;L;;;;;N;;;;; +13A66;EGYPTIAN HIEROGLYPH-13A66;Lo;0;L;;;;;N;;;;; +13A67;EGYPTIAN HIEROGLYPH-13A67;Lo;0;L;;;;;N;;;;; +13A68;EGYPTIAN HIEROGLYPH-13A68;Lo;0;L;;;;;N;;;;; +13A69;EGYPTIAN HIEROGLYPH-13A69;Lo;0;L;;;;;N;;;;; +13A6A;EGYPTIAN HIEROGLYPH-13A6A;Lo;0;L;;;;;N;;;;; +13A6B;EGYPTIAN HIEROGLYPH-13A6B;Lo;0;L;;;;;N;;;;; +13A6C;EGYPTIAN HIEROGLYPH-13A6C;Lo;0;L;;;;;N;;;;; +13A6D;EGYPTIAN HIEROGLYPH-13A6D;Lo;0;L;;;;;N;;;;; +13A6E;EGYPTIAN HIEROGLYPH-13A6E;Lo;0;L;;;;;N;;;;; +13A6F;EGYPTIAN HIEROGLYPH-13A6F;Lo;0;L;;;;;N;;;;; +13A70;EGYPTIAN HIEROGLYPH-13A70;Lo;0;L;;;;;N;;;;; +13A71;EGYPTIAN HIEROGLYPH-13A71;Lo;0;L;;;;;N;;;;; +13A72;EGYPTIAN HIEROGLYPH-13A72;Lo;0;L;;;;;N;;;;; +13A73;EGYPTIAN HIEROGLYPH-13A73;Lo;0;L;;;;;N;;;;; +13A74;EGYPTIAN HIEROGLYPH-13A74;Lo;0;L;;;;;N;;;;; +13A75;EGYPTIAN HIEROGLYPH-13A75;Lo;0;L;;;;;N;;;;; +13A76;EGYPTIAN HIEROGLYPH-13A76;Lo;0;L;;;;;N;;;;; +13A77;EGYPTIAN HIEROGLYPH-13A77;Lo;0;L;;;;;N;;;;; +13A78;EGYPTIAN HIEROGLYPH-13A78;Lo;0;L;;;;;N;;;;; +13A79;EGYPTIAN HIEROGLYPH-13A79;Lo;0;L;;;;;N;;;;; +13A7A;EGYPTIAN HIEROGLYPH-13A7A;Lo;0;L;;;;;N;;;;; +13A7B;EGYPTIAN HIEROGLYPH-13A7B;Lo;0;L;;;;;N;;;;; +13A7C;EGYPTIAN HIEROGLYPH-13A7C;Lo;0;L;;;;;N;;;;; +13A7D;EGYPTIAN HIEROGLYPH-13A7D;Lo;0;L;;;;;N;;;;; +13A7E;EGYPTIAN HIEROGLYPH-13A7E;Lo;0;L;;;;;N;;;;; +13A7F;EGYPTIAN HIEROGLYPH-13A7F;Lo;0;L;;;;;N;;;;; +13A80;EGYPTIAN HIEROGLYPH-13A80;Lo;0;L;;;;;N;;;;; +13A81;EGYPTIAN HIEROGLYPH-13A81;Lo;0;L;;;;;N;;;;; +13A82;EGYPTIAN HIEROGLYPH-13A82;Lo;0;L;;;;;N;;;;; +13A83;EGYPTIAN HIEROGLYPH-13A83;Lo;0;L;;;;;N;;;;; +13A84;EGYPTIAN HIEROGLYPH-13A84;Lo;0;L;;;;;N;;;;; +13A85;EGYPTIAN HIEROGLYPH-13A85;Lo;0;L;;;;;N;;;;; +13A86;EGYPTIAN HIEROGLYPH-13A86;Lo;0;L;;;;;N;;;;; +13A87;EGYPTIAN HIEROGLYPH-13A87;Lo;0;L;;;;;N;;;;; +13A88;EGYPTIAN HIEROGLYPH-13A88;Lo;0;L;;;;;N;;;;; +13A89;EGYPTIAN HIEROGLYPH-13A89;Lo;0;L;;;;;N;;;;; +13A8A;EGYPTIAN HIEROGLYPH-13A8A;Lo;0;L;;;;;N;;;;; +13A8B;EGYPTIAN HIEROGLYPH-13A8B;Lo;0;L;;;;;N;;;;; +13A8C;EGYPTIAN HIEROGLYPH-13A8C;Lo;0;L;;;;;N;;;;; +13A8D;EGYPTIAN HIEROGLYPH-13A8D;Lo;0;L;;;;;N;;;;; +13A8E;EGYPTIAN HIEROGLYPH-13A8E;Lo;0;L;;;;;N;;;;; +13A8F;EGYPTIAN HIEROGLYPH-13A8F;Lo;0;L;;;;;N;;;;; +13A90;EGYPTIAN HIEROGLYPH-13A90;Lo;0;L;;;;;N;;;;; +13A91;EGYPTIAN HIEROGLYPH-13A91;Lo;0;L;;;;;N;;;;; +13A92;EGYPTIAN HIEROGLYPH-13A92;Lo;0;L;;;;;N;;;;; +13A93;EGYPTIAN HIEROGLYPH-13A93;Lo;0;L;;;;;N;;;;; +13A94;EGYPTIAN HIEROGLYPH-13A94;Lo;0;L;;;;;N;;;;; +13A95;EGYPTIAN HIEROGLYPH-13A95;Lo;0;L;;;;;N;;;;; +13A96;EGYPTIAN HIEROGLYPH-13A96;Lo;0;L;;;;;N;;;;; +13A97;EGYPTIAN HIEROGLYPH-13A97;Lo;0;L;;;;;N;;;;; +13A98;EGYPTIAN HIEROGLYPH-13A98;Lo;0;L;;;;;N;;;;; +13A99;EGYPTIAN HIEROGLYPH-13A99;Lo;0;L;;;;;N;;;;; +13A9A;EGYPTIAN HIEROGLYPH-13A9A;Lo;0;L;;;;;N;;;;; +13A9B;EGYPTIAN HIEROGLYPH-13A9B;Lo;0;L;;;;;N;;;;; +13A9C;EGYPTIAN HIEROGLYPH-13A9C;Lo;0;L;;;;;N;;;;; +13A9D;EGYPTIAN HIEROGLYPH-13A9D;Lo;0;L;;;;;N;;;;; +13A9E;EGYPTIAN HIEROGLYPH-13A9E;Lo;0;L;;;;;N;;;;; +13A9F;EGYPTIAN HIEROGLYPH-13A9F;Lo;0;L;;;;;N;;;;; +13AA0;EGYPTIAN HIEROGLYPH-13AA0;Lo;0;L;;;;;N;;;;; +13AA1;EGYPTIAN HIEROGLYPH-13AA1;Lo;0;L;;;;;N;;;;; +13AA2;EGYPTIAN HIEROGLYPH-13AA2;Lo;0;L;;;;;N;;;;; +13AA3;EGYPTIAN HIEROGLYPH-13AA3;Lo;0;L;;;;;N;;;;; +13AA4;EGYPTIAN HIEROGLYPH-13AA4;Lo;0;L;;;;;N;;;;; +13AA5;EGYPTIAN HIEROGLYPH-13AA5;Lo;0;L;;;;;N;;;;; +13AA6;EGYPTIAN HIEROGLYPH-13AA6;Lo;0;L;;;;;N;;;;; +13AA7;EGYPTIAN HIEROGLYPH-13AA7;Lo;0;L;;;;;N;;;;; +13AA8;EGYPTIAN HIEROGLYPH-13AA8;Lo;0;L;;;;;N;;;;; +13AA9;EGYPTIAN HIEROGLYPH-13AA9;Lo;0;L;;;;;N;;;;; +13AAA;EGYPTIAN HIEROGLYPH-13AAA;Lo;0;L;;;;;N;;;;; +13AAB;EGYPTIAN HIEROGLYPH-13AAB;Lo;0;L;;;;;N;;;;; +13AAC;EGYPTIAN HIEROGLYPH-13AAC;Lo;0;L;;;;;N;;;;; +13AAD;EGYPTIAN HIEROGLYPH-13AAD;Lo;0;L;;;;;N;;;;; +13AAE;EGYPTIAN HIEROGLYPH-13AAE;Lo;0;L;;;;;N;;;;; +13AAF;EGYPTIAN HIEROGLYPH-13AAF;Lo;0;L;;;;;N;;;;; +13AB0;EGYPTIAN HIEROGLYPH-13AB0;Lo;0;L;;;;;N;;;;; +13AB1;EGYPTIAN HIEROGLYPH-13AB1;Lo;0;L;;;;;N;;;;; +13AB2;EGYPTIAN HIEROGLYPH-13AB2;Lo;0;L;;;;;N;;;;; +13AB3;EGYPTIAN HIEROGLYPH-13AB3;Lo;0;L;;;;;N;;;;; +13AB4;EGYPTIAN HIEROGLYPH-13AB4;Lo;0;L;;;;;N;;;;; +13AB5;EGYPTIAN HIEROGLYPH-13AB5;Lo;0;L;;;;;N;;;;; +13AB6;EGYPTIAN HIEROGLYPH-13AB6;Lo;0;L;;;;;N;;;;; +13AB7;EGYPTIAN HIEROGLYPH-13AB7;Lo;0;L;;;;;N;;;;; +13AB8;EGYPTIAN HIEROGLYPH-13AB8;Lo;0;L;;;;;N;;;;; +13AB9;EGYPTIAN HIEROGLYPH-13AB9;Lo;0;L;;;;;N;;;;; +13ABA;EGYPTIAN HIEROGLYPH-13ABA;Lo;0;L;;;;;N;;;;; +13ABB;EGYPTIAN HIEROGLYPH-13ABB;Lo;0;L;;;;;N;;;;; +13ABC;EGYPTIAN HIEROGLYPH-13ABC;Lo;0;L;;;;;N;;;;; +13ABD;EGYPTIAN HIEROGLYPH-13ABD;Lo;0;L;;;;;N;;;;; +13ABE;EGYPTIAN HIEROGLYPH-13ABE;Lo;0;L;;;;;N;;;;; +13ABF;EGYPTIAN HIEROGLYPH-13ABF;Lo;0;L;;;;;N;;;;; +13AC0;EGYPTIAN HIEROGLYPH-13AC0;Lo;0;L;;;;;N;;;;; +13AC1;EGYPTIAN HIEROGLYPH-13AC1;Lo;0;L;;;;;N;;;;; +13AC2;EGYPTIAN HIEROGLYPH-13AC2;Lo;0;L;;;;;N;;;;; +13AC3;EGYPTIAN HIEROGLYPH-13AC3;Lo;0;L;;;;;N;;;;; +13AC4;EGYPTIAN HIEROGLYPH-13AC4;Lo;0;L;;;;;N;;;;; +13AC5;EGYPTIAN HIEROGLYPH-13AC5;Lo;0;L;;;;;N;;;;; +13AC6;EGYPTIAN HIEROGLYPH-13AC6;Lo;0;L;;;;;N;;;;; +13AC7;EGYPTIAN HIEROGLYPH-13AC7;Lo;0;L;;;;;N;;;;; +13AC8;EGYPTIAN HIEROGLYPH-13AC8;Lo;0;L;;;;;N;;;;; +13AC9;EGYPTIAN HIEROGLYPH-13AC9;Lo;0;L;;;;;N;;;;; +13ACA;EGYPTIAN HIEROGLYPH-13ACA;Lo;0;L;;;;;N;;;;; +13ACB;EGYPTIAN HIEROGLYPH-13ACB;Lo;0;L;;;;;N;;;;; +13ACC;EGYPTIAN HIEROGLYPH-13ACC;Lo;0;L;;;;;N;;;;; +13ACD;EGYPTIAN HIEROGLYPH-13ACD;Lo;0;L;;;;;N;;;;; +13ACE;EGYPTIAN HIEROGLYPH-13ACE;Lo;0;L;;;;;N;;;;; +13ACF;EGYPTIAN HIEROGLYPH-13ACF;Lo;0;L;;;;;N;;;;; +13AD0;EGYPTIAN HIEROGLYPH-13AD0;Lo;0;L;;;;;N;;;;; +13AD1;EGYPTIAN HIEROGLYPH-13AD1;Lo;0;L;;;;;N;;;;; +13AD2;EGYPTIAN HIEROGLYPH-13AD2;Lo;0;L;;;;;N;;;;; +13AD3;EGYPTIAN HIEROGLYPH-13AD3;Lo;0;L;;;;;N;;;;; +13AD4;EGYPTIAN HIEROGLYPH-13AD4;Lo;0;L;;;;;N;;;;; +13AD5;EGYPTIAN HIEROGLYPH-13AD5;Lo;0;L;;;;;N;;;;; +13AD6;EGYPTIAN HIEROGLYPH-13AD6;Lo;0;L;;;;;N;;;;; +13AD7;EGYPTIAN HIEROGLYPH-13AD7;Lo;0;L;;;;;N;;;;; +13AD8;EGYPTIAN HIEROGLYPH-13AD8;Lo;0;L;;;;;N;;;;; +13AD9;EGYPTIAN HIEROGLYPH-13AD9;Lo;0;L;;;;;N;;;;; +13ADA;EGYPTIAN HIEROGLYPH-13ADA;Lo;0;L;;;;;N;;;;; +13ADB;EGYPTIAN HIEROGLYPH-13ADB;Lo;0;L;;;;;N;;;;; +13ADC;EGYPTIAN HIEROGLYPH-13ADC;Lo;0;L;;;;;N;;;;; +13ADD;EGYPTIAN HIEROGLYPH-13ADD;Lo;0;L;;;;;N;;;;; +13ADE;EGYPTIAN HIEROGLYPH-13ADE;Lo;0;L;;;;;N;;;;; +13ADF;EGYPTIAN HIEROGLYPH-13ADF;Lo;0;L;;;;;N;;;;; +13AE0;EGYPTIAN HIEROGLYPH-13AE0;Lo;0;L;;;;;N;;;;; +13AE1;EGYPTIAN HIEROGLYPH-13AE1;Lo;0;L;;;;;N;;;;; +13AE2;EGYPTIAN HIEROGLYPH-13AE2;Lo;0;L;;;;;N;;;;; +13AE3;EGYPTIAN HIEROGLYPH-13AE3;Lo;0;L;;;;;N;;;;; +13AE4;EGYPTIAN HIEROGLYPH-13AE4;Lo;0;L;;;;;N;;;;; +13AE5;EGYPTIAN HIEROGLYPH-13AE5;Lo;0;L;;;;;N;;;;; +13AE6;EGYPTIAN HIEROGLYPH-13AE6;Lo;0;L;;;;;N;;;;; +13AE7;EGYPTIAN HIEROGLYPH-13AE7;Lo;0;L;;;;;N;;;;; +13AE8;EGYPTIAN HIEROGLYPH-13AE8;Lo;0;L;;;;;N;;;;; +13AE9;EGYPTIAN HIEROGLYPH-13AE9;Lo;0;L;;;;;N;;;;; +13AEA;EGYPTIAN HIEROGLYPH-13AEA;Lo;0;L;;;;;N;;;;; +13AEB;EGYPTIAN HIEROGLYPH-13AEB;Lo;0;L;;;;;N;;;;; +13AEC;EGYPTIAN HIEROGLYPH-13AEC;Lo;0;L;;;;;N;;;;; +13AED;EGYPTIAN HIEROGLYPH-13AED;Lo;0;L;;;;;N;;;;; +13AEE;EGYPTIAN HIEROGLYPH-13AEE;Lo;0;L;;;;;N;;;;; +13AEF;EGYPTIAN HIEROGLYPH-13AEF;Lo;0;L;;;;;N;;;;; +13AF0;EGYPTIAN HIEROGLYPH-13AF0;Lo;0;L;;;;;N;;;;; +13AF1;EGYPTIAN HIEROGLYPH-13AF1;Lo;0;L;;;;;N;;;;; +13AF2;EGYPTIAN HIEROGLYPH-13AF2;Lo;0;L;;;;;N;;;;; +13AF3;EGYPTIAN HIEROGLYPH-13AF3;Lo;0;L;;;;;N;;;;; +13AF4;EGYPTIAN HIEROGLYPH-13AF4;Lo;0;L;;;;;N;;;;; +13AF5;EGYPTIAN HIEROGLYPH-13AF5;Lo;0;L;;;;;N;;;;; +13AF6;EGYPTIAN HIEROGLYPH-13AF6;Lo;0;L;;;;;N;;;;; +13AF7;EGYPTIAN HIEROGLYPH-13AF7;Lo;0;L;;;;;N;;;;; +13AF8;EGYPTIAN HIEROGLYPH-13AF8;Lo;0;L;;;;;N;;;;; +13AF9;EGYPTIAN HIEROGLYPH-13AF9;Lo;0;L;;;;;N;;;;; +13AFA;EGYPTIAN HIEROGLYPH-13AFA;Lo;0;L;;;;;N;;;;; +13AFB;EGYPTIAN HIEROGLYPH-13AFB;Lo;0;L;;;;;N;;;;; +13AFC;EGYPTIAN HIEROGLYPH-13AFC;Lo;0;L;;;;;N;;;;; +13AFD;EGYPTIAN HIEROGLYPH-13AFD;Lo;0;L;;;;;N;;;;; +13AFE;EGYPTIAN HIEROGLYPH-13AFE;Lo;0;L;;;;;N;;;;; +13AFF;EGYPTIAN HIEROGLYPH-13AFF;Lo;0;L;;;;;N;;;;; +13B00;EGYPTIAN HIEROGLYPH-13B00;Lo;0;L;;;;;N;;;;; +13B01;EGYPTIAN HIEROGLYPH-13B01;Lo;0;L;;;;;N;;;;; +13B02;EGYPTIAN HIEROGLYPH-13B02;Lo;0;L;;;;;N;;;;; +13B03;EGYPTIAN HIEROGLYPH-13B03;Lo;0;L;;;;;N;;;;; +13B04;EGYPTIAN HIEROGLYPH-13B04;Lo;0;L;;;;;N;;;;; +13B05;EGYPTIAN HIEROGLYPH-13B05;Lo;0;L;;;;;N;;;;; +13B06;EGYPTIAN HIEROGLYPH-13B06;Lo;0;L;;;;;N;;;;; +13B07;EGYPTIAN HIEROGLYPH-13B07;Lo;0;L;;;;;N;;;;; +13B08;EGYPTIAN HIEROGLYPH-13B08;Lo;0;L;;;;;N;;;;; +13B09;EGYPTIAN HIEROGLYPH-13B09;Lo;0;L;;;;;N;;;;; +13B0A;EGYPTIAN HIEROGLYPH-13B0A;Lo;0;L;;;;;N;;;;; +13B0B;EGYPTIAN HIEROGLYPH-13B0B;Lo;0;L;;;;;N;;;;; +13B0C;EGYPTIAN HIEROGLYPH-13B0C;Lo;0;L;;;;;N;;;;; +13B0D;EGYPTIAN HIEROGLYPH-13B0D;Lo;0;L;;;;;N;;;;; +13B0E;EGYPTIAN HIEROGLYPH-13B0E;Lo;0;L;;;;;N;;;;; +13B0F;EGYPTIAN HIEROGLYPH-13B0F;Lo;0;L;;;;;N;;;;; +13B10;EGYPTIAN HIEROGLYPH-13B10;Lo;0;L;;;;;N;;;;; +13B11;EGYPTIAN HIEROGLYPH-13B11;Lo;0;L;;;;;N;;;;; +13B12;EGYPTIAN HIEROGLYPH-13B12;Lo;0;L;;;;;N;;;;; +13B13;EGYPTIAN HIEROGLYPH-13B13;Lo;0;L;;;;;N;;;;; +13B14;EGYPTIAN HIEROGLYPH-13B14;Lo;0;L;;;;;N;;;;; +13B15;EGYPTIAN HIEROGLYPH-13B15;Lo;0;L;;;;;N;;;;; +13B16;EGYPTIAN HIEROGLYPH-13B16;Lo;0;L;;;;;N;;;;; +13B17;EGYPTIAN HIEROGLYPH-13B17;Lo;0;L;;;;;N;;;;; +13B18;EGYPTIAN HIEROGLYPH-13B18;Lo;0;L;;;;;N;;;;; +13B19;EGYPTIAN HIEROGLYPH-13B19;Lo;0;L;;;;;N;;;;; +13B1A;EGYPTIAN HIEROGLYPH-13B1A;Lo;0;L;;;;;N;;;;; +13B1B;EGYPTIAN HIEROGLYPH-13B1B;Lo;0;L;;;;;N;;;;; +13B1C;EGYPTIAN HIEROGLYPH-13B1C;Lo;0;L;;;;;N;;;;; +13B1D;EGYPTIAN HIEROGLYPH-13B1D;Lo;0;L;;;;;N;;;;; +13B1E;EGYPTIAN HIEROGLYPH-13B1E;Lo;0;L;;;;;N;;;;; +13B1F;EGYPTIAN HIEROGLYPH-13B1F;Lo;0;L;;;;;N;;;;; +13B20;EGYPTIAN HIEROGLYPH-13B20;Lo;0;L;;;;;N;;;;; +13B21;EGYPTIAN HIEROGLYPH-13B21;Lo;0;L;;;;;N;;;;; +13B22;EGYPTIAN HIEROGLYPH-13B22;Lo;0;L;;;;;N;;;;; +13B23;EGYPTIAN HIEROGLYPH-13B23;Lo;0;L;;;;;N;;;;; +13B24;EGYPTIAN HIEROGLYPH-13B24;Lo;0;L;;;;;N;;;;; +13B25;EGYPTIAN HIEROGLYPH-13B25;Lo;0;L;;;;;N;;;;; +13B26;EGYPTIAN HIEROGLYPH-13B26;Lo;0;L;;;;;N;;;;; +13B27;EGYPTIAN HIEROGLYPH-13B27;Lo;0;L;;;;;N;;;;; +13B28;EGYPTIAN HIEROGLYPH-13B28;Lo;0;L;;;;;N;;;;; +13B29;EGYPTIAN HIEROGLYPH-13B29;Lo;0;L;;;;;N;;;;; +13B2A;EGYPTIAN HIEROGLYPH-13B2A;Lo;0;L;;;;;N;;;;; +13B2B;EGYPTIAN HIEROGLYPH-13B2B;Lo;0;L;;;;;N;;;;; +13B2C;EGYPTIAN HIEROGLYPH-13B2C;Lo;0;L;;;;;N;;;;; +13B2D;EGYPTIAN HIEROGLYPH-13B2D;Lo;0;L;;;;;N;;;;; +13B2E;EGYPTIAN HIEROGLYPH-13B2E;Lo;0;L;;;;;N;;;;; +13B2F;EGYPTIAN HIEROGLYPH-13B2F;Lo;0;L;;;;;N;;;;; +13B30;EGYPTIAN HIEROGLYPH-13B30;Lo;0;L;;;;;N;;;;; +13B31;EGYPTIAN HIEROGLYPH-13B31;Lo;0;L;;;;;N;;;;; +13B32;EGYPTIAN HIEROGLYPH-13B32;Lo;0;L;;;;;N;;;;; +13B33;EGYPTIAN HIEROGLYPH-13B33;Lo;0;L;;;;;N;;;;; +13B34;EGYPTIAN HIEROGLYPH-13B34;Lo;0;L;;;;;N;;;;; +13B35;EGYPTIAN HIEROGLYPH-13B35;Lo;0;L;;;;;N;;;;; +13B36;EGYPTIAN HIEROGLYPH-13B36;Lo;0;L;;;;;N;;;;; +13B37;EGYPTIAN HIEROGLYPH-13B37;Lo;0;L;;;;;N;;;;; +13B38;EGYPTIAN HIEROGLYPH-13B38;Lo;0;L;;;;;N;;;;; +13B39;EGYPTIAN HIEROGLYPH-13B39;Lo;0;L;;;;;N;;;;; +13B3A;EGYPTIAN HIEROGLYPH-13B3A;Lo;0;L;;;;;N;;;;; +13B3B;EGYPTIAN HIEROGLYPH-13B3B;Lo;0;L;;;;;N;;;;; +13B3C;EGYPTIAN HIEROGLYPH-13B3C;Lo;0;L;;;;;N;;;;; +13B3D;EGYPTIAN HIEROGLYPH-13B3D;Lo;0;L;;;;;N;;;;; +13B3E;EGYPTIAN HIEROGLYPH-13B3E;Lo;0;L;;;;;N;;;;; +13B3F;EGYPTIAN HIEROGLYPH-13B3F;Lo;0;L;;;;;N;;;;; +13B40;EGYPTIAN HIEROGLYPH-13B40;Lo;0;L;;;;;N;;;;; +13B41;EGYPTIAN HIEROGLYPH-13B41;Lo;0;L;;;;;N;;;;; +13B42;EGYPTIAN HIEROGLYPH-13B42;Lo;0;L;;;;;N;;;;; +13B43;EGYPTIAN HIEROGLYPH-13B43;Lo;0;L;;;;;N;;;;; +13B44;EGYPTIAN HIEROGLYPH-13B44;Lo;0;L;;;;;N;;;;; +13B45;EGYPTIAN HIEROGLYPH-13B45;Lo;0;L;;;;;N;;;;; +13B46;EGYPTIAN HIEROGLYPH-13B46;Lo;0;L;;;;;N;;;;; +13B47;EGYPTIAN HIEROGLYPH-13B47;Lo;0;L;;;;;N;;;;; +13B48;EGYPTIAN HIEROGLYPH-13B48;Lo;0;L;;;;;N;;;;; +13B49;EGYPTIAN HIEROGLYPH-13B49;Lo;0;L;;;;;N;;;;; +13B4A;EGYPTIAN HIEROGLYPH-13B4A;Lo;0;L;;;;;N;;;;; +13B4B;EGYPTIAN HIEROGLYPH-13B4B;Lo;0;L;;;;;N;;;;; +13B4C;EGYPTIAN HIEROGLYPH-13B4C;Lo;0;L;;;;;N;;;;; +13B4D;EGYPTIAN HIEROGLYPH-13B4D;Lo;0;L;;;;;N;;;;; +13B4E;EGYPTIAN HIEROGLYPH-13B4E;Lo;0;L;;;;;N;;;;; +13B4F;EGYPTIAN HIEROGLYPH-13B4F;Lo;0;L;;;;;N;;;;; +13B50;EGYPTIAN HIEROGLYPH-13B50;Lo;0;L;;;;;N;;;;; +13B51;EGYPTIAN HIEROGLYPH-13B51;Lo;0;L;;;;;N;;;;; +13B52;EGYPTIAN HIEROGLYPH-13B52;Lo;0;L;;;;;N;;;;; +13B53;EGYPTIAN HIEROGLYPH-13B53;Lo;0;L;;;;;N;;;;; +13B54;EGYPTIAN HIEROGLYPH-13B54;Lo;0;L;;;;;N;;;;; +13B55;EGYPTIAN HIEROGLYPH-13B55;Lo;0;L;;;;;N;;;;; +13B56;EGYPTIAN HIEROGLYPH-13B56;Lo;0;L;;;;;N;;;;; +13B57;EGYPTIAN HIEROGLYPH-13B57;Lo;0;L;;;;;N;;;;; +13B58;EGYPTIAN HIEROGLYPH-13B58;Lo;0;L;;;;;N;;;;; +13B59;EGYPTIAN HIEROGLYPH-13B59;Lo;0;L;;;;;N;;;;; +13B5A;EGYPTIAN HIEROGLYPH-13B5A;Lo;0;L;;;;;N;;;;; +13B5B;EGYPTIAN HIEROGLYPH-13B5B;Lo;0;L;;;;;N;;;;; +13B5C;EGYPTIAN HIEROGLYPH-13B5C;Lo;0;L;;;;;N;;;;; +13B5D;EGYPTIAN HIEROGLYPH-13B5D;Lo;0;L;;;;;N;;;;; +13B5E;EGYPTIAN HIEROGLYPH-13B5E;Lo;0;L;;;;;N;;;;; +13B5F;EGYPTIAN HIEROGLYPH-13B5F;Lo;0;L;;;;;N;;;;; +13B60;EGYPTIAN HIEROGLYPH-13B60;Lo;0;L;;;;;N;;;;; +13B61;EGYPTIAN HIEROGLYPH-13B61;Lo;0;L;;;;;N;;;;; +13B62;EGYPTIAN HIEROGLYPH-13B62;Lo;0;L;;;;;N;;;;; +13B63;EGYPTIAN HIEROGLYPH-13B63;Lo;0;L;;;;;N;;;;; +13B64;EGYPTIAN HIEROGLYPH-13B64;Lo;0;L;;;;;N;;;;; +13B65;EGYPTIAN HIEROGLYPH-13B65;Lo;0;L;;;;;N;;;;; +13B66;EGYPTIAN HIEROGLYPH-13B66;Lo;0;L;;;;;N;;;;; +13B67;EGYPTIAN HIEROGLYPH-13B67;Lo;0;L;;;;;N;;;;; +13B68;EGYPTIAN HIEROGLYPH-13B68;Lo;0;L;;;;;N;;;;; +13B69;EGYPTIAN HIEROGLYPH-13B69;Lo;0;L;;;;;N;;;;; +13B6A;EGYPTIAN HIEROGLYPH-13B6A;Lo;0;L;;;;;N;;;;; +13B6B;EGYPTIAN HIEROGLYPH-13B6B;Lo;0;L;;;;;N;;;;; +13B6C;EGYPTIAN HIEROGLYPH-13B6C;Lo;0;L;;;;;N;;;;; +13B6D;EGYPTIAN HIEROGLYPH-13B6D;Lo;0;L;;;;;N;;;;; +13B6E;EGYPTIAN HIEROGLYPH-13B6E;Lo;0;L;;;;;N;;;;; +13B6F;EGYPTIAN HIEROGLYPH-13B6F;Lo;0;L;;;;;N;;;;; +13B70;EGYPTIAN HIEROGLYPH-13B70;Lo;0;L;;;;;N;;;;; +13B71;EGYPTIAN HIEROGLYPH-13B71;Lo;0;L;;;;;N;;;;; +13B72;EGYPTIAN HIEROGLYPH-13B72;Lo;0;L;;;;;N;;;;; +13B73;EGYPTIAN HIEROGLYPH-13B73;Lo;0;L;;;;;N;;;;; +13B74;EGYPTIAN HIEROGLYPH-13B74;Lo;0;L;;;;;N;;;;; +13B75;EGYPTIAN HIEROGLYPH-13B75;Lo;0;L;;;;;N;;;;; +13B76;EGYPTIAN HIEROGLYPH-13B76;Lo;0;L;;;;;N;;;;; +13B77;EGYPTIAN HIEROGLYPH-13B77;Lo;0;L;;;;;N;;;;; +13B78;EGYPTIAN HIEROGLYPH-13B78;Lo;0;L;;;;;N;;;;; +13B79;EGYPTIAN HIEROGLYPH-13B79;Lo;0;L;;;;;N;;;;; +13B7A;EGYPTIAN HIEROGLYPH-13B7A;Lo;0;L;;;;;N;;;;; +13B7B;EGYPTIAN HIEROGLYPH-13B7B;Lo;0;L;;;;;N;;;;; +13B7C;EGYPTIAN HIEROGLYPH-13B7C;Lo;0;L;;;;;N;;;;; +13B7D;EGYPTIAN HIEROGLYPH-13B7D;Lo;0;L;;;;;N;;;;; +13B7E;EGYPTIAN HIEROGLYPH-13B7E;Lo;0;L;;;;;N;;;;; +13B7F;EGYPTIAN HIEROGLYPH-13B7F;Lo;0;L;;;;;N;;;;; +13B80;EGYPTIAN HIEROGLYPH-13B80;Lo;0;L;;;;;N;;;;; +13B81;EGYPTIAN HIEROGLYPH-13B81;Lo;0;L;;;;;N;;;;; +13B82;EGYPTIAN HIEROGLYPH-13B82;Lo;0;L;;;;;N;;;;; +13B83;EGYPTIAN HIEROGLYPH-13B83;Lo;0;L;;;;;N;;;;; +13B84;EGYPTIAN HIEROGLYPH-13B84;Lo;0;L;;;;;N;;;;; +13B85;EGYPTIAN HIEROGLYPH-13B85;Lo;0;L;;;;;N;;;;; +13B86;EGYPTIAN HIEROGLYPH-13B86;Lo;0;L;;;;;N;;;;; +13B87;EGYPTIAN HIEROGLYPH-13B87;Lo;0;L;;;;;N;;;;; +13B88;EGYPTIAN HIEROGLYPH-13B88;Lo;0;L;;;;;N;;;;; +13B89;EGYPTIAN HIEROGLYPH-13B89;Lo;0;L;;;;;N;;;;; +13B8A;EGYPTIAN HIEROGLYPH-13B8A;Lo;0;L;;;;;N;;;;; +13B8B;EGYPTIAN HIEROGLYPH-13B8B;Lo;0;L;;;;;N;;;;; +13B8C;EGYPTIAN HIEROGLYPH-13B8C;Lo;0;L;;;;;N;;;;; +13B8D;EGYPTIAN HIEROGLYPH-13B8D;Lo;0;L;;;;;N;;;;; +13B8E;EGYPTIAN HIEROGLYPH-13B8E;Lo;0;L;;;;;N;;;;; +13B8F;EGYPTIAN HIEROGLYPH-13B8F;Lo;0;L;;;;;N;;;;; +13B90;EGYPTIAN HIEROGLYPH-13B90;Lo;0;L;;;;;N;;;;; +13B91;EGYPTIAN HIEROGLYPH-13B91;Lo;0;L;;;;;N;;;;; +13B92;EGYPTIAN HIEROGLYPH-13B92;Lo;0;L;;;;;N;;;;; +13B93;EGYPTIAN HIEROGLYPH-13B93;Lo;0;L;;;;;N;;;;; +13B94;EGYPTIAN HIEROGLYPH-13B94;Lo;0;L;;;;;N;;;;; +13B95;EGYPTIAN HIEROGLYPH-13B95;Lo;0;L;;;;;N;;;;; +13B96;EGYPTIAN HIEROGLYPH-13B96;Lo;0;L;;;;;N;;;;; +13B97;EGYPTIAN HIEROGLYPH-13B97;Lo;0;L;;;;;N;;;;; +13B98;EGYPTIAN HIEROGLYPH-13B98;Lo;0;L;;;;;N;;;;; +13B99;EGYPTIAN HIEROGLYPH-13B99;Lo;0;L;;;;;N;;;;; +13B9A;EGYPTIAN HIEROGLYPH-13B9A;Lo;0;L;;;;;N;;;;; +13B9B;EGYPTIAN HIEROGLYPH-13B9B;Lo;0;L;;;;;N;;;;; +13B9C;EGYPTIAN HIEROGLYPH-13B9C;Lo;0;L;;;;;N;;;;; +13B9D;EGYPTIAN HIEROGLYPH-13B9D;Lo;0;L;;;;;N;;;;; +13B9E;EGYPTIAN HIEROGLYPH-13B9E;Lo;0;L;;;;;N;;;;; +13B9F;EGYPTIAN HIEROGLYPH-13B9F;Lo;0;L;;;;;N;;;;; +13BA0;EGYPTIAN HIEROGLYPH-13BA0;Lo;0;L;;;;;N;;;;; +13BA1;EGYPTIAN HIEROGLYPH-13BA1;Lo;0;L;;;;;N;;;;; +13BA2;EGYPTIAN HIEROGLYPH-13BA2;Lo;0;L;;;;;N;;;;; +13BA3;EGYPTIAN HIEROGLYPH-13BA3;Lo;0;L;;;;;N;;;;; +13BA4;EGYPTIAN HIEROGLYPH-13BA4;Lo;0;L;;;;;N;;;;; +13BA5;EGYPTIAN HIEROGLYPH-13BA5;Lo;0;L;;;;;N;;;;; +13BA6;EGYPTIAN HIEROGLYPH-13BA6;Lo;0;L;;;;;N;;;;; +13BA7;EGYPTIAN HIEROGLYPH-13BA7;Lo;0;L;;;;;N;;;;; +13BA8;EGYPTIAN HIEROGLYPH-13BA8;Lo;0;L;;;;;N;;;;; +13BA9;EGYPTIAN HIEROGLYPH-13BA9;Lo;0;L;;;;;N;;;;; +13BAA;EGYPTIAN HIEROGLYPH-13BAA;Lo;0;L;;;;;N;;;;; +13BAB;EGYPTIAN HIEROGLYPH-13BAB;Lo;0;L;;;;;N;;;;; +13BAC;EGYPTIAN HIEROGLYPH-13BAC;Lo;0;L;;;;;N;;;;; +13BAD;EGYPTIAN HIEROGLYPH-13BAD;Lo;0;L;;;;;N;;;;; +13BAE;EGYPTIAN HIEROGLYPH-13BAE;Lo;0;L;;;;;N;;;;; +13BAF;EGYPTIAN HIEROGLYPH-13BAF;Lo;0;L;;;;;N;;;;; +13BB0;EGYPTIAN HIEROGLYPH-13BB0;Lo;0;L;;;;;N;;;;; +13BB1;EGYPTIAN HIEROGLYPH-13BB1;Lo;0;L;;;;;N;;;;; +13BB2;EGYPTIAN HIEROGLYPH-13BB2;Lo;0;L;;;;;N;;;;; +13BB3;EGYPTIAN HIEROGLYPH-13BB3;Lo;0;L;;;;;N;;;;; +13BB4;EGYPTIAN HIEROGLYPH-13BB4;Lo;0;L;;;;;N;;;;; +13BB5;EGYPTIAN HIEROGLYPH-13BB5;Lo;0;L;;;;;N;;;;; +13BB6;EGYPTIAN HIEROGLYPH-13BB6;Lo;0;L;;;;;N;;;;; +13BB7;EGYPTIAN HIEROGLYPH-13BB7;Lo;0;L;;;;;N;;;;; +13BB8;EGYPTIAN HIEROGLYPH-13BB8;Lo;0;L;;;;;N;;;;; +13BB9;EGYPTIAN HIEROGLYPH-13BB9;Lo;0;L;;;;;N;;;;; +13BBA;EGYPTIAN HIEROGLYPH-13BBA;Lo;0;L;;;;;N;;;;; +13BBB;EGYPTIAN HIEROGLYPH-13BBB;Lo;0;L;;;;;N;;;;; +13BBC;EGYPTIAN HIEROGLYPH-13BBC;Lo;0;L;;;;;N;;;;; +13BBD;EGYPTIAN HIEROGLYPH-13BBD;Lo;0;L;;;;;N;;;;; +13BBE;EGYPTIAN HIEROGLYPH-13BBE;Lo;0;L;;;;;N;;;;; +13BBF;EGYPTIAN HIEROGLYPH-13BBF;Lo;0;L;;;;;N;;;;; +13BC0;EGYPTIAN HIEROGLYPH-13BC0;Lo;0;L;;;;;N;;;;; +13BC1;EGYPTIAN HIEROGLYPH-13BC1;Lo;0;L;;;;;N;;;;; +13BC2;EGYPTIAN HIEROGLYPH-13BC2;Lo;0;L;;;;;N;;;;; +13BC3;EGYPTIAN HIEROGLYPH-13BC3;Lo;0;L;;;;;N;;;;; +13BC4;EGYPTIAN HIEROGLYPH-13BC4;Lo;0;L;;;;;N;;;;; +13BC5;EGYPTIAN HIEROGLYPH-13BC5;Lo;0;L;;;;;N;;;;; +13BC6;EGYPTIAN HIEROGLYPH-13BC6;Lo;0;L;;;;;N;;;;; +13BC7;EGYPTIAN HIEROGLYPH-13BC7;Lo;0;L;;;;;N;;;;; +13BC8;EGYPTIAN HIEROGLYPH-13BC8;Lo;0;L;;;;;N;;;;; +13BC9;EGYPTIAN HIEROGLYPH-13BC9;Lo;0;L;;;;;N;;;;; +13BCA;EGYPTIAN HIEROGLYPH-13BCA;Lo;0;L;;;;;N;;;;; +13BCB;EGYPTIAN HIEROGLYPH-13BCB;Lo;0;L;;;;;N;;;;; +13BCC;EGYPTIAN HIEROGLYPH-13BCC;Lo;0;L;;;;;N;;;;; +13BCD;EGYPTIAN HIEROGLYPH-13BCD;Lo;0;L;;;;;N;;;;; +13BCE;EGYPTIAN HIEROGLYPH-13BCE;Lo;0;L;;;;;N;;;;; +13BCF;EGYPTIAN HIEROGLYPH-13BCF;Lo;0;L;;;;;N;;;;; +13BD0;EGYPTIAN HIEROGLYPH-13BD0;Lo;0;L;;;;;N;;;;; +13BD1;EGYPTIAN HIEROGLYPH-13BD1;Lo;0;L;;;;;N;;;;; +13BD2;EGYPTIAN HIEROGLYPH-13BD2;Lo;0;L;;;;;N;;;;; +13BD3;EGYPTIAN HIEROGLYPH-13BD3;Lo;0;L;;;;;N;;;;; +13BD4;EGYPTIAN HIEROGLYPH-13BD4;Lo;0;L;;;;;N;;;;; +13BD5;EGYPTIAN HIEROGLYPH-13BD5;Lo;0;L;;;;;N;;;;; +13BD6;EGYPTIAN HIEROGLYPH-13BD6;Lo;0;L;;;;;N;;;;; +13BD7;EGYPTIAN HIEROGLYPH-13BD7;Lo;0;L;;;;;N;;;;; +13BD8;EGYPTIAN HIEROGLYPH-13BD8;Lo;0;L;;;;;N;;;;; +13BD9;EGYPTIAN HIEROGLYPH-13BD9;Lo;0;L;;;;;N;;;;; +13BDA;EGYPTIAN HIEROGLYPH-13BDA;Lo;0;L;;;;;N;;;;; +13BDB;EGYPTIAN HIEROGLYPH-13BDB;Lo;0;L;;;;;N;;;;; +13BDC;EGYPTIAN HIEROGLYPH-13BDC;Lo;0;L;;;;;N;;;;; +13BDD;EGYPTIAN HIEROGLYPH-13BDD;Lo;0;L;;;;;N;;;;; +13BDE;EGYPTIAN HIEROGLYPH-13BDE;Lo;0;L;;;;;N;;;;; +13BDF;EGYPTIAN HIEROGLYPH-13BDF;Lo;0;L;;;;;N;;;;; +13BE0;EGYPTIAN HIEROGLYPH-13BE0;Lo;0;L;;;;;N;;;;; +13BE1;EGYPTIAN HIEROGLYPH-13BE1;Lo;0;L;;;;;N;;;;; +13BE2;EGYPTIAN HIEROGLYPH-13BE2;Lo;0;L;;;;;N;;;;; +13BE3;EGYPTIAN HIEROGLYPH-13BE3;Lo;0;L;;;;;N;;;;; +13BE4;EGYPTIAN HIEROGLYPH-13BE4;Lo;0;L;;;;;N;;;;; +13BE5;EGYPTIAN HIEROGLYPH-13BE5;Lo;0;L;;;;;N;;;;; +13BE6;EGYPTIAN HIEROGLYPH-13BE6;Lo;0;L;;;;;N;;;;; +13BE7;EGYPTIAN HIEROGLYPH-13BE7;Lo;0;L;;;;;N;;;;; +13BE8;EGYPTIAN HIEROGLYPH-13BE8;Lo;0;L;;;;;N;;;;; +13BE9;EGYPTIAN HIEROGLYPH-13BE9;Lo;0;L;;;;;N;;;;; +13BEA;EGYPTIAN HIEROGLYPH-13BEA;Lo;0;L;;;;;N;;;;; +13BEB;EGYPTIAN HIEROGLYPH-13BEB;Lo;0;L;;;;;N;;;;; +13BEC;EGYPTIAN HIEROGLYPH-13BEC;Lo;0;L;;;;;N;;;;; +13BED;EGYPTIAN HIEROGLYPH-13BED;Lo;0;L;;;;;N;;;;; +13BEE;EGYPTIAN HIEROGLYPH-13BEE;Lo;0;L;;;;;N;;;;; +13BEF;EGYPTIAN HIEROGLYPH-13BEF;Lo;0;L;;;;;N;;;;; +13BF0;EGYPTIAN HIEROGLYPH-13BF0;Lo;0;L;;;;;N;;;;; +13BF1;EGYPTIAN HIEROGLYPH-13BF1;Lo;0;L;;;;;N;;;;; +13BF2;EGYPTIAN HIEROGLYPH-13BF2;Lo;0;L;;;;;N;;;;; +13BF3;EGYPTIAN HIEROGLYPH-13BF3;Lo;0;L;;;;;N;;;;; +13BF4;EGYPTIAN HIEROGLYPH-13BF4;Lo;0;L;;;;;N;;;;; +13BF5;EGYPTIAN HIEROGLYPH-13BF5;Lo;0;L;;;;;N;;;;; +13BF6;EGYPTIAN HIEROGLYPH-13BF6;Lo;0;L;;;;;N;;;;; +13BF7;EGYPTIAN HIEROGLYPH-13BF7;Lo;0;L;;;;;N;;;;; +13BF8;EGYPTIAN HIEROGLYPH-13BF8;Lo;0;L;;;;;N;;;;; +13BF9;EGYPTIAN HIEROGLYPH-13BF9;Lo;0;L;;;;;N;;;;; +13BFA;EGYPTIAN HIEROGLYPH-13BFA;Lo;0;L;;;;;N;;;;; +13BFB;EGYPTIAN HIEROGLYPH-13BFB;Lo;0;L;;;;;N;;;;; +13BFC;EGYPTIAN HIEROGLYPH-13BFC;Lo;0;L;;;;;N;;;;; +13BFD;EGYPTIAN HIEROGLYPH-13BFD;Lo;0;L;;;;;N;;;;; +13BFE;EGYPTIAN HIEROGLYPH-13BFE;Lo;0;L;;;;;N;;;;; +13BFF;EGYPTIAN HIEROGLYPH-13BFF;Lo;0;L;;;;;N;;;;; +13C00;EGYPTIAN HIEROGLYPH-13C00;Lo;0;L;;;;;N;;;;; +13C01;EGYPTIAN HIEROGLYPH-13C01;Lo;0;L;;;;;N;;;;; +13C02;EGYPTIAN HIEROGLYPH-13C02;Lo;0;L;;;;;N;;;;; +13C03;EGYPTIAN HIEROGLYPH-13C03;Lo;0;L;;;;;N;;;;; +13C04;EGYPTIAN HIEROGLYPH-13C04;Lo;0;L;;;;;N;;;;; +13C05;EGYPTIAN HIEROGLYPH-13C05;Lo;0;L;;;;;N;;;;; +13C06;EGYPTIAN HIEROGLYPH-13C06;Lo;0;L;;;;;N;;;;; +13C07;EGYPTIAN HIEROGLYPH-13C07;Lo;0;L;;;;;N;;;;; +13C08;EGYPTIAN HIEROGLYPH-13C08;Lo;0;L;;;;;N;;;;; +13C09;EGYPTIAN HIEROGLYPH-13C09;Lo;0;L;;;;;N;;;;; +13C0A;EGYPTIAN HIEROGLYPH-13C0A;Lo;0;L;;;;;N;;;;; +13C0B;EGYPTIAN HIEROGLYPH-13C0B;Lo;0;L;;;;;N;;;;; +13C0C;EGYPTIAN HIEROGLYPH-13C0C;Lo;0;L;;;;;N;;;;; +13C0D;EGYPTIAN HIEROGLYPH-13C0D;Lo;0;L;;;;;N;;;;; +13C0E;EGYPTIAN HIEROGLYPH-13C0E;Lo;0;L;;;;;N;;;;; +13C0F;EGYPTIAN HIEROGLYPH-13C0F;Lo;0;L;;;;;N;;;;; +13C10;EGYPTIAN HIEROGLYPH-13C10;Lo;0;L;;;;;N;;;;; +13C11;EGYPTIAN HIEROGLYPH-13C11;Lo;0;L;;;;;N;;;;; +13C12;EGYPTIAN HIEROGLYPH-13C12;Lo;0;L;;;;;N;;;;; +13C13;EGYPTIAN HIEROGLYPH-13C13;Lo;0;L;;;;;N;;;;; +13C14;EGYPTIAN HIEROGLYPH-13C14;Lo;0;L;;;;;N;;;;; +13C15;EGYPTIAN HIEROGLYPH-13C15;Lo;0;L;;;;;N;;;;; +13C16;EGYPTIAN HIEROGLYPH-13C16;Lo;0;L;;;;;N;;;;; +13C17;EGYPTIAN HIEROGLYPH-13C17;Lo;0;L;;;;;N;;;;; +13C18;EGYPTIAN HIEROGLYPH-13C18;Lo;0;L;;;;;N;;;;; +13C19;EGYPTIAN HIEROGLYPH-13C19;Lo;0;L;;;;;N;;;;; +13C1A;EGYPTIAN HIEROGLYPH-13C1A;Lo;0;L;;;;;N;;;;; +13C1B;EGYPTIAN HIEROGLYPH-13C1B;Lo;0;L;;;;;N;;;;; +13C1C;EGYPTIAN HIEROGLYPH-13C1C;Lo;0;L;;;;;N;;;;; +13C1D;EGYPTIAN HIEROGLYPH-13C1D;Lo;0;L;;;;;N;;;;; +13C1E;EGYPTIAN HIEROGLYPH-13C1E;Lo;0;L;;;;;N;;;;; +13C1F;EGYPTIAN HIEROGLYPH-13C1F;Lo;0;L;;;;;N;;;;; +13C20;EGYPTIAN HIEROGLYPH-13C20;Lo;0;L;;;;;N;;;;; +13C21;EGYPTIAN HIEROGLYPH-13C21;Lo;0;L;;;;;N;;;;; +13C22;EGYPTIAN HIEROGLYPH-13C22;Lo;0;L;;;;;N;;;;; +13C23;EGYPTIAN HIEROGLYPH-13C23;Lo;0;L;;;;;N;;;;; +13C24;EGYPTIAN HIEROGLYPH-13C24;Lo;0;L;;;;;N;;;;; +13C25;EGYPTIAN HIEROGLYPH-13C25;Lo;0;L;;;;;N;;;;; +13C26;EGYPTIAN HIEROGLYPH-13C26;Lo;0;L;;;;;N;;;;; +13C27;EGYPTIAN HIEROGLYPH-13C27;Lo;0;L;;;;;N;;;;; +13C28;EGYPTIAN HIEROGLYPH-13C28;Lo;0;L;;;;;N;;;;; +13C29;EGYPTIAN HIEROGLYPH-13C29;Lo;0;L;;;;;N;;;;; +13C2A;EGYPTIAN HIEROGLYPH-13C2A;Lo;0;L;;;;;N;;;;; +13C2B;EGYPTIAN HIEROGLYPH-13C2B;Lo;0;L;;;;;N;;;;; +13C2C;EGYPTIAN HIEROGLYPH-13C2C;Lo;0;L;;;;;N;;;;; +13C2D;EGYPTIAN HIEROGLYPH-13C2D;Lo;0;L;;;;;N;;;;; +13C2E;EGYPTIAN HIEROGLYPH-13C2E;Lo;0;L;;;;;N;;;;; +13C2F;EGYPTIAN HIEROGLYPH-13C2F;Lo;0;L;;;;;N;;;;; +13C30;EGYPTIAN HIEROGLYPH-13C30;Lo;0;L;;;;;N;;;;; +13C31;EGYPTIAN HIEROGLYPH-13C31;Lo;0;L;;;;;N;;;;; +13C32;EGYPTIAN HIEROGLYPH-13C32;Lo;0;L;;;;;N;;;;; +13C33;EGYPTIAN HIEROGLYPH-13C33;Lo;0;L;;;;;N;;;;; +13C34;EGYPTIAN HIEROGLYPH-13C34;Lo;0;L;;;;;N;;;;; +13C35;EGYPTIAN HIEROGLYPH-13C35;Lo;0;L;;;;;N;;;;; +13C36;EGYPTIAN HIEROGLYPH-13C36;Lo;0;L;;;;;N;;;;; +13C37;EGYPTIAN HIEROGLYPH-13C37;Lo;0;L;;;;;N;;;;; +13C38;EGYPTIAN HIEROGLYPH-13C38;Lo;0;L;;;;;N;;;;; +13C39;EGYPTIAN HIEROGLYPH-13C39;Lo;0;L;;;;;N;;;;; +13C3A;EGYPTIAN HIEROGLYPH-13C3A;Lo;0;L;;;;;N;;;;; +13C3B;EGYPTIAN HIEROGLYPH-13C3B;Lo;0;L;;;;;N;;;;; +13C3C;EGYPTIAN HIEROGLYPH-13C3C;Lo;0;L;;;;;N;;;;; +13C3D;EGYPTIAN HIEROGLYPH-13C3D;Lo;0;L;;;;;N;;;;; +13C3E;EGYPTIAN HIEROGLYPH-13C3E;Lo;0;L;;;;;N;;;;; +13C3F;EGYPTIAN HIEROGLYPH-13C3F;Lo;0;L;;;;;N;;;;; +13C40;EGYPTIAN HIEROGLYPH-13C40;Lo;0;L;;;;;N;;;;; +13C41;EGYPTIAN HIEROGLYPH-13C41;Lo;0;L;;;;;N;;;;; +13C42;EGYPTIAN HIEROGLYPH-13C42;Lo;0;L;;;;;N;;;;; +13C43;EGYPTIAN HIEROGLYPH-13C43;Lo;0;L;;;;;N;;;;; +13C44;EGYPTIAN HIEROGLYPH-13C44;Lo;0;L;;;;;N;;;;; +13C45;EGYPTIAN HIEROGLYPH-13C45;Lo;0;L;;;;;N;;;;; +13C46;EGYPTIAN HIEROGLYPH-13C46;Lo;0;L;;;;;N;;;;; +13C47;EGYPTIAN HIEROGLYPH-13C47;Lo;0;L;;;;;N;;;;; +13C48;EGYPTIAN HIEROGLYPH-13C48;Lo;0;L;;;;;N;;;;; +13C49;EGYPTIAN HIEROGLYPH-13C49;Lo;0;L;;;;;N;;;;; +13C4A;EGYPTIAN HIEROGLYPH-13C4A;Lo;0;L;;;;;N;;;;; +13C4B;EGYPTIAN HIEROGLYPH-13C4B;Lo;0;L;;;;;N;;;;; +13C4C;EGYPTIAN HIEROGLYPH-13C4C;Lo;0;L;;;;;N;;;;; +13C4D;EGYPTIAN HIEROGLYPH-13C4D;Lo;0;L;;;;;N;;;;; +13C4E;EGYPTIAN HIEROGLYPH-13C4E;Lo;0;L;;;;;N;;;;; +13C4F;EGYPTIAN HIEROGLYPH-13C4F;Lo;0;L;;;;;N;;;;; +13C50;EGYPTIAN HIEROGLYPH-13C50;Lo;0;L;;;;;N;;;;; +13C51;EGYPTIAN HIEROGLYPH-13C51;Lo;0;L;;;;;N;;;;; +13C52;EGYPTIAN HIEROGLYPH-13C52;Lo;0;L;;;;;N;;;;; +13C53;EGYPTIAN HIEROGLYPH-13C53;Lo;0;L;;;;;N;;;;; +13C54;EGYPTIAN HIEROGLYPH-13C54;Lo;0;L;;;;;N;;;;; +13C55;EGYPTIAN HIEROGLYPH-13C55;Lo;0;L;;;;;N;;;;; +13C56;EGYPTIAN HIEROGLYPH-13C56;Lo;0;L;;;;;N;;;;; +13C57;EGYPTIAN HIEROGLYPH-13C57;Lo;0;L;;;;;N;;;;; +13C58;EGYPTIAN HIEROGLYPH-13C58;Lo;0;L;;;;;N;;;;; +13C59;EGYPTIAN HIEROGLYPH-13C59;Lo;0;L;;;;;N;;;;; +13C5A;EGYPTIAN HIEROGLYPH-13C5A;Lo;0;L;;;;;N;;;;; +13C5B;EGYPTIAN HIEROGLYPH-13C5B;Lo;0;L;;;;;N;;;;; +13C5C;EGYPTIAN HIEROGLYPH-13C5C;Lo;0;L;;;;;N;;;;; +13C5D;EGYPTIAN HIEROGLYPH-13C5D;Lo;0;L;;;;;N;;;;; +13C5E;EGYPTIAN HIEROGLYPH-13C5E;Lo;0;L;;;;;N;;;;; +13C5F;EGYPTIAN HIEROGLYPH-13C5F;Lo;0;L;;;;;N;;;;; +13C60;EGYPTIAN HIEROGLYPH-13C60;Lo;0;L;;;;;N;;;;; +13C61;EGYPTIAN HIEROGLYPH-13C61;Lo;0;L;;;;;N;;;;; +13C62;EGYPTIAN HIEROGLYPH-13C62;Lo;0;L;;;;;N;;;;; +13C63;EGYPTIAN HIEROGLYPH-13C63;Lo;0;L;;;;;N;;;;; +13C64;EGYPTIAN HIEROGLYPH-13C64;Lo;0;L;;;;;N;;;;; +13C65;EGYPTIAN HIEROGLYPH-13C65;Lo;0;L;;;;;N;;;;; +13C66;EGYPTIAN HIEROGLYPH-13C66;Lo;0;L;;;;;N;;;;; +13C67;EGYPTIAN HIEROGLYPH-13C67;Lo;0;L;;;;;N;;;;; +13C68;EGYPTIAN HIEROGLYPH-13C68;Lo;0;L;;;;;N;;;;; +13C69;EGYPTIAN HIEROGLYPH-13C69;Lo;0;L;;;;;N;;;;; +13C6A;EGYPTIAN HIEROGLYPH-13C6A;Lo;0;L;;;;;N;;;;; +13C6B;EGYPTIAN HIEROGLYPH-13C6B;Lo;0;L;;;;;N;;;;; +13C6C;EGYPTIAN HIEROGLYPH-13C6C;Lo;0;L;;;;;N;;;;; +13C6D;EGYPTIAN HIEROGLYPH-13C6D;Lo;0;L;;;;;N;;;;; +13C6E;EGYPTIAN HIEROGLYPH-13C6E;Lo;0;L;;;;;N;;;;; +13C6F;EGYPTIAN HIEROGLYPH-13C6F;Lo;0;L;;;;;N;;;;; +13C70;EGYPTIAN HIEROGLYPH-13C70;Lo;0;L;;;;;N;;;;; +13C71;EGYPTIAN HIEROGLYPH-13C71;Lo;0;L;;;;;N;;;;; +13C72;EGYPTIAN HIEROGLYPH-13C72;Lo;0;L;;;;;N;;;;; +13C73;EGYPTIAN HIEROGLYPH-13C73;Lo;0;L;;;;;N;;;;; +13C74;EGYPTIAN HIEROGLYPH-13C74;Lo;0;L;;;;;N;;;;; +13C75;EGYPTIAN HIEROGLYPH-13C75;Lo;0;L;;;;;N;;;;; +13C76;EGYPTIAN HIEROGLYPH-13C76;Lo;0;L;;;;;N;;;;; +13C77;EGYPTIAN HIEROGLYPH-13C77;Lo;0;L;;;;;N;;;;; +13C78;EGYPTIAN HIEROGLYPH-13C78;Lo;0;L;;;;;N;;;;; +13C79;EGYPTIAN HIEROGLYPH-13C79;Lo;0;L;;;;;N;;;;; +13C7A;EGYPTIAN HIEROGLYPH-13C7A;Lo;0;L;;;;;N;;;;; +13C7B;EGYPTIAN HIEROGLYPH-13C7B;Lo;0;L;;;;;N;;;;; +13C7C;EGYPTIAN HIEROGLYPH-13C7C;Lo;0;L;;;;;N;;;;; +13C7D;EGYPTIAN HIEROGLYPH-13C7D;Lo;0;L;;;;;N;;;;; +13C7E;EGYPTIAN HIEROGLYPH-13C7E;Lo;0;L;;;;;N;;;;; +13C7F;EGYPTIAN HIEROGLYPH-13C7F;Lo;0;L;;;;;N;;;;; +13C80;EGYPTIAN HIEROGLYPH-13C80;Lo;0;L;;;;;N;;;;; +13C81;EGYPTIAN HIEROGLYPH-13C81;Lo;0;L;;;;;N;;;;; +13C82;EGYPTIAN HIEROGLYPH-13C82;Lo;0;L;;;;;N;;;;; +13C83;EGYPTIAN HIEROGLYPH-13C83;Lo;0;L;;;;;N;;;;; +13C84;EGYPTIAN HIEROGLYPH-13C84;Lo;0;L;;;;;N;;;;; +13C85;EGYPTIAN HIEROGLYPH-13C85;Lo;0;L;;;;;N;;;;; +13C86;EGYPTIAN HIEROGLYPH-13C86;Lo;0;L;;;;;N;;;;; +13C87;EGYPTIAN HIEROGLYPH-13C87;Lo;0;L;;;;;N;;;;; +13C88;EGYPTIAN HIEROGLYPH-13C88;Lo;0;L;;;;;N;;;;; +13C89;EGYPTIAN HIEROGLYPH-13C89;Lo;0;L;;;;;N;;;;; +13C8A;EGYPTIAN HIEROGLYPH-13C8A;Lo;0;L;;;;;N;;;;; +13C8B;EGYPTIAN HIEROGLYPH-13C8B;Lo;0;L;;;;;N;;;;; +13C8C;EGYPTIAN HIEROGLYPH-13C8C;Lo;0;L;;;;;N;;;;; +13C8D;EGYPTIAN HIEROGLYPH-13C8D;Lo;0;L;;;;;N;;;;; +13C8E;EGYPTIAN HIEROGLYPH-13C8E;Lo;0;L;;;;;N;;;;; +13C8F;EGYPTIAN HIEROGLYPH-13C8F;Lo;0;L;;;;;N;;;;; +13C90;EGYPTIAN HIEROGLYPH-13C90;Lo;0;L;;;;;N;;;;; +13C91;EGYPTIAN HIEROGLYPH-13C91;Lo;0;L;;;;;N;;;;; +13C92;EGYPTIAN HIEROGLYPH-13C92;Lo;0;L;;;;;N;;;;; +13C93;EGYPTIAN HIEROGLYPH-13C93;Lo;0;L;;;;;N;;;;; +13C94;EGYPTIAN HIEROGLYPH-13C94;Lo;0;L;;;;;N;;;;; +13C95;EGYPTIAN HIEROGLYPH-13C95;Lo;0;L;;;;;N;;;;; +13C96;EGYPTIAN HIEROGLYPH-13C96;Lo;0;L;;;;;N;;;;; +13C97;EGYPTIAN HIEROGLYPH-13C97;Lo;0;L;;;;;N;;;;; +13C98;EGYPTIAN HIEROGLYPH-13C98;Lo;0;L;;;;;N;;;;; +13C99;EGYPTIAN HIEROGLYPH-13C99;Lo;0;L;;;;;N;;;;; +13C9A;EGYPTIAN HIEROGLYPH-13C9A;Lo;0;L;;;;;N;;;;; +13C9B;EGYPTIAN HIEROGLYPH-13C9B;Lo;0;L;;;;;N;;;;; +13C9C;EGYPTIAN HIEROGLYPH-13C9C;Lo;0;L;;;;;N;;;;; +13C9D;EGYPTIAN HIEROGLYPH-13C9D;Lo;0;L;;;;;N;;;;; +13C9E;EGYPTIAN HIEROGLYPH-13C9E;Lo;0;L;;;;;N;;;;; +13C9F;EGYPTIAN HIEROGLYPH-13C9F;Lo;0;L;;;;;N;;;;; +13CA0;EGYPTIAN HIEROGLYPH-13CA0;Lo;0;L;;;;;N;;;;; +13CA1;EGYPTIAN HIEROGLYPH-13CA1;Lo;0;L;;;;;N;;;;; +13CA2;EGYPTIAN HIEROGLYPH-13CA2;Lo;0;L;;;;;N;;;;; +13CA3;EGYPTIAN HIEROGLYPH-13CA3;Lo;0;L;;;;;N;;;;; +13CA4;EGYPTIAN HIEROGLYPH-13CA4;Lo;0;L;;;;;N;;;;; +13CA5;EGYPTIAN HIEROGLYPH-13CA5;Lo;0;L;;;;;N;;;;; +13CA6;EGYPTIAN HIEROGLYPH-13CA6;Lo;0;L;;;;;N;;;;; +13CA7;EGYPTIAN HIEROGLYPH-13CA7;Lo;0;L;;;;;N;;;;; +13CA8;EGYPTIAN HIEROGLYPH-13CA8;Lo;0;L;;;;;N;;;;; +13CA9;EGYPTIAN HIEROGLYPH-13CA9;Lo;0;L;;;;;N;;;;; +13CAA;EGYPTIAN HIEROGLYPH-13CAA;Lo;0;L;;;;;N;;;;; +13CAB;EGYPTIAN HIEROGLYPH-13CAB;Lo;0;L;;;;;N;;;;; +13CAC;EGYPTIAN HIEROGLYPH-13CAC;Lo;0;L;;;;;N;;;;; +13CAD;EGYPTIAN HIEROGLYPH-13CAD;Lo;0;L;;;;;N;;;;; +13CAE;EGYPTIAN HIEROGLYPH-13CAE;Lo;0;L;;;;;N;;;;; +13CAF;EGYPTIAN HIEROGLYPH-13CAF;Lo;0;L;;;;;N;;;;; +13CB0;EGYPTIAN HIEROGLYPH-13CB0;Lo;0;L;;;;;N;;;;; +13CB1;EGYPTIAN HIEROGLYPH-13CB1;Lo;0;L;;;;;N;;;;; +13CB2;EGYPTIAN HIEROGLYPH-13CB2;Lo;0;L;;;;;N;;;;; +13CB3;EGYPTIAN HIEROGLYPH-13CB3;Lo;0;L;;;;;N;;;;; +13CB4;EGYPTIAN HIEROGLYPH-13CB4;Lo;0;L;;;;;N;;;;; +13CB5;EGYPTIAN HIEROGLYPH-13CB5;Lo;0;L;;;;;N;;;;; +13CB6;EGYPTIAN HIEROGLYPH-13CB6;Lo;0;L;;;;;N;;;;; +13CB7;EGYPTIAN HIEROGLYPH-13CB7;Lo;0;L;;;;;N;;;;; +13CB8;EGYPTIAN HIEROGLYPH-13CB8;Lo;0;L;;;;;N;;;;; +13CB9;EGYPTIAN HIEROGLYPH-13CB9;Lo;0;L;;;;;N;;;;; +13CBA;EGYPTIAN HIEROGLYPH-13CBA;Lo;0;L;;;;;N;;;;; +13CBB;EGYPTIAN HIEROGLYPH-13CBB;Lo;0;L;;;;;N;;;;; +13CBC;EGYPTIAN HIEROGLYPH-13CBC;Lo;0;L;;;;;N;;;;; +13CBD;EGYPTIAN HIEROGLYPH-13CBD;Lo;0;L;;;;;N;;;;; +13CBE;EGYPTIAN HIEROGLYPH-13CBE;Lo;0;L;;;;;N;;;;; +13CBF;EGYPTIAN HIEROGLYPH-13CBF;Lo;0;L;;;;;N;;;;; +13CC0;EGYPTIAN HIEROGLYPH-13CC0;Lo;0;L;;;;;N;;;;; +13CC1;EGYPTIAN HIEROGLYPH-13CC1;Lo;0;L;;;;;N;;;;; +13CC2;EGYPTIAN HIEROGLYPH-13CC2;Lo;0;L;;;;;N;;;;; +13CC3;EGYPTIAN HIEROGLYPH-13CC3;Lo;0;L;;;;;N;;;;; +13CC4;EGYPTIAN HIEROGLYPH-13CC4;Lo;0;L;;;;;N;;;;; +13CC5;EGYPTIAN HIEROGLYPH-13CC5;Lo;0;L;;;;;N;;;;; +13CC6;EGYPTIAN HIEROGLYPH-13CC6;Lo;0;L;;;;;N;;;;; +13CC7;EGYPTIAN HIEROGLYPH-13CC7;Lo;0;L;;;;;N;;;;; +13CC8;EGYPTIAN HIEROGLYPH-13CC8;Lo;0;L;;;;;N;;;;; +13CC9;EGYPTIAN HIEROGLYPH-13CC9;Lo;0;L;;;;;N;;;;; +13CCA;EGYPTIAN HIEROGLYPH-13CCA;Lo;0;L;;;;;N;;;;; +13CCB;EGYPTIAN HIEROGLYPH-13CCB;Lo;0;L;;;;;N;;;;; +13CCC;EGYPTIAN HIEROGLYPH-13CCC;Lo;0;L;;;;;N;;;;; +13CCD;EGYPTIAN HIEROGLYPH-13CCD;Lo;0;L;;;;;N;;;;; +13CCE;EGYPTIAN HIEROGLYPH-13CCE;Lo;0;L;;;;;N;;;;; +13CCF;EGYPTIAN HIEROGLYPH-13CCF;Lo;0;L;;;;;N;;;;; +13CD0;EGYPTIAN HIEROGLYPH-13CD0;Lo;0;L;;;;;N;;;;; +13CD1;EGYPTIAN HIEROGLYPH-13CD1;Lo;0;L;;;;;N;;;;; +13CD2;EGYPTIAN HIEROGLYPH-13CD2;Lo;0;L;;;;;N;;;;; +13CD3;EGYPTIAN HIEROGLYPH-13CD3;Lo;0;L;;;;;N;;;;; +13CD4;EGYPTIAN HIEROGLYPH-13CD4;Lo;0;L;;;;;N;;;;; +13CD5;EGYPTIAN HIEROGLYPH-13CD5;Lo;0;L;;;;;N;;;;; +13CD6;EGYPTIAN HIEROGLYPH-13CD6;Lo;0;L;;;;;N;;;;; +13CD7;EGYPTIAN HIEROGLYPH-13CD7;Lo;0;L;;;;;N;;;;; +13CD8;EGYPTIAN HIEROGLYPH-13CD8;Lo;0;L;;;;;N;;;;; +13CD9;EGYPTIAN HIEROGLYPH-13CD9;Lo;0;L;;;;;N;;;;; +13CDA;EGYPTIAN HIEROGLYPH-13CDA;Lo;0;L;;;;;N;;;;; +13CDB;EGYPTIAN HIEROGLYPH-13CDB;Lo;0;L;;;;;N;;;;; +13CDC;EGYPTIAN HIEROGLYPH-13CDC;Lo;0;L;;;;;N;;;;; +13CDD;EGYPTIAN HIEROGLYPH-13CDD;Lo;0;L;;;;;N;;;;; +13CDE;EGYPTIAN HIEROGLYPH-13CDE;Lo;0;L;;;;;N;;;;; +13CDF;EGYPTIAN HIEROGLYPH-13CDF;Lo;0;L;;;;;N;;;;; +13CE0;EGYPTIAN HIEROGLYPH-13CE0;Lo;0;L;;;;;N;;;;; +13CE1;EGYPTIAN HIEROGLYPH-13CE1;Lo;0;L;;;;;N;;;;; +13CE2;EGYPTIAN HIEROGLYPH-13CE2;Lo;0;L;;;;;N;;;;; +13CE3;EGYPTIAN HIEROGLYPH-13CE3;Lo;0;L;;;;;N;;;;; +13CE4;EGYPTIAN HIEROGLYPH-13CE4;Lo;0;L;;;;;N;;;;; +13CE5;EGYPTIAN HIEROGLYPH-13CE5;Lo;0;L;;;;;N;;;;; +13CE6;EGYPTIAN HIEROGLYPH-13CE6;Lo;0;L;;;;;N;;;;; +13CE7;EGYPTIAN HIEROGLYPH-13CE7;Lo;0;L;;;;;N;;;;; +13CE8;EGYPTIAN HIEROGLYPH-13CE8;Lo;0;L;;;;;N;;;;; +13CE9;EGYPTIAN HIEROGLYPH-13CE9;Lo;0;L;;;;;N;;;;; +13CEA;EGYPTIAN HIEROGLYPH-13CEA;Lo;0;L;;;;;N;;;;; +13CEB;EGYPTIAN HIEROGLYPH-13CEB;Lo;0;L;;;;;N;;;;; +13CEC;EGYPTIAN HIEROGLYPH-13CEC;Lo;0;L;;;;;N;;;;; +13CED;EGYPTIAN HIEROGLYPH-13CED;Lo;0;L;;;;;N;;;;; +13CEE;EGYPTIAN HIEROGLYPH-13CEE;Lo;0;L;;;;;N;;;;; +13CEF;EGYPTIAN HIEROGLYPH-13CEF;Lo;0;L;;;;;N;;;;; +13CF0;EGYPTIAN HIEROGLYPH-13CF0;Lo;0;L;;;;;N;;;;; +13CF1;EGYPTIAN HIEROGLYPH-13CF1;Lo;0;L;;;;;N;;;;; +13CF2;EGYPTIAN HIEROGLYPH-13CF2;Lo;0;L;;;;;N;;;;; +13CF3;EGYPTIAN HIEROGLYPH-13CF3;Lo;0;L;;;;;N;;;;; +13CF4;EGYPTIAN HIEROGLYPH-13CF4;Lo;0;L;;;;;N;;;;; +13CF5;EGYPTIAN HIEROGLYPH-13CF5;Lo;0;L;;;;;N;;;;; +13CF6;EGYPTIAN HIEROGLYPH-13CF6;Lo;0;L;;;;;N;;;;; +13CF7;EGYPTIAN HIEROGLYPH-13CF7;Lo;0;L;;;;;N;;;;; +13CF8;EGYPTIAN HIEROGLYPH-13CF8;Lo;0;L;;;;;N;;;;; +13CF9;EGYPTIAN HIEROGLYPH-13CF9;Lo;0;L;;;;;N;;;;; +13CFA;EGYPTIAN HIEROGLYPH-13CFA;Lo;0;L;;;;;N;;;;; +13CFB;EGYPTIAN HIEROGLYPH-13CFB;Lo;0;L;;;;;N;;;;; +13CFC;EGYPTIAN HIEROGLYPH-13CFC;Lo;0;L;;;;;N;;;;; +13CFD;EGYPTIAN HIEROGLYPH-13CFD;Lo;0;L;;;;;N;;;;; +13CFE;EGYPTIAN HIEROGLYPH-13CFE;Lo;0;L;;;;;N;;;;; +13CFF;EGYPTIAN HIEROGLYPH-13CFF;Lo;0;L;;;;;N;;;;; +13D00;EGYPTIAN HIEROGLYPH-13D00;Lo;0;L;;;;;N;;;;; +13D01;EGYPTIAN HIEROGLYPH-13D01;Lo;0;L;;;;;N;;;;; +13D02;EGYPTIAN HIEROGLYPH-13D02;Lo;0;L;;;;;N;;;;; +13D03;EGYPTIAN HIEROGLYPH-13D03;Lo;0;L;;;;;N;;;;; +13D04;EGYPTIAN HIEROGLYPH-13D04;Lo;0;L;;;;;N;;;;; +13D05;EGYPTIAN HIEROGLYPH-13D05;Lo;0;L;;;;;N;;;;; +13D06;EGYPTIAN HIEROGLYPH-13D06;Lo;0;L;;;;;N;;;;; +13D07;EGYPTIAN HIEROGLYPH-13D07;Lo;0;L;;;;;N;;;;; +13D08;EGYPTIAN HIEROGLYPH-13D08;Lo;0;L;;;;;N;;;;; +13D09;EGYPTIAN HIEROGLYPH-13D09;Lo;0;L;;;;;N;;;;; +13D0A;EGYPTIAN HIEROGLYPH-13D0A;Lo;0;L;;;;;N;;;;; +13D0B;EGYPTIAN HIEROGLYPH-13D0B;Lo;0;L;;;;;N;;;;; +13D0C;EGYPTIAN HIEROGLYPH-13D0C;Lo;0;L;;;;;N;;;;; +13D0D;EGYPTIAN HIEROGLYPH-13D0D;Lo;0;L;;;;;N;;;;; +13D0E;EGYPTIAN HIEROGLYPH-13D0E;Lo;0;L;;;;;N;;;;; +13D0F;EGYPTIAN HIEROGLYPH-13D0F;Lo;0;L;;;;;N;;;;; +13D10;EGYPTIAN HIEROGLYPH-13D10;Lo;0;L;;;;;N;;;;; +13D11;EGYPTIAN HIEROGLYPH-13D11;Lo;0;L;;;;;N;;;;; +13D12;EGYPTIAN HIEROGLYPH-13D12;Lo;0;L;;;;;N;;;;; +13D13;EGYPTIAN HIEROGLYPH-13D13;Lo;0;L;;;;;N;;;;; +13D14;EGYPTIAN HIEROGLYPH-13D14;Lo;0;L;;;;;N;;;;; +13D15;EGYPTIAN HIEROGLYPH-13D15;Lo;0;L;;;;;N;;;;; +13D16;EGYPTIAN HIEROGLYPH-13D16;Lo;0;L;;;;;N;;;;; +13D17;EGYPTIAN HIEROGLYPH-13D17;Lo;0;L;;;;;N;;;;; +13D18;EGYPTIAN HIEROGLYPH-13D18;Lo;0;L;;;;;N;;;;; +13D19;EGYPTIAN HIEROGLYPH-13D19;Lo;0;L;;;;;N;;;;; +13D1A;EGYPTIAN HIEROGLYPH-13D1A;Lo;0;L;;;;;N;;;;; +13D1B;EGYPTIAN HIEROGLYPH-13D1B;Lo;0;L;;;;;N;;;;; +13D1C;EGYPTIAN HIEROGLYPH-13D1C;Lo;0;L;;;;;N;;;;; +13D1D;EGYPTIAN HIEROGLYPH-13D1D;Lo;0;L;;;;;N;;;;; +13D1E;EGYPTIAN HIEROGLYPH-13D1E;Lo;0;L;;;;;N;;;;; +13D1F;EGYPTIAN HIEROGLYPH-13D1F;Lo;0;L;;;;;N;;;;; +13D20;EGYPTIAN HIEROGLYPH-13D20;Lo;0;L;;;;;N;;;;; +13D21;EGYPTIAN HIEROGLYPH-13D21;Lo;0;L;;;;;N;;;;; +13D22;EGYPTIAN HIEROGLYPH-13D22;Lo;0;L;;;;;N;;;;; +13D23;EGYPTIAN HIEROGLYPH-13D23;Lo;0;L;;;;;N;;;;; +13D24;EGYPTIAN HIEROGLYPH-13D24;Lo;0;L;;;;;N;;;;; +13D25;EGYPTIAN HIEROGLYPH-13D25;Lo;0;L;;;;;N;;;;; +13D26;EGYPTIAN HIEROGLYPH-13D26;Lo;0;L;;;;;N;;;;; +13D27;EGYPTIAN HIEROGLYPH-13D27;Lo;0;L;;;;;N;;;;; +13D28;EGYPTIAN HIEROGLYPH-13D28;Lo;0;L;;;;;N;;;;; +13D29;EGYPTIAN HIEROGLYPH-13D29;Lo;0;L;;;;;N;;;;; +13D2A;EGYPTIAN HIEROGLYPH-13D2A;Lo;0;L;;;;;N;;;;; +13D2B;EGYPTIAN HIEROGLYPH-13D2B;Lo;0;L;;;;;N;;;;; +13D2C;EGYPTIAN HIEROGLYPH-13D2C;Lo;0;L;;;;;N;;;;; +13D2D;EGYPTIAN HIEROGLYPH-13D2D;Lo;0;L;;;;;N;;;;; +13D2E;EGYPTIAN HIEROGLYPH-13D2E;Lo;0;L;;;;;N;;;;; +13D2F;EGYPTIAN HIEROGLYPH-13D2F;Lo;0;L;;;;;N;;;;; +13D30;EGYPTIAN HIEROGLYPH-13D30;Lo;0;L;;;;;N;;;;; +13D31;EGYPTIAN HIEROGLYPH-13D31;Lo;0;L;;;;;N;;;;; +13D32;EGYPTIAN HIEROGLYPH-13D32;Lo;0;L;;;;;N;;;;; +13D33;EGYPTIAN HIEROGLYPH-13D33;Lo;0;L;;;;;N;;;;; +13D34;EGYPTIAN HIEROGLYPH-13D34;Lo;0;L;;;;;N;;;;; +13D35;EGYPTIAN HIEROGLYPH-13D35;Lo;0;L;;;;;N;;;;; +13D36;EGYPTIAN HIEROGLYPH-13D36;Lo;0;L;;;;;N;;;;; +13D37;EGYPTIAN HIEROGLYPH-13D37;Lo;0;L;;;;;N;;;;; +13D38;EGYPTIAN HIEROGLYPH-13D38;Lo;0;L;;;;;N;;;;; +13D39;EGYPTIAN HIEROGLYPH-13D39;Lo;0;L;;;;;N;;;;; +13D3A;EGYPTIAN HIEROGLYPH-13D3A;Lo;0;L;;;;;N;;;;; +13D3B;EGYPTIAN HIEROGLYPH-13D3B;Lo;0;L;;;;;N;;;;; +13D3C;EGYPTIAN HIEROGLYPH-13D3C;Lo;0;L;;;;;N;;;;; +13D3D;EGYPTIAN HIEROGLYPH-13D3D;Lo;0;L;;;;;N;;;;; +13D3E;EGYPTIAN HIEROGLYPH-13D3E;Lo;0;L;;;;;N;;;;; +13D3F;EGYPTIAN HIEROGLYPH-13D3F;Lo;0;L;;;;;N;;;;; +13D40;EGYPTIAN HIEROGLYPH-13D40;Lo;0;L;;;;;N;;;;; +13D41;EGYPTIAN HIEROGLYPH-13D41;Lo;0;L;;;;;N;;;;; +13D42;EGYPTIAN HIEROGLYPH-13D42;Lo;0;L;;;;;N;;;;; +13D43;EGYPTIAN HIEROGLYPH-13D43;Lo;0;L;;;;;N;;;;; +13D44;EGYPTIAN HIEROGLYPH-13D44;Lo;0;L;;;;;N;;;;; +13D45;EGYPTIAN HIEROGLYPH-13D45;Lo;0;L;;;;;N;;;;; +13D46;EGYPTIAN HIEROGLYPH-13D46;Lo;0;L;;;;;N;;;;; +13D47;EGYPTIAN HIEROGLYPH-13D47;Lo;0;L;;;;;N;;;;; +13D48;EGYPTIAN HIEROGLYPH-13D48;Lo;0;L;;;;;N;;;;; +13D49;EGYPTIAN HIEROGLYPH-13D49;Lo;0;L;;;;;N;;;;; +13D4A;EGYPTIAN HIEROGLYPH-13D4A;Lo;0;L;;;;;N;;;;; +13D4B;EGYPTIAN HIEROGLYPH-13D4B;Lo;0;L;;;;;N;;;;; +13D4C;EGYPTIAN HIEROGLYPH-13D4C;Lo;0;L;;;;;N;;;;; +13D4D;EGYPTIAN HIEROGLYPH-13D4D;Lo;0;L;;;;;N;;;;; +13D4E;EGYPTIAN HIEROGLYPH-13D4E;Lo;0;L;;;;;N;;;;; +13D4F;EGYPTIAN HIEROGLYPH-13D4F;Lo;0;L;;;;;N;;;;; +13D50;EGYPTIAN HIEROGLYPH-13D50;Lo;0;L;;;;;N;;;;; +13D51;EGYPTIAN HIEROGLYPH-13D51;Lo;0;L;;;;;N;;;;; +13D52;EGYPTIAN HIEROGLYPH-13D52;Lo;0;L;;;;;N;;;;; +13D53;EGYPTIAN HIEROGLYPH-13D53;Lo;0;L;;;;;N;;;;; +13D54;EGYPTIAN HIEROGLYPH-13D54;Lo;0;L;;;;;N;;;;; +13D55;EGYPTIAN HIEROGLYPH-13D55;Lo;0;L;;;;;N;;;;; +13D56;EGYPTIAN HIEROGLYPH-13D56;Lo;0;L;;;;;N;;;;; +13D57;EGYPTIAN HIEROGLYPH-13D57;Lo;0;L;;;;;N;;;;; +13D58;EGYPTIAN HIEROGLYPH-13D58;Lo;0;L;;;;;N;;;;; +13D59;EGYPTIAN HIEROGLYPH-13D59;Lo;0;L;;;;;N;;;;; +13D5A;EGYPTIAN HIEROGLYPH-13D5A;Lo;0;L;;;;;N;;;;; +13D5B;EGYPTIAN HIEROGLYPH-13D5B;Lo;0;L;;;;;N;;;;; +13D5C;EGYPTIAN HIEROGLYPH-13D5C;Lo;0;L;;;;;N;;;;; +13D5D;EGYPTIAN HIEROGLYPH-13D5D;Lo;0;L;;;;;N;;;;; +13D5E;EGYPTIAN HIEROGLYPH-13D5E;Lo;0;L;;;;;N;;;;; +13D5F;EGYPTIAN HIEROGLYPH-13D5F;Lo;0;L;;;;;N;;;;; +13D60;EGYPTIAN HIEROGLYPH-13D60;Lo;0;L;;;;;N;;;;; +13D61;EGYPTIAN HIEROGLYPH-13D61;Lo;0;L;;;;;N;;;;; +13D62;EGYPTIAN HIEROGLYPH-13D62;Lo;0;L;;;;;N;;;;; +13D63;EGYPTIAN HIEROGLYPH-13D63;Lo;0;L;;;;;N;;;;; +13D64;EGYPTIAN HIEROGLYPH-13D64;Lo;0;L;;;;;N;;;;; +13D65;EGYPTIAN HIEROGLYPH-13D65;Lo;0;L;;;;;N;;;;; +13D66;EGYPTIAN HIEROGLYPH-13D66;Lo;0;L;;;;;N;;;;; +13D67;EGYPTIAN HIEROGLYPH-13D67;Lo;0;L;;;;;N;;;;; +13D68;EGYPTIAN HIEROGLYPH-13D68;Lo;0;L;;;;;N;;;;; +13D69;EGYPTIAN HIEROGLYPH-13D69;Lo;0;L;;;;;N;;;;; +13D6A;EGYPTIAN HIEROGLYPH-13D6A;Lo;0;L;;;;;N;;;;; +13D6B;EGYPTIAN HIEROGLYPH-13D6B;Lo;0;L;;;;;N;;;;; +13D6C;EGYPTIAN HIEROGLYPH-13D6C;Lo;0;L;;;;;N;;;;; +13D6D;EGYPTIAN HIEROGLYPH-13D6D;Lo;0;L;;;;;N;;;;; +13D6E;EGYPTIAN HIEROGLYPH-13D6E;Lo;0;L;;;;;N;;;;; +13D6F;EGYPTIAN HIEROGLYPH-13D6F;Lo;0;L;;;;;N;;;;; +13D70;EGYPTIAN HIEROGLYPH-13D70;Lo;0;L;;;;;N;;;;; +13D71;EGYPTIAN HIEROGLYPH-13D71;Lo;0;L;;;;;N;;;;; +13D72;EGYPTIAN HIEROGLYPH-13D72;Lo;0;L;;;;;N;;;;; +13D73;EGYPTIAN HIEROGLYPH-13D73;Lo;0;L;;;;;N;;;;; +13D74;EGYPTIAN HIEROGLYPH-13D74;Lo;0;L;;;;;N;;;;; +13D75;EGYPTIAN HIEROGLYPH-13D75;Lo;0;L;;;;;N;;;;; +13D76;EGYPTIAN HIEROGLYPH-13D76;Lo;0;L;;;;;N;;;;; +13D77;EGYPTIAN HIEROGLYPH-13D77;Lo;0;L;;;;;N;;;;; +13D78;EGYPTIAN HIEROGLYPH-13D78;Lo;0;L;;;;;N;;;;; +13D79;EGYPTIAN HIEROGLYPH-13D79;Lo;0;L;;;;;N;;;;; +13D7A;EGYPTIAN HIEROGLYPH-13D7A;Lo;0;L;;;;;N;;;;; +13D7B;EGYPTIAN HIEROGLYPH-13D7B;Lo;0;L;;;;;N;;;;; +13D7C;EGYPTIAN HIEROGLYPH-13D7C;Lo;0;L;;;;;N;;;;; +13D7D;EGYPTIAN HIEROGLYPH-13D7D;Lo;0;L;;;;;N;;;;; +13D7E;EGYPTIAN HIEROGLYPH-13D7E;Lo;0;L;;;;;N;;;;; +13D7F;EGYPTIAN HIEROGLYPH-13D7F;Lo;0;L;;;;;N;;;;; +13D80;EGYPTIAN HIEROGLYPH-13D80;Lo;0;L;;;;;N;;;;; +13D81;EGYPTIAN HIEROGLYPH-13D81;Lo;0;L;;;;;N;;;;; +13D82;EGYPTIAN HIEROGLYPH-13D82;Lo;0;L;;;;;N;;;;; +13D83;EGYPTIAN HIEROGLYPH-13D83;Lo;0;L;;;;;N;;;;; +13D84;EGYPTIAN HIEROGLYPH-13D84;Lo;0;L;;;;;N;;;;; +13D85;EGYPTIAN HIEROGLYPH-13D85;Lo;0;L;;;;;N;;;;; +13D86;EGYPTIAN HIEROGLYPH-13D86;Lo;0;L;;;;;N;;;;; +13D87;EGYPTIAN HIEROGLYPH-13D87;Lo;0;L;;;;;N;;;;; +13D88;EGYPTIAN HIEROGLYPH-13D88;Lo;0;L;;;;;N;;;;; +13D89;EGYPTIAN HIEROGLYPH-13D89;Lo;0;L;;;;;N;;;;; +13D8A;EGYPTIAN HIEROGLYPH-13D8A;Lo;0;L;;;;;N;;;;; +13D8B;EGYPTIAN HIEROGLYPH-13D8B;Lo;0;L;;;;;N;;;;; +13D8C;EGYPTIAN HIEROGLYPH-13D8C;Lo;0;L;;;;;N;;;;; +13D8D;EGYPTIAN HIEROGLYPH-13D8D;Lo;0;L;;;;;N;;;;; +13D8E;EGYPTIAN HIEROGLYPH-13D8E;Lo;0;L;;;;;N;;;;; +13D8F;EGYPTIAN HIEROGLYPH-13D8F;Lo;0;L;;;;;N;;;;; +13D90;EGYPTIAN HIEROGLYPH-13D90;Lo;0;L;;;;;N;;;;; +13D91;EGYPTIAN HIEROGLYPH-13D91;Lo;0;L;;;;;N;;;;; +13D92;EGYPTIAN HIEROGLYPH-13D92;Lo;0;L;;;;;N;;;;; +13D93;EGYPTIAN HIEROGLYPH-13D93;Lo;0;L;;;;;N;;;;; +13D94;EGYPTIAN HIEROGLYPH-13D94;Lo;0;L;;;;;N;;;;; +13D95;EGYPTIAN HIEROGLYPH-13D95;Lo;0;L;;;;;N;;;;; +13D96;EGYPTIAN HIEROGLYPH-13D96;Lo;0;L;;;;;N;;;;; +13D97;EGYPTIAN HIEROGLYPH-13D97;Lo;0;L;;;;;N;;;;; +13D98;EGYPTIAN HIEROGLYPH-13D98;Lo;0;L;;;;;N;;;;; +13D99;EGYPTIAN HIEROGLYPH-13D99;Lo;0;L;;;;;N;;;;; +13D9A;EGYPTIAN HIEROGLYPH-13D9A;Lo;0;L;;;;;N;;;;; +13D9B;EGYPTIAN HIEROGLYPH-13D9B;Lo;0;L;;;;;N;;;;; +13D9C;EGYPTIAN HIEROGLYPH-13D9C;Lo;0;L;;;;;N;;;;; +13D9D;EGYPTIAN HIEROGLYPH-13D9D;Lo;0;L;;;;;N;;;;; +13D9E;EGYPTIAN HIEROGLYPH-13D9E;Lo;0;L;;;;;N;;;;; +13D9F;EGYPTIAN HIEROGLYPH-13D9F;Lo;0;L;;;;;N;;;;; +13DA0;EGYPTIAN HIEROGLYPH-13DA0;Lo;0;L;;;;;N;;;;; +13DA1;EGYPTIAN HIEROGLYPH-13DA1;Lo;0;L;;;;;N;;;;; +13DA2;EGYPTIAN HIEROGLYPH-13DA2;Lo;0;L;;;;;N;;;;; +13DA3;EGYPTIAN HIEROGLYPH-13DA3;Lo;0;L;;;;;N;;;;; +13DA4;EGYPTIAN HIEROGLYPH-13DA4;Lo;0;L;;;;;N;;;;; +13DA5;EGYPTIAN HIEROGLYPH-13DA5;Lo;0;L;;;;;N;;;;; +13DA6;EGYPTIAN HIEROGLYPH-13DA6;Lo;0;L;;;;;N;;;;; +13DA7;EGYPTIAN HIEROGLYPH-13DA7;Lo;0;L;;;;;N;;;;; +13DA8;EGYPTIAN HIEROGLYPH-13DA8;Lo;0;L;;;;;N;;;;; +13DA9;EGYPTIAN HIEROGLYPH-13DA9;Lo;0;L;;;;;N;;;;; +13DAA;EGYPTIAN HIEROGLYPH-13DAA;Lo;0;L;;;;;N;;;;; +13DAB;EGYPTIAN HIEROGLYPH-13DAB;Lo;0;L;;;;;N;;;;; +13DAC;EGYPTIAN HIEROGLYPH-13DAC;Lo;0;L;;;;;N;;;;; +13DAD;EGYPTIAN HIEROGLYPH-13DAD;Lo;0;L;;;;;N;;;;; +13DAE;EGYPTIAN HIEROGLYPH-13DAE;Lo;0;L;;;;;N;;;;; +13DAF;EGYPTIAN HIEROGLYPH-13DAF;Lo;0;L;;;;;N;;;;; +13DB0;EGYPTIAN HIEROGLYPH-13DB0;Lo;0;L;;;;;N;;;;; +13DB1;EGYPTIAN HIEROGLYPH-13DB1;Lo;0;L;;;;;N;;;;; +13DB2;EGYPTIAN HIEROGLYPH-13DB2;Lo;0;L;;;;;N;;;;; +13DB3;EGYPTIAN HIEROGLYPH-13DB3;Lo;0;L;;;;;N;;;;; +13DB4;EGYPTIAN HIEROGLYPH-13DB4;Lo;0;L;;;;;N;;;;; +13DB5;EGYPTIAN HIEROGLYPH-13DB5;Lo;0;L;;;;;N;;;;; +13DB6;EGYPTIAN HIEROGLYPH-13DB6;Lo;0;L;;;;;N;;;;; +13DB7;EGYPTIAN HIEROGLYPH-13DB7;Lo;0;L;;;;;N;;;;; +13DB8;EGYPTIAN HIEROGLYPH-13DB8;Lo;0;L;;;;;N;;;;; +13DB9;EGYPTIAN HIEROGLYPH-13DB9;Lo;0;L;;;;;N;;;;; +13DBA;EGYPTIAN HIEROGLYPH-13DBA;Lo;0;L;;;;;N;;;;; +13DBB;EGYPTIAN HIEROGLYPH-13DBB;Lo;0;L;;;;;N;;;;; +13DBC;EGYPTIAN HIEROGLYPH-13DBC;Lo;0;L;;;;;N;;;;; +13DBD;EGYPTIAN HIEROGLYPH-13DBD;Lo;0;L;;;;;N;;;;; +13DBE;EGYPTIAN HIEROGLYPH-13DBE;Lo;0;L;;;;;N;;;;; +13DBF;EGYPTIAN HIEROGLYPH-13DBF;Lo;0;L;;;;;N;;;;; +13DC0;EGYPTIAN HIEROGLYPH-13DC0;Lo;0;L;;;;;N;;;;; +13DC1;EGYPTIAN HIEROGLYPH-13DC1;Lo;0;L;;;;;N;;;;; +13DC2;EGYPTIAN HIEROGLYPH-13DC2;Lo;0;L;;;;;N;;;;; +13DC3;EGYPTIAN HIEROGLYPH-13DC3;Lo;0;L;;;;;N;;;;; +13DC4;EGYPTIAN HIEROGLYPH-13DC4;Lo;0;L;;;;;N;;;;; +13DC5;EGYPTIAN HIEROGLYPH-13DC5;Lo;0;L;;;;;N;;;;; +13DC6;EGYPTIAN HIEROGLYPH-13DC6;Lo;0;L;;;;;N;;;;; +13DC7;EGYPTIAN HIEROGLYPH-13DC7;Lo;0;L;;;;;N;;;;; +13DC8;EGYPTIAN HIEROGLYPH-13DC8;Lo;0;L;;;;;N;;;;; +13DC9;EGYPTIAN HIEROGLYPH-13DC9;Lo;0;L;;;;;N;;;;; +13DCA;EGYPTIAN HIEROGLYPH-13DCA;Lo;0;L;;;;;N;;;;; +13DCB;EGYPTIAN HIEROGLYPH-13DCB;Lo;0;L;;;;;N;;;;; +13DCC;EGYPTIAN HIEROGLYPH-13DCC;Lo;0;L;;;;;N;;;;; +13DCD;EGYPTIAN HIEROGLYPH-13DCD;Lo;0;L;;;;;N;;;;; +13DCE;EGYPTIAN HIEROGLYPH-13DCE;Lo;0;L;;;;;N;;;;; +13DCF;EGYPTIAN HIEROGLYPH-13DCF;Lo;0;L;;;;;N;;;;; +13DD0;EGYPTIAN HIEROGLYPH-13DD0;Lo;0;L;;;;;N;;;;; +13DD1;EGYPTIAN HIEROGLYPH-13DD1;Lo;0;L;;;;;N;;;;; +13DD2;EGYPTIAN HIEROGLYPH-13DD2;Lo;0;L;;;;;N;;;;; +13DD3;EGYPTIAN HIEROGLYPH-13DD3;Lo;0;L;;;;;N;;;;; +13DD4;EGYPTIAN HIEROGLYPH-13DD4;Lo;0;L;;;;;N;;;;; +13DD5;EGYPTIAN HIEROGLYPH-13DD5;Lo;0;L;;;;;N;;;;; +13DD6;EGYPTIAN HIEROGLYPH-13DD6;Lo;0;L;;;;;N;;;;; +13DD7;EGYPTIAN HIEROGLYPH-13DD7;Lo;0;L;;;;;N;;;;; +13DD8;EGYPTIAN HIEROGLYPH-13DD8;Lo;0;L;;;;;N;;;;; +13DD9;EGYPTIAN HIEROGLYPH-13DD9;Lo;0;L;;;;;N;;;;; +13DDA;EGYPTIAN HIEROGLYPH-13DDA;Lo;0;L;;;;;N;;;;; +13DDB;EGYPTIAN HIEROGLYPH-13DDB;Lo;0;L;;;;;N;;;;; +13DDC;EGYPTIAN HIEROGLYPH-13DDC;Lo;0;L;;;;;N;;;;; +13DDD;EGYPTIAN HIEROGLYPH-13DDD;Lo;0;L;;;;;N;;;;; +13DDE;EGYPTIAN HIEROGLYPH-13DDE;Lo;0;L;;;;;N;;;;; +13DDF;EGYPTIAN HIEROGLYPH-13DDF;Lo;0;L;;;;;N;;;;; +13DE0;EGYPTIAN HIEROGLYPH-13DE0;Lo;0;L;;;;;N;;;;; +13DE1;EGYPTIAN HIEROGLYPH-13DE1;Lo;0;L;;;;;N;;;;; +13DE2;EGYPTIAN HIEROGLYPH-13DE2;Lo;0;L;;;;;N;;;;; +13DE3;EGYPTIAN HIEROGLYPH-13DE3;Lo;0;L;;;;;N;;;;; +13DE4;EGYPTIAN HIEROGLYPH-13DE4;Lo;0;L;;;;;N;;;;; +13DE5;EGYPTIAN HIEROGLYPH-13DE5;Lo;0;L;;;;;N;;;;; +13DE6;EGYPTIAN HIEROGLYPH-13DE6;Lo;0;L;;;;;N;;;;; +13DE7;EGYPTIAN HIEROGLYPH-13DE7;Lo;0;L;;;;;N;;;;; +13DE8;EGYPTIAN HIEROGLYPH-13DE8;Lo;0;L;;;;;N;;;;; +13DE9;EGYPTIAN HIEROGLYPH-13DE9;Lo;0;L;;;;;N;;;;; +13DEA;EGYPTIAN HIEROGLYPH-13DEA;Lo;0;L;;;;;N;;;;; +13DEB;EGYPTIAN HIEROGLYPH-13DEB;Lo;0;L;;;;;N;;;;; +13DEC;EGYPTIAN HIEROGLYPH-13DEC;Lo;0;L;;;;;N;;;;; +13DED;EGYPTIAN HIEROGLYPH-13DED;Lo;0;L;;;;;N;;;;; +13DEE;EGYPTIAN HIEROGLYPH-13DEE;Lo;0;L;;;;;N;;;;; +13DEF;EGYPTIAN HIEROGLYPH-13DEF;Lo;0;L;;;;;N;;;;; +13DF0;EGYPTIAN HIEROGLYPH-13DF0;Lo;0;L;;;;;N;;;;; +13DF1;EGYPTIAN HIEROGLYPH-13DF1;Lo;0;L;;;;;N;;;;; +13DF2;EGYPTIAN HIEROGLYPH-13DF2;Lo;0;L;;;;;N;;;;; +13DF3;EGYPTIAN HIEROGLYPH-13DF3;Lo;0;L;;;;;N;;;;; +13DF4;EGYPTIAN HIEROGLYPH-13DF4;Lo;0;L;;;;;N;;;;; +13DF5;EGYPTIAN HIEROGLYPH-13DF5;Lo;0;L;;;;;N;;;;; +13DF6;EGYPTIAN HIEROGLYPH-13DF6;Lo;0;L;;;;;N;;;;; +13DF7;EGYPTIAN HIEROGLYPH-13DF7;Lo;0;L;;;;;N;;;;; +13DF8;EGYPTIAN HIEROGLYPH-13DF8;Lo;0;L;;;;;N;;;;; +13DF9;EGYPTIAN HIEROGLYPH-13DF9;Lo;0;L;;;;;N;;;;; +13DFA;EGYPTIAN HIEROGLYPH-13DFA;Lo;0;L;;;;;N;;;;; +13DFB;EGYPTIAN HIEROGLYPH-13DFB;Lo;0;L;;;;;N;;;;; +13DFC;EGYPTIAN HIEROGLYPH-13DFC;Lo;0;L;;;;;N;;;;; +13DFD;EGYPTIAN HIEROGLYPH-13DFD;Lo;0;L;;;;;N;;;;; +13DFE;EGYPTIAN HIEROGLYPH-13DFE;Lo;0;L;;;;;N;;;;; +13DFF;EGYPTIAN HIEROGLYPH-13DFF;Lo;0;L;;;;;N;;;;; +13E00;EGYPTIAN HIEROGLYPH-13E00;Lo;0;L;;;;;N;;;;; +13E01;EGYPTIAN HIEROGLYPH-13E01;Lo;0;L;;;;;N;;;;; +13E02;EGYPTIAN HIEROGLYPH-13E02;Lo;0;L;;;;;N;;;;; +13E03;EGYPTIAN HIEROGLYPH-13E03;Lo;0;L;;;;;N;;;;; +13E04;EGYPTIAN HIEROGLYPH-13E04;Lo;0;L;;;;;N;;;;; +13E05;EGYPTIAN HIEROGLYPH-13E05;Lo;0;L;;;;;N;;;;; +13E06;EGYPTIAN HIEROGLYPH-13E06;Lo;0;L;;;;;N;;;;; +13E07;EGYPTIAN HIEROGLYPH-13E07;Lo;0;L;;;;;N;;;;; +13E08;EGYPTIAN HIEROGLYPH-13E08;Lo;0;L;;;;;N;;;;; +13E09;EGYPTIAN HIEROGLYPH-13E09;Lo;0;L;;;;;N;;;;; +13E0A;EGYPTIAN HIEROGLYPH-13E0A;Lo;0;L;;;;;N;;;;; +13E0B;EGYPTIAN HIEROGLYPH-13E0B;Lo;0;L;;;;;N;;;;; +13E0C;EGYPTIAN HIEROGLYPH-13E0C;Lo;0;L;;;;;N;;;;; +13E0D;EGYPTIAN HIEROGLYPH-13E0D;Lo;0;L;;;;;N;;;;; +13E0E;EGYPTIAN HIEROGLYPH-13E0E;Lo;0;L;;;;;N;;;;; +13E0F;EGYPTIAN HIEROGLYPH-13E0F;Lo;0;L;;;;;N;;;;; +13E10;EGYPTIAN HIEROGLYPH-13E10;Lo;0;L;;;;;N;;;;; +13E11;EGYPTIAN HIEROGLYPH-13E11;Lo;0;L;;;;;N;;;;; +13E12;EGYPTIAN HIEROGLYPH-13E12;Lo;0;L;;;;;N;;;;; +13E13;EGYPTIAN HIEROGLYPH-13E13;Lo;0;L;;;;;N;;;;; +13E14;EGYPTIAN HIEROGLYPH-13E14;Lo;0;L;;;;;N;;;;; +13E15;EGYPTIAN HIEROGLYPH-13E15;Lo;0;L;;;;;N;;;;; +13E16;EGYPTIAN HIEROGLYPH-13E16;Lo;0;L;;;;;N;;;;; +13E17;EGYPTIAN HIEROGLYPH-13E17;Lo;0;L;;;;;N;;;;; +13E18;EGYPTIAN HIEROGLYPH-13E18;Lo;0;L;;;;;N;;;;; +13E19;EGYPTIAN HIEROGLYPH-13E19;Lo;0;L;;;;;N;;;;; +13E1A;EGYPTIAN HIEROGLYPH-13E1A;Lo;0;L;;;;;N;;;;; +13E1B;EGYPTIAN HIEROGLYPH-13E1B;Lo;0;L;;;;;N;;;;; +13E1C;EGYPTIAN HIEROGLYPH-13E1C;Lo;0;L;;;;;N;;;;; +13E1D;EGYPTIAN HIEROGLYPH-13E1D;Lo;0;L;;;;;N;;;;; +13E1E;EGYPTIAN HIEROGLYPH-13E1E;Lo;0;L;;;;;N;;;;; +13E1F;EGYPTIAN HIEROGLYPH-13E1F;Lo;0;L;;;;;N;;;;; +13E20;EGYPTIAN HIEROGLYPH-13E20;Lo;0;L;;;;;N;;;;; +13E21;EGYPTIAN HIEROGLYPH-13E21;Lo;0;L;;;;;N;;;;; +13E22;EGYPTIAN HIEROGLYPH-13E22;Lo;0;L;;;;;N;;;;; +13E23;EGYPTIAN HIEROGLYPH-13E23;Lo;0;L;;;;;N;;;;; +13E24;EGYPTIAN HIEROGLYPH-13E24;Lo;0;L;;;;;N;;;;; +13E25;EGYPTIAN HIEROGLYPH-13E25;Lo;0;L;;;;;N;;;;; +13E26;EGYPTIAN HIEROGLYPH-13E26;Lo;0;L;;;;;N;;;;; +13E27;EGYPTIAN HIEROGLYPH-13E27;Lo;0;L;;;;;N;;;;; +13E28;EGYPTIAN HIEROGLYPH-13E28;Lo;0;L;;;;;N;;;;; +13E29;EGYPTIAN HIEROGLYPH-13E29;Lo;0;L;;;;;N;;;;; +13E2A;EGYPTIAN HIEROGLYPH-13E2A;Lo;0;L;;;;;N;;;;; +13E2B;EGYPTIAN HIEROGLYPH-13E2B;Lo;0;L;;;;;N;;;;; +13E2C;EGYPTIAN HIEROGLYPH-13E2C;Lo;0;L;;;;;N;;;;; +13E2D;EGYPTIAN HIEROGLYPH-13E2D;Lo;0;L;;;;;N;;;;; +13E2E;EGYPTIAN HIEROGLYPH-13E2E;Lo;0;L;;;;;N;;;;; +13E2F;EGYPTIAN HIEROGLYPH-13E2F;Lo;0;L;;;;;N;;;;; +13E30;EGYPTIAN HIEROGLYPH-13E30;Lo;0;L;;;;;N;;;;; +13E31;EGYPTIAN HIEROGLYPH-13E31;Lo;0;L;;;;;N;;;;; +13E32;EGYPTIAN HIEROGLYPH-13E32;Lo;0;L;;;;;N;;;;; +13E33;EGYPTIAN HIEROGLYPH-13E33;Lo;0;L;;;;;N;;;;; +13E34;EGYPTIAN HIEROGLYPH-13E34;Lo;0;L;;;;;N;;;;; +13E35;EGYPTIAN HIEROGLYPH-13E35;Lo;0;L;;;;;N;;;;; +13E36;EGYPTIAN HIEROGLYPH-13E36;Lo;0;L;;;;;N;;;;; +13E37;EGYPTIAN HIEROGLYPH-13E37;Lo;0;L;;;;;N;;;;; +13E38;EGYPTIAN HIEROGLYPH-13E38;Lo;0;L;;;;;N;;;;; +13E39;EGYPTIAN HIEROGLYPH-13E39;Lo;0;L;;;;;N;;;;; +13E3A;EGYPTIAN HIEROGLYPH-13E3A;Lo;0;L;;;;;N;;;;; +13E3B;EGYPTIAN HIEROGLYPH-13E3B;Lo;0;L;;;;;N;;;;; +13E3C;EGYPTIAN HIEROGLYPH-13E3C;Lo;0;L;;;;;N;;;;; +13E3D;EGYPTIAN HIEROGLYPH-13E3D;Lo;0;L;;;;;N;;;;; +13E3E;EGYPTIAN HIEROGLYPH-13E3E;Lo;0;L;;;;;N;;;;; +13E3F;EGYPTIAN HIEROGLYPH-13E3F;Lo;0;L;;;;;N;;;;; +13E40;EGYPTIAN HIEROGLYPH-13E40;Lo;0;L;;;;;N;;;;; +13E41;EGYPTIAN HIEROGLYPH-13E41;Lo;0;L;;;;;N;;;;; +13E42;EGYPTIAN HIEROGLYPH-13E42;Lo;0;L;;;;;N;;;;; +13E43;EGYPTIAN HIEROGLYPH-13E43;Lo;0;L;;;;;N;;;;; +13E44;EGYPTIAN HIEROGLYPH-13E44;Lo;0;L;;;;;N;;;;; +13E45;EGYPTIAN HIEROGLYPH-13E45;Lo;0;L;;;;;N;;;;; +13E46;EGYPTIAN HIEROGLYPH-13E46;Lo;0;L;;;;;N;;;;; +13E47;EGYPTIAN HIEROGLYPH-13E47;Lo;0;L;;;;;N;;;;; +13E48;EGYPTIAN HIEROGLYPH-13E48;Lo;0;L;;;;;N;;;;; +13E49;EGYPTIAN HIEROGLYPH-13E49;Lo;0;L;;;;;N;;;;; +13E4A;EGYPTIAN HIEROGLYPH-13E4A;Lo;0;L;;;;;N;;;;; +13E4B;EGYPTIAN HIEROGLYPH-13E4B;Lo;0;L;;;;;N;;;;; +13E4C;EGYPTIAN HIEROGLYPH-13E4C;Lo;0;L;;;;;N;;;;; +13E4D;EGYPTIAN HIEROGLYPH-13E4D;Lo;0;L;;;;;N;;;;; +13E4E;EGYPTIAN HIEROGLYPH-13E4E;Lo;0;L;;;;;N;;;;; +13E4F;EGYPTIAN HIEROGLYPH-13E4F;Lo;0;L;;;;;N;;;;; +13E50;EGYPTIAN HIEROGLYPH-13E50;Lo;0;L;;;;;N;;;;; +13E51;EGYPTIAN HIEROGLYPH-13E51;Lo;0;L;;;;;N;;;;; +13E52;EGYPTIAN HIEROGLYPH-13E52;Lo;0;L;;;;;N;;;;; +13E53;EGYPTIAN HIEROGLYPH-13E53;Lo;0;L;;;;;N;;;;; +13E54;EGYPTIAN HIEROGLYPH-13E54;Lo;0;L;;;;;N;;;;; +13E55;EGYPTIAN HIEROGLYPH-13E55;Lo;0;L;;;;;N;;;;; +13E56;EGYPTIAN HIEROGLYPH-13E56;Lo;0;L;;;;;N;;;;; +13E57;EGYPTIAN HIEROGLYPH-13E57;Lo;0;L;;;;;N;;;;; +13E58;EGYPTIAN HIEROGLYPH-13E58;Lo;0;L;;;;;N;;;;; +13E59;EGYPTIAN HIEROGLYPH-13E59;Lo;0;L;;;;;N;;;;; +13E5A;EGYPTIAN HIEROGLYPH-13E5A;Lo;0;L;;;;;N;;;;; +13E5B;EGYPTIAN HIEROGLYPH-13E5B;Lo;0;L;;;;;N;;;;; +13E5C;EGYPTIAN HIEROGLYPH-13E5C;Lo;0;L;;;;;N;;;;; +13E5D;EGYPTIAN HIEROGLYPH-13E5D;Lo;0;L;;;;;N;;;;; +13E5E;EGYPTIAN HIEROGLYPH-13E5E;Lo;0;L;;;;;N;;;;; +13E5F;EGYPTIAN HIEROGLYPH-13E5F;Lo;0;L;;;;;N;;;;; +13E60;EGYPTIAN HIEROGLYPH-13E60;Lo;0;L;;;;;N;;;;; +13E61;EGYPTIAN HIEROGLYPH-13E61;Lo;0;L;;;;;N;;;;; +13E62;EGYPTIAN HIEROGLYPH-13E62;Lo;0;L;;;;;N;;;;; +13E63;EGYPTIAN HIEROGLYPH-13E63;Lo;0;L;;;;;N;;;;; +13E64;EGYPTIAN HIEROGLYPH-13E64;Lo;0;L;;;;;N;;;;; +13E65;EGYPTIAN HIEROGLYPH-13E65;Lo;0;L;;;;;N;;;;; +13E66;EGYPTIAN HIEROGLYPH-13E66;Lo;0;L;;;;;N;;;;; +13E67;EGYPTIAN HIEROGLYPH-13E67;Lo;0;L;;;;;N;;;;; +13E68;EGYPTIAN HIEROGLYPH-13E68;Lo;0;L;;;;;N;;;;; +13E69;EGYPTIAN HIEROGLYPH-13E69;Lo;0;L;;;;;N;;;;; +13E6A;EGYPTIAN HIEROGLYPH-13E6A;Lo;0;L;;;;;N;;;;; +13E6B;EGYPTIAN HIEROGLYPH-13E6B;Lo;0;L;;;;;N;;;;; +13E6C;EGYPTIAN HIEROGLYPH-13E6C;Lo;0;L;;;;;N;;;;; +13E6D;EGYPTIAN HIEROGLYPH-13E6D;Lo;0;L;;;;;N;;;;; +13E6E;EGYPTIAN HIEROGLYPH-13E6E;Lo;0;L;;;;;N;;;;; +13E6F;EGYPTIAN HIEROGLYPH-13E6F;Lo;0;L;;;;;N;;;;; +13E70;EGYPTIAN HIEROGLYPH-13E70;Lo;0;L;;;;;N;;;;; +13E71;EGYPTIAN HIEROGLYPH-13E71;Lo;0;L;;;;;N;;;;; +13E72;EGYPTIAN HIEROGLYPH-13E72;Lo;0;L;;;;;N;;;;; +13E73;EGYPTIAN HIEROGLYPH-13E73;Lo;0;L;;;;;N;;;;; +13E74;EGYPTIAN HIEROGLYPH-13E74;Lo;0;L;;;;;N;;;;; +13E75;EGYPTIAN HIEROGLYPH-13E75;Lo;0;L;;;;;N;;;;; +13E76;EGYPTIAN HIEROGLYPH-13E76;Lo;0;L;;;;;N;;;;; +13E77;EGYPTIAN HIEROGLYPH-13E77;Lo;0;L;;;;;N;;;;; +13E78;EGYPTIAN HIEROGLYPH-13E78;Lo;0;L;;;;;N;;;;; +13E79;EGYPTIAN HIEROGLYPH-13E79;Lo;0;L;;;;;N;;;;; +13E7A;EGYPTIAN HIEROGLYPH-13E7A;Lo;0;L;;;;;N;;;;; +13E7B;EGYPTIAN HIEROGLYPH-13E7B;Lo;0;L;;;;;N;;;;; +13E7C;EGYPTIAN HIEROGLYPH-13E7C;Lo;0;L;;;;;N;;;;; +13E7D;EGYPTIAN HIEROGLYPH-13E7D;Lo;0;L;;;;;N;;;;; +13E7E;EGYPTIAN HIEROGLYPH-13E7E;Lo;0;L;;;;;N;;;;; +13E7F;EGYPTIAN HIEROGLYPH-13E7F;Lo;0;L;;;;;N;;;;; +13E80;EGYPTIAN HIEROGLYPH-13E80;Lo;0;L;;;;;N;;;;; +13E81;EGYPTIAN HIEROGLYPH-13E81;Lo;0;L;;;;;N;;;;; +13E82;EGYPTIAN HIEROGLYPH-13E82;Lo;0;L;;;;;N;;;;; +13E83;EGYPTIAN HIEROGLYPH-13E83;Lo;0;L;;;;;N;;;;; +13E84;EGYPTIAN HIEROGLYPH-13E84;Lo;0;L;;;;;N;;;;; +13E85;EGYPTIAN HIEROGLYPH-13E85;Lo;0;L;;;;;N;;;;; +13E86;EGYPTIAN HIEROGLYPH-13E86;Lo;0;L;;;;;N;;;;; +13E87;EGYPTIAN HIEROGLYPH-13E87;Lo;0;L;;;;;N;;;;; +13E88;EGYPTIAN HIEROGLYPH-13E88;Lo;0;L;;;;;N;;;;; +13E89;EGYPTIAN HIEROGLYPH-13E89;Lo;0;L;;;;;N;;;;; +13E8A;EGYPTIAN HIEROGLYPH-13E8A;Lo;0;L;;;;;N;;;;; +13E8B;EGYPTIAN HIEROGLYPH-13E8B;Lo;0;L;;;;;N;;;;; +13E8C;EGYPTIAN HIEROGLYPH-13E8C;Lo;0;L;;;;;N;;;;; +13E8D;EGYPTIAN HIEROGLYPH-13E8D;Lo;0;L;;;;;N;;;;; +13E8E;EGYPTIAN HIEROGLYPH-13E8E;Lo;0;L;;;;;N;;;;; +13E8F;EGYPTIAN HIEROGLYPH-13E8F;Lo;0;L;;;;;N;;;;; +13E90;EGYPTIAN HIEROGLYPH-13E90;Lo;0;L;;;;;N;;;;; +13E91;EGYPTIAN HIEROGLYPH-13E91;Lo;0;L;;;;;N;;;;; +13E92;EGYPTIAN HIEROGLYPH-13E92;Lo;0;L;;;;;N;;;;; +13E93;EGYPTIAN HIEROGLYPH-13E93;Lo;0;L;;;;;N;;;;; +13E94;EGYPTIAN HIEROGLYPH-13E94;Lo;0;L;;;;;N;;;;; +13E95;EGYPTIAN HIEROGLYPH-13E95;Lo;0;L;;;;;N;;;;; +13E96;EGYPTIAN HIEROGLYPH-13E96;Lo;0;L;;;;;N;;;;; +13E97;EGYPTIAN HIEROGLYPH-13E97;Lo;0;L;;;;;N;;;;; +13E98;EGYPTIAN HIEROGLYPH-13E98;Lo;0;L;;;;;N;;;;; +13E99;EGYPTIAN HIEROGLYPH-13E99;Lo;0;L;;;;;N;;;;; +13E9A;EGYPTIAN HIEROGLYPH-13E9A;Lo;0;L;;;;;N;;;;; +13E9B;EGYPTIAN HIEROGLYPH-13E9B;Lo;0;L;;;;;N;;;;; +13E9C;EGYPTIAN HIEROGLYPH-13E9C;Lo;0;L;;;;;N;;;;; +13E9D;EGYPTIAN HIEROGLYPH-13E9D;Lo;0;L;;;;;N;;;;; +13E9E;EGYPTIAN HIEROGLYPH-13E9E;Lo;0;L;;;;;N;;;;; +13E9F;EGYPTIAN HIEROGLYPH-13E9F;Lo;0;L;;;;;N;;;;; +13EA0;EGYPTIAN HIEROGLYPH-13EA0;Lo;0;L;;;;;N;;;;; +13EA1;EGYPTIAN HIEROGLYPH-13EA1;Lo;0;L;;;;;N;;;;; +13EA2;EGYPTIAN HIEROGLYPH-13EA2;Lo;0;L;;;;;N;;;;; +13EA3;EGYPTIAN HIEROGLYPH-13EA3;Lo;0;L;;;;;N;;;;; +13EA4;EGYPTIAN HIEROGLYPH-13EA4;Lo;0;L;;;;;N;;;;; +13EA5;EGYPTIAN HIEROGLYPH-13EA5;Lo;0;L;;;;;N;;;;; +13EA6;EGYPTIAN HIEROGLYPH-13EA6;Lo;0;L;;;;;N;;;;; +13EA7;EGYPTIAN HIEROGLYPH-13EA7;Lo;0;L;;;;;N;;;;; +13EA8;EGYPTIAN HIEROGLYPH-13EA8;Lo;0;L;;;;;N;;;;; +13EA9;EGYPTIAN HIEROGLYPH-13EA9;Lo;0;L;;;;;N;;;;; +13EAA;EGYPTIAN HIEROGLYPH-13EAA;Lo;0;L;;;;;N;;;;; +13EAB;EGYPTIAN HIEROGLYPH-13EAB;Lo;0;L;;;;;N;;;;; +13EAC;EGYPTIAN HIEROGLYPH-13EAC;Lo;0;L;;;;;N;;;;; +13EAD;EGYPTIAN HIEROGLYPH-13EAD;Lo;0;L;;;;;N;;;;; +13EAE;EGYPTIAN HIEROGLYPH-13EAE;Lo;0;L;;;;;N;;;;; +13EAF;EGYPTIAN HIEROGLYPH-13EAF;Lo;0;L;;;;;N;;;;; +13EB0;EGYPTIAN HIEROGLYPH-13EB0;Lo;0;L;;;;;N;;;;; +13EB1;EGYPTIAN HIEROGLYPH-13EB1;Lo;0;L;;;;;N;;;;; +13EB2;EGYPTIAN HIEROGLYPH-13EB2;Lo;0;L;;;;;N;;;;; +13EB3;EGYPTIAN HIEROGLYPH-13EB3;Lo;0;L;;;;;N;;;;; +13EB4;EGYPTIAN HIEROGLYPH-13EB4;Lo;0;L;;;;;N;;;;; +13EB5;EGYPTIAN HIEROGLYPH-13EB5;Lo;0;L;;;;;N;;;;; +13EB6;EGYPTIAN HIEROGLYPH-13EB6;Lo;0;L;;;;;N;;;;; +13EB7;EGYPTIAN HIEROGLYPH-13EB7;Lo;0;L;;;;;N;;;;; +13EB8;EGYPTIAN HIEROGLYPH-13EB8;Lo;0;L;;;;;N;;;;; +13EB9;EGYPTIAN HIEROGLYPH-13EB9;Lo;0;L;;;;;N;;;;; +13EBA;EGYPTIAN HIEROGLYPH-13EBA;Lo;0;L;;;;;N;;;;; +13EBB;EGYPTIAN HIEROGLYPH-13EBB;Lo;0;L;;;;;N;;;;; +13EBC;EGYPTIAN HIEROGLYPH-13EBC;Lo;0;L;;;;;N;;;;; +13EBD;EGYPTIAN HIEROGLYPH-13EBD;Lo;0;L;;;;;N;;;;; +13EBE;EGYPTIAN HIEROGLYPH-13EBE;Lo;0;L;;;;;N;;;;; +13EBF;EGYPTIAN HIEROGLYPH-13EBF;Lo;0;L;;;;;N;;;;; +13EC0;EGYPTIAN HIEROGLYPH-13EC0;Lo;0;L;;;;;N;;;;; +13EC1;EGYPTIAN HIEROGLYPH-13EC1;Lo;0;L;;;;;N;;;;; +13EC2;EGYPTIAN HIEROGLYPH-13EC2;Lo;0;L;;;;;N;;;;; +13EC3;EGYPTIAN HIEROGLYPH-13EC3;Lo;0;L;;;;;N;;;;; +13EC4;EGYPTIAN HIEROGLYPH-13EC4;Lo;0;L;;;;;N;;;;; +13EC5;EGYPTIAN HIEROGLYPH-13EC5;Lo;0;L;;;;;N;;;;; +13EC6;EGYPTIAN HIEROGLYPH-13EC6;Lo;0;L;;;;;N;;;;; +13EC7;EGYPTIAN HIEROGLYPH-13EC7;Lo;0;L;;;;;N;;;;; +13EC8;EGYPTIAN HIEROGLYPH-13EC8;Lo;0;L;;;;;N;;;;; +13EC9;EGYPTIAN HIEROGLYPH-13EC9;Lo;0;L;;;;;N;;;;; +13ECA;EGYPTIAN HIEROGLYPH-13ECA;Lo;0;L;;;;;N;;;;; +13ECB;EGYPTIAN HIEROGLYPH-13ECB;Lo;0;L;;;;;N;;;;; +13ECC;EGYPTIAN HIEROGLYPH-13ECC;Lo;0;L;;;;;N;;;;; +13ECD;EGYPTIAN HIEROGLYPH-13ECD;Lo;0;L;;;;;N;;;;; +13ECE;EGYPTIAN HIEROGLYPH-13ECE;Lo;0;L;;;;;N;;;;; +13ECF;EGYPTIAN HIEROGLYPH-13ECF;Lo;0;L;;;;;N;;;;; +13ED0;EGYPTIAN HIEROGLYPH-13ED0;Lo;0;L;;;;;N;;;;; +13ED1;EGYPTIAN HIEROGLYPH-13ED1;Lo;0;L;;;;;N;;;;; +13ED2;EGYPTIAN HIEROGLYPH-13ED2;Lo;0;L;;;;;N;;;;; +13ED3;EGYPTIAN HIEROGLYPH-13ED3;Lo;0;L;;;;;N;;;;; +13ED4;EGYPTIAN HIEROGLYPH-13ED4;Lo;0;L;;;;;N;;;;; +13ED5;EGYPTIAN HIEROGLYPH-13ED5;Lo;0;L;;;;;N;;;;; +13ED6;EGYPTIAN HIEROGLYPH-13ED6;Lo;0;L;;;;;N;;;;; +13ED7;EGYPTIAN HIEROGLYPH-13ED7;Lo;0;L;;;;;N;;;;; +13ED8;EGYPTIAN HIEROGLYPH-13ED8;Lo;0;L;;;;;N;;;;; +13ED9;EGYPTIAN HIEROGLYPH-13ED9;Lo;0;L;;;;;N;;;;; +13EDA;EGYPTIAN HIEROGLYPH-13EDA;Lo;0;L;;;;;N;;;;; +13EDB;EGYPTIAN HIEROGLYPH-13EDB;Lo;0;L;;;;;N;;;;; +13EDC;EGYPTIAN HIEROGLYPH-13EDC;Lo;0;L;;;;;N;;;;; +13EDD;EGYPTIAN HIEROGLYPH-13EDD;Lo;0;L;;;;;N;;;;; +13EDE;EGYPTIAN HIEROGLYPH-13EDE;Lo;0;L;;;;;N;;;;; +13EDF;EGYPTIAN HIEROGLYPH-13EDF;Lo;0;L;;;;;N;;;;; +13EE0;EGYPTIAN HIEROGLYPH-13EE0;Lo;0;L;;;;;N;;;;; +13EE1;EGYPTIAN HIEROGLYPH-13EE1;Lo;0;L;;;;;N;;;;; +13EE2;EGYPTIAN HIEROGLYPH-13EE2;Lo;0;L;;;;;N;;;;; +13EE3;EGYPTIAN HIEROGLYPH-13EE3;Lo;0;L;;;;;N;;;;; +13EE4;EGYPTIAN HIEROGLYPH-13EE4;Lo;0;L;;;;;N;;;;; +13EE5;EGYPTIAN HIEROGLYPH-13EE5;Lo;0;L;;;;;N;;;;; +13EE6;EGYPTIAN HIEROGLYPH-13EE6;Lo;0;L;;;;;N;;;;; +13EE7;EGYPTIAN HIEROGLYPH-13EE7;Lo;0;L;;;;;N;;;;; +13EE8;EGYPTIAN HIEROGLYPH-13EE8;Lo;0;L;;;;;N;;;;; +13EE9;EGYPTIAN HIEROGLYPH-13EE9;Lo;0;L;;;;;N;;;;; +13EEA;EGYPTIAN HIEROGLYPH-13EEA;Lo;0;L;;;;;N;;;;; +13EEB;EGYPTIAN HIEROGLYPH-13EEB;Lo;0;L;;;;;N;;;;; +13EEC;EGYPTIAN HIEROGLYPH-13EEC;Lo;0;L;;;;;N;;;;; +13EED;EGYPTIAN HIEROGLYPH-13EED;Lo;0;L;;;;;N;;;;; +13EEE;EGYPTIAN HIEROGLYPH-13EEE;Lo;0;L;;;;;N;;;;; +13EEF;EGYPTIAN HIEROGLYPH-13EEF;Lo;0;L;;;;;N;;;;; +13EF0;EGYPTIAN HIEROGLYPH-13EF0;Lo;0;L;;;;;N;;;;; +13EF1;EGYPTIAN HIEROGLYPH-13EF1;Lo;0;L;;;;;N;;;;; +13EF2;EGYPTIAN HIEROGLYPH-13EF2;Lo;0;L;;;;;N;;;;; +13EF3;EGYPTIAN HIEROGLYPH-13EF3;Lo;0;L;;;;;N;;;;; +13EF4;EGYPTIAN HIEROGLYPH-13EF4;Lo;0;L;;;;;N;;;;; +13EF5;EGYPTIAN HIEROGLYPH-13EF5;Lo;0;L;;;;;N;;;;; +13EF6;EGYPTIAN HIEROGLYPH-13EF6;Lo;0;L;;;;;N;;;;; +13EF7;EGYPTIAN HIEROGLYPH-13EF7;Lo;0;L;;;;;N;;;;; +13EF8;EGYPTIAN HIEROGLYPH-13EF8;Lo;0;L;;;;;N;;;;; +13EF9;EGYPTIAN HIEROGLYPH-13EF9;Lo;0;L;;;;;N;;;;; +13EFA;EGYPTIAN HIEROGLYPH-13EFA;Lo;0;L;;;;;N;;;;; +13EFB;EGYPTIAN HIEROGLYPH-13EFB;Lo;0;L;;;;;N;;;;; +13EFC;EGYPTIAN HIEROGLYPH-13EFC;Lo;0;L;;;;;N;;;;; +13EFD;EGYPTIAN HIEROGLYPH-13EFD;Lo;0;L;;;;;N;;;;; +13EFE;EGYPTIAN HIEROGLYPH-13EFE;Lo;0;L;;;;;N;;;;; +13EFF;EGYPTIAN HIEROGLYPH-13EFF;Lo;0;L;;;;;N;;;;; +13F00;EGYPTIAN HIEROGLYPH-13F00;Lo;0;L;;;;;N;;;;; +13F01;EGYPTIAN HIEROGLYPH-13F01;Lo;0;L;;;;;N;;;;; +13F02;EGYPTIAN HIEROGLYPH-13F02;Lo;0;L;;;;;N;;;;; +13F03;EGYPTIAN HIEROGLYPH-13F03;Lo;0;L;;;;;N;;;;; +13F04;EGYPTIAN HIEROGLYPH-13F04;Lo;0;L;;;;;N;;;;; +13F05;EGYPTIAN HIEROGLYPH-13F05;Lo;0;L;;;;;N;;;;; +13F06;EGYPTIAN HIEROGLYPH-13F06;Lo;0;L;;;;;N;;;;; +13F07;EGYPTIAN HIEROGLYPH-13F07;Lo;0;L;;;;;N;;;;; +13F08;EGYPTIAN HIEROGLYPH-13F08;Lo;0;L;;;;;N;;;;; +13F09;EGYPTIAN HIEROGLYPH-13F09;Lo;0;L;;;;;N;;;;; +13F0A;EGYPTIAN HIEROGLYPH-13F0A;Lo;0;L;;;;;N;;;;; +13F0B;EGYPTIAN HIEROGLYPH-13F0B;Lo;0;L;;;;;N;;;;; +13F0C;EGYPTIAN HIEROGLYPH-13F0C;Lo;0;L;;;;;N;;;;; +13F0D;EGYPTIAN HIEROGLYPH-13F0D;Lo;0;L;;;;;N;;;;; +13F0E;EGYPTIAN HIEROGLYPH-13F0E;Lo;0;L;;;;;N;;;;; +13F0F;EGYPTIAN HIEROGLYPH-13F0F;Lo;0;L;;;;;N;;;;; +13F10;EGYPTIAN HIEROGLYPH-13F10;Lo;0;L;;;;;N;;;;; +13F11;EGYPTIAN HIEROGLYPH-13F11;Lo;0;L;;;;;N;;;;; +13F12;EGYPTIAN HIEROGLYPH-13F12;Lo;0;L;;;;;N;;;;; +13F13;EGYPTIAN HIEROGLYPH-13F13;Lo;0;L;;;;;N;;;;; +13F14;EGYPTIAN HIEROGLYPH-13F14;Lo;0;L;;;;;N;;;;; +13F15;EGYPTIAN HIEROGLYPH-13F15;Lo;0;L;;;;;N;;;;; +13F16;EGYPTIAN HIEROGLYPH-13F16;Lo;0;L;;;;;N;;;;; +13F17;EGYPTIAN HIEROGLYPH-13F17;Lo;0;L;;;;;N;;;;; +13F18;EGYPTIAN HIEROGLYPH-13F18;Lo;0;L;;;;;N;;;;; +13F19;EGYPTIAN HIEROGLYPH-13F19;Lo;0;L;;;;;N;;;;; +13F1A;EGYPTIAN HIEROGLYPH-13F1A;Lo;0;L;;;;;N;;;;; +13F1B;EGYPTIAN HIEROGLYPH-13F1B;Lo;0;L;;;;;N;;;;; +13F1C;EGYPTIAN HIEROGLYPH-13F1C;Lo;0;L;;;;;N;;;;; +13F1D;EGYPTIAN HIEROGLYPH-13F1D;Lo;0;L;;;;;N;;;;; +13F1E;EGYPTIAN HIEROGLYPH-13F1E;Lo;0;L;;;;;N;;;;; +13F1F;EGYPTIAN HIEROGLYPH-13F1F;Lo;0;L;;;;;N;;;;; +13F20;EGYPTIAN HIEROGLYPH-13F20;Lo;0;L;;;;;N;;;;; +13F21;EGYPTIAN HIEROGLYPH-13F21;Lo;0;L;;;;;N;;;;; +13F22;EGYPTIAN HIEROGLYPH-13F22;Lo;0;L;;;;;N;;;;; +13F23;EGYPTIAN HIEROGLYPH-13F23;Lo;0;L;;;;;N;;;;; +13F24;EGYPTIAN HIEROGLYPH-13F24;Lo;0;L;;;;;N;;;;; +13F25;EGYPTIAN HIEROGLYPH-13F25;Lo;0;L;;;;;N;;;;; +13F26;EGYPTIAN HIEROGLYPH-13F26;Lo;0;L;;;;;N;;;;; +13F27;EGYPTIAN HIEROGLYPH-13F27;Lo;0;L;;;;;N;;;;; +13F28;EGYPTIAN HIEROGLYPH-13F28;Lo;0;L;;;;;N;;;;; +13F29;EGYPTIAN HIEROGLYPH-13F29;Lo;0;L;;;;;N;;;;; +13F2A;EGYPTIAN HIEROGLYPH-13F2A;Lo;0;L;;;;;N;;;;; +13F2B;EGYPTIAN HIEROGLYPH-13F2B;Lo;0;L;;;;;N;;;;; +13F2C;EGYPTIAN HIEROGLYPH-13F2C;Lo;0;L;;;;;N;;;;; +13F2D;EGYPTIAN HIEROGLYPH-13F2D;Lo;0;L;;;;;N;;;;; +13F2E;EGYPTIAN HIEROGLYPH-13F2E;Lo;0;L;;;;;N;;;;; +13F2F;EGYPTIAN HIEROGLYPH-13F2F;Lo;0;L;;;;;N;;;;; +13F30;EGYPTIAN HIEROGLYPH-13F30;Lo;0;L;;;;;N;;;;; +13F31;EGYPTIAN HIEROGLYPH-13F31;Lo;0;L;;;;;N;;;;; +13F32;EGYPTIAN HIEROGLYPH-13F32;Lo;0;L;;;;;N;;;;; +13F33;EGYPTIAN HIEROGLYPH-13F33;Lo;0;L;;;;;N;;;;; +13F34;EGYPTIAN HIEROGLYPH-13F34;Lo;0;L;;;;;N;;;;; +13F35;EGYPTIAN HIEROGLYPH-13F35;Lo;0;L;;;;;N;;;;; +13F36;EGYPTIAN HIEROGLYPH-13F36;Lo;0;L;;;;;N;;;;; +13F37;EGYPTIAN HIEROGLYPH-13F37;Lo;0;L;;;;;N;;;;; +13F38;EGYPTIAN HIEROGLYPH-13F38;Lo;0;L;;;;;N;;;;; +13F39;EGYPTIAN HIEROGLYPH-13F39;Lo;0;L;;;;;N;;;;; +13F3A;EGYPTIAN HIEROGLYPH-13F3A;Lo;0;L;;;;;N;;;;; +13F3B;EGYPTIAN HIEROGLYPH-13F3B;Lo;0;L;;;;;N;;;;; +13F3C;EGYPTIAN HIEROGLYPH-13F3C;Lo;0;L;;;;;N;;;;; +13F3D;EGYPTIAN HIEROGLYPH-13F3D;Lo;0;L;;;;;N;;;;; +13F3E;EGYPTIAN HIEROGLYPH-13F3E;Lo;0;L;;;;;N;;;;; +13F3F;EGYPTIAN HIEROGLYPH-13F3F;Lo;0;L;;;;;N;;;;; +13F40;EGYPTIAN HIEROGLYPH-13F40;Lo;0;L;;;;;N;;;;; +13F41;EGYPTIAN HIEROGLYPH-13F41;Lo;0;L;;;;;N;;;;; +13F42;EGYPTIAN HIEROGLYPH-13F42;Lo;0;L;;;;;N;;;;; +13F43;EGYPTIAN HIEROGLYPH-13F43;Lo;0;L;;;;;N;;;;; +13F44;EGYPTIAN HIEROGLYPH-13F44;Lo;0;L;;;;;N;;;;; +13F45;EGYPTIAN HIEROGLYPH-13F45;Lo;0;L;;;;;N;;;;; +13F46;EGYPTIAN HIEROGLYPH-13F46;Lo;0;L;;;;;N;;;;; +13F47;EGYPTIAN HIEROGLYPH-13F47;Lo;0;L;;;;;N;;;;; +13F48;EGYPTIAN HIEROGLYPH-13F48;Lo;0;L;;;;;N;;;;; +13F49;EGYPTIAN HIEROGLYPH-13F49;Lo;0;L;;;;;N;;;;; +13F4A;EGYPTIAN HIEROGLYPH-13F4A;Lo;0;L;;;;;N;;;;; +13F4B;EGYPTIAN HIEROGLYPH-13F4B;Lo;0;L;;;;;N;;;;; +13F4C;EGYPTIAN HIEROGLYPH-13F4C;Lo;0;L;;;;;N;;;;; +13F4D;EGYPTIAN HIEROGLYPH-13F4D;Lo;0;L;;;;;N;;;;; +13F4E;EGYPTIAN HIEROGLYPH-13F4E;Lo;0;L;;;;;N;;;;; +13F4F;EGYPTIAN HIEROGLYPH-13F4F;Lo;0;L;;;;;N;;;;; +13F50;EGYPTIAN HIEROGLYPH-13F50;Lo;0;L;;;;;N;;;;; +13F51;EGYPTIAN HIEROGLYPH-13F51;Lo;0;L;;;;;N;;;;; +13F52;EGYPTIAN HIEROGLYPH-13F52;Lo;0;L;;;;;N;;;;; +13F53;EGYPTIAN HIEROGLYPH-13F53;Lo;0;L;;;;;N;;;;; +13F54;EGYPTIAN HIEROGLYPH-13F54;Lo;0;L;;;;;N;;;;; +13F55;EGYPTIAN HIEROGLYPH-13F55;Lo;0;L;;;;;N;;;;; +13F56;EGYPTIAN HIEROGLYPH-13F56;Lo;0;L;;;;;N;;;;; +13F57;EGYPTIAN HIEROGLYPH-13F57;Lo;0;L;;;;;N;;;;; +13F58;EGYPTIAN HIEROGLYPH-13F58;Lo;0;L;;;;;N;;;;; +13F59;EGYPTIAN HIEROGLYPH-13F59;Lo;0;L;;;;;N;;;;; +13F5A;EGYPTIAN HIEROGLYPH-13F5A;Lo;0;L;;;;;N;;;;; +13F5B;EGYPTIAN HIEROGLYPH-13F5B;Lo;0;L;;;;;N;;;;; +13F5C;EGYPTIAN HIEROGLYPH-13F5C;Lo;0;L;;;;;N;;;;; +13F5D;EGYPTIAN HIEROGLYPH-13F5D;Lo;0;L;;;;;N;;;;; +13F5E;EGYPTIAN HIEROGLYPH-13F5E;Lo;0;L;;;;;N;;;;; +13F5F;EGYPTIAN HIEROGLYPH-13F5F;Lo;0;L;;;;;N;;;;; +13F60;EGYPTIAN HIEROGLYPH-13F60;Lo;0;L;;;;;N;;;;; +13F61;EGYPTIAN HIEROGLYPH-13F61;Lo;0;L;;;;;N;;;;; +13F62;EGYPTIAN HIEROGLYPH-13F62;Lo;0;L;;;;;N;;;;; +13F63;EGYPTIAN HIEROGLYPH-13F63;Lo;0;L;;;;;N;;;;; +13F64;EGYPTIAN HIEROGLYPH-13F64;Lo;0;L;;;;;N;;;;; +13F65;EGYPTIAN HIEROGLYPH-13F65;Lo;0;L;;;;;N;;;;; +13F66;EGYPTIAN HIEROGLYPH-13F66;Lo;0;L;;;;;N;;;;; +13F67;EGYPTIAN HIEROGLYPH-13F67;Lo;0;L;;;;;N;;;;; +13F68;EGYPTIAN HIEROGLYPH-13F68;Lo;0;L;;;;;N;;;;; +13F69;EGYPTIAN HIEROGLYPH-13F69;Lo;0;L;;;;;N;;;;; +13F6A;EGYPTIAN HIEROGLYPH-13F6A;Lo;0;L;;;;;N;;;;; +13F6B;EGYPTIAN HIEROGLYPH-13F6B;Lo;0;L;;;;;N;;;;; +13F6C;EGYPTIAN HIEROGLYPH-13F6C;Lo;0;L;;;;;N;;;;; +13F6D;EGYPTIAN HIEROGLYPH-13F6D;Lo;0;L;;;;;N;;;;; +13F6E;EGYPTIAN HIEROGLYPH-13F6E;Lo;0;L;;;;;N;;;;; +13F6F;EGYPTIAN HIEROGLYPH-13F6F;Lo;0;L;;;;;N;;;;; +13F70;EGYPTIAN HIEROGLYPH-13F70;Lo;0;L;;;;;N;;;;; +13F71;EGYPTIAN HIEROGLYPH-13F71;Lo;0;L;;;;;N;;;;; +13F72;EGYPTIAN HIEROGLYPH-13F72;Lo;0;L;;;;;N;;;;; +13F73;EGYPTIAN HIEROGLYPH-13F73;Lo;0;L;;;;;N;;;;; +13F74;EGYPTIAN HIEROGLYPH-13F74;Lo;0;L;;;;;N;;;;; +13F75;EGYPTIAN HIEROGLYPH-13F75;Lo;0;L;;;;;N;;;;; +13F76;EGYPTIAN HIEROGLYPH-13F76;Lo;0;L;;;;;N;;;;; +13F77;EGYPTIAN HIEROGLYPH-13F77;Lo;0;L;;;;;N;;;;; +13F78;EGYPTIAN HIEROGLYPH-13F78;Lo;0;L;;;;;N;;;;; +13F79;EGYPTIAN HIEROGLYPH-13F79;Lo;0;L;;;;;N;;;;; +13F7A;EGYPTIAN HIEROGLYPH-13F7A;Lo;0;L;;;;;N;;;;; +13F7B;EGYPTIAN HIEROGLYPH-13F7B;Lo;0;L;;;;;N;;;;; +13F7C;EGYPTIAN HIEROGLYPH-13F7C;Lo;0;L;;;;;N;;;;; +13F7D;EGYPTIAN HIEROGLYPH-13F7D;Lo;0;L;;;;;N;;;;; +13F7E;EGYPTIAN HIEROGLYPH-13F7E;Lo;0;L;;;;;N;;;;; +13F7F;EGYPTIAN HIEROGLYPH-13F7F;Lo;0;L;;;;;N;;;;; +13F80;EGYPTIAN HIEROGLYPH-13F80;Lo;0;L;;;;;N;;;;; +13F81;EGYPTIAN HIEROGLYPH-13F81;Lo;0;L;;;;;N;;;;; +13F82;EGYPTIAN HIEROGLYPH-13F82;Lo;0;L;;;;;N;;;;; +13F83;EGYPTIAN HIEROGLYPH-13F83;Lo;0;L;;;;;N;;;;; +13F84;EGYPTIAN HIEROGLYPH-13F84;Lo;0;L;;;;;N;;;;; +13F85;EGYPTIAN HIEROGLYPH-13F85;Lo;0;L;;;;;N;;;;; +13F86;EGYPTIAN HIEROGLYPH-13F86;Lo;0;L;;;;;N;;;;; +13F87;EGYPTIAN HIEROGLYPH-13F87;Lo;0;L;;;;;N;;;;; +13F88;EGYPTIAN HIEROGLYPH-13F88;Lo;0;L;;;;;N;;;;; +13F89;EGYPTIAN HIEROGLYPH-13F89;Lo;0;L;;;;;N;;;;; +13F8A;EGYPTIAN HIEROGLYPH-13F8A;Lo;0;L;;;;;N;;;;; +13F8B;EGYPTIAN HIEROGLYPH-13F8B;Lo;0;L;;;;;N;;;;; +13F8C;EGYPTIAN HIEROGLYPH-13F8C;Lo;0;L;;;;;N;;;;; +13F8D;EGYPTIAN HIEROGLYPH-13F8D;Lo;0;L;;;;;N;;;;; +13F8E;EGYPTIAN HIEROGLYPH-13F8E;Lo;0;L;;;;;N;;;;; +13F8F;EGYPTIAN HIEROGLYPH-13F8F;Lo;0;L;;;;;N;;;;; +13F90;EGYPTIAN HIEROGLYPH-13F90;Lo;0;L;;;;;N;;;;; +13F91;EGYPTIAN HIEROGLYPH-13F91;Lo;0;L;;;;;N;;;;; +13F92;EGYPTIAN HIEROGLYPH-13F92;Lo;0;L;;;;;N;;;;; +13F93;EGYPTIAN HIEROGLYPH-13F93;Lo;0;L;;;;;N;;;;; +13F94;EGYPTIAN HIEROGLYPH-13F94;Lo;0;L;;;;;N;;;;; +13F95;EGYPTIAN HIEROGLYPH-13F95;Lo;0;L;;;;;N;;;;; +13F96;EGYPTIAN HIEROGLYPH-13F96;Lo;0;L;;;;;N;;;;; +13F97;EGYPTIAN HIEROGLYPH-13F97;Lo;0;L;;;;;N;;;;; +13F98;EGYPTIAN HIEROGLYPH-13F98;Lo;0;L;;;;;N;;;;; +13F99;EGYPTIAN HIEROGLYPH-13F99;Lo;0;L;;;;;N;;;;; +13F9A;EGYPTIAN HIEROGLYPH-13F9A;Lo;0;L;;;;;N;;;;; +13F9B;EGYPTIAN HIEROGLYPH-13F9B;Lo;0;L;;;;;N;;;;; +13F9C;EGYPTIAN HIEROGLYPH-13F9C;Lo;0;L;;;;;N;;;;; +13F9D;EGYPTIAN HIEROGLYPH-13F9D;Lo;0;L;;;;;N;;;;; +13F9E;EGYPTIAN HIEROGLYPH-13F9E;Lo;0;L;;;;;N;;;;; +13F9F;EGYPTIAN HIEROGLYPH-13F9F;Lo;0;L;;;;;N;;;;; +13FA0;EGYPTIAN HIEROGLYPH-13FA0;Lo;0;L;;;;;N;;;;; +13FA1;EGYPTIAN HIEROGLYPH-13FA1;Lo;0;L;;;;;N;;;;; +13FA2;EGYPTIAN HIEROGLYPH-13FA2;Lo;0;L;;;;;N;;;;; +13FA3;EGYPTIAN HIEROGLYPH-13FA3;Lo;0;L;;;;;N;;;;; +13FA4;EGYPTIAN HIEROGLYPH-13FA4;Lo;0;L;;;;;N;;;;; +13FA5;EGYPTIAN HIEROGLYPH-13FA5;Lo;0;L;;;;;N;;;;; +13FA6;EGYPTIAN HIEROGLYPH-13FA6;Lo;0;L;;;;;N;;;;; +13FA7;EGYPTIAN HIEROGLYPH-13FA7;Lo;0;L;;;;;N;;;;; +13FA8;EGYPTIAN HIEROGLYPH-13FA8;Lo;0;L;;;;;N;;;;; +13FA9;EGYPTIAN HIEROGLYPH-13FA9;Lo;0;L;;;;;N;;;;; +13FAA;EGYPTIAN HIEROGLYPH-13FAA;Lo;0;L;;;;;N;;;;; +13FAB;EGYPTIAN HIEROGLYPH-13FAB;Lo;0;L;;;;;N;;;;; +13FAC;EGYPTIAN HIEROGLYPH-13FAC;Lo;0;L;;;;;N;;;;; +13FAD;EGYPTIAN HIEROGLYPH-13FAD;Lo;0;L;;;;;N;;;;; +13FAE;EGYPTIAN HIEROGLYPH-13FAE;Lo;0;L;;;;;N;;;;; +13FAF;EGYPTIAN HIEROGLYPH-13FAF;Lo;0;L;;;;;N;;;;; +13FB0;EGYPTIAN HIEROGLYPH-13FB0;Lo;0;L;;;;;N;;;;; +13FB1;EGYPTIAN HIEROGLYPH-13FB1;Lo;0;L;;;;;N;;;;; +13FB2;EGYPTIAN HIEROGLYPH-13FB2;Lo;0;L;;;;;N;;;;; +13FB3;EGYPTIAN HIEROGLYPH-13FB3;Lo;0;L;;;;;N;;;;; +13FB4;EGYPTIAN HIEROGLYPH-13FB4;Lo;0;L;;;;;N;;;;; +13FB5;EGYPTIAN HIEROGLYPH-13FB5;Lo;0;L;;;;;N;;;;; +13FB6;EGYPTIAN HIEROGLYPH-13FB6;Lo;0;L;;;;;N;;;;; +13FB7;EGYPTIAN HIEROGLYPH-13FB7;Lo;0;L;;;;;N;;;;; +13FB8;EGYPTIAN HIEROGLYPH-13FB8;Lo;0;L;;;;;N;;;;; +13FB9;EGYPTIAN HIEROGLYPH-13FB9;Lo;0;L;;;;;N;;;;; +13FBA;EGYPTIAN HIEROGLYPH-13FBA;Lo;0;L;;;;;N;;;;; +13FBB;EGYPTIAN HIEROGLYPH-13FBB;Lo;0;L;;;;;N;;;;; +13FBC;EGYPTIAN HIEROGLYPH-13FBC;Lo;0;L;;;;;N;;;;; +13FBD;EGYPTIAN HIEROGLYPH-13FBD;Lo;0;L;;;;;N;;;;; +13FBE;EGYPTIAN HIEROGLYPH-13FBE;Lo;0;L;;;;;N;;;;; +13FBF;EGYPTIAN HIEROGLYPH-13FBF;Lo;0;L;;;;;N;;;;; +13FC0;EGYPTIAN HIEROGLYPH-13FC0;Lo;0;L;;;;;N;;;;; +13FC1;EGYPTIAN HIEROGLYPH-13FC1;Lo;0;L;;;;;N;;;;; +13FC2;EGYPTIAN HIEROGLYPH-13FC2;Lo;0;L;;;;;N;;;;; +13FC3;EGYPTIAN HIEROGLYPH-13FC3;Lo;0;L;;;;;N;;;;; +13FC4;EGYPTIAN HIEROGLYPH-13FC4;Lo;0;L;;;;;N;;;;; +13FC5;EGYPTIAN HIEROGLYPH-13FC5;Lo;0;L;;;;;N;;;;; +13FC6;EGYPTIAN HIEROGLYPH-13FC6;Lo;0;L;;;;;N;;;;; +13FC7;EGYPTIAN HIEROGLYPH-13FC7;Lo;0;L;;;;;N;;;;; +13FC8;EGYPTIAN HIEROGLYPH-13FC8;Lo;0;L;;;;;N;;;;; +13FC9;EGYPTIAN HIEROGLYPH-13FC9;Lo;0;L;;;;;N;;;;; +13FCA;EGYPTIAN HIEROGLYPH-13FCA;Lo;0;L;;;;;N;;;;; +13FCB;EGYPTIAN HIEROGLYPH-13FCB;Lo;0;L;;;;;N;;;;; +13FCC;EGYPTIAN HIEROGLYPH-13FCC;Lo;0;L;;;;;N;;;;; +13FCD;EGYPTIAN HIEROGLYPH-13FCD;Lo;0;L;;;;;N;;;;; +13FCE;EGYPTIAN HIEROGLYPH-13FCE;Lo;0;L;;;;;N;;;;; +13FCF;EGYPTIAN HIEROGLYPH-13FCF;Lo;0;L;;;;;N;;;;; +13FD0;EGYPTIAN HIEROGLYPH-13FD0;Lo;0;L;;;;;N;;;;; +13FD1;EGYPTIAN HIEROGLYPH-13FD1;Lo;0;L;;;;;N;;;;; +13FD2;EGYPTIAN HIEROGLYPH-13FD2;Lo;0;L;;;;;N;;;;; +13FD3;EGYPTIAN HIEROGLYPH-13FD3;Lo;0;L;;;;;N;;;;; +13FD4;EGYPTIAN HIEROGLYPH-13FD4;Lo;0;L;;;;;N;;;;; +13FD5;EGYPTIAN HIEROGLYPH-13FD5;Lo;0;L;;;;;N;;;;; +13FD6;EGYPTIAN HIEROGLYPH-13FD6;Lo;0;L;;;;;N;;;;; +13FD7;EGYPTIAN HIEROGLYPH-13FD7;Lo;0;L;;;;;N;;;;; +13FD8;EGYPTIAN HIEROGLYPH-13FD8;Lo;0;L;;;;;N;;;;; +13FD9;EGYPTIAN HIEROGLYPH-13FD9;Lo;0;L;;;;;N;;;;; +13FDA;EGYPTIAN HIEROGLYPH-13FDA;Lo;0;L;;;;;N;;;;; +13FDB;EGYPTIAN HIEROGLYPH-13FDB;Lo;0;L;;;;;N;;;;; +13FDC;EGYPTIAN HIEROGLYPH-13FDC;Lo;0;L;;;;;N;;;;; +13FDD;EGYPTIAN HIEROGLYPH-13FDD;Lo;0;L;;;;;N;;;;; +13FDE;EGYPTIAN HIEROGLYPH-13FDE;Lo;0;L;;;;;N;;;;; +13FDF;EGYPTIAN HIEROGLYPH-13FDF;Lo;0;L;;;;;N;;;;; +13FE0;EGYPTIAN HIEROGLYPH-13FE0;Lo;0;L;;;;;N;;;;; +13FE1;EGYPTIAN HIEROGLYPH-13FE1;Lo;0;L;;;;;N;;;;; +13FE2;EGYPTIAN HIEROGLYPH-13FE2;Lo;0;L;;;;;N;;;;; +13FE3;EGYPTIAN HIEROGLYPH-13FE3;Lo;0;L;;;;;N;;;;; +13FE4;EGYPTIAN HIEROGLYPH-13FE4;Lo;0;L;;;;;N;;;;; +13FE5;EGYPTIAN HIEROGLYPH-13FE5;Lo;0;L;;;;;N;;;;; +13FE6;EGYPTIAN HIEROGLYPH-13FE6;Lo;0;L;;;;;N;;;;; +13FE7;EGYPTIAN HIEROGLYPH-13FE7;Lo;0;L;;;;;N;;;;; +13FE8;EGYPTIAN HIEROGLYPH-13FE8;Lo;0;L;;;;;N;;;;; +13FE9;EGYPTIAN HIEROGLYPH-13FE9;Lo;0;L;;;;;N;;;;; +13FEA;EGYPTIAN HIEROGLYPH-13FEA;Lo;0;L;;;;;N;;;;; +13FEB;EGYPTIAN HIEROGLYPH-13FEB;Lo;0;L;;;;;N;;;;; +13FEC;EGYPTIAN HIEROGLYPH-13FEC;Lo;0;L;;;;;N;;;;; +13FED;EGYPTIAN HIEROGLYPH-13FED;Lo;0;L;;;;;N;;;;; +13FEE;EGYPTIAN HIEROGLYPH-13FEE;Lo;0;L;;;;;N;;;;; +13FEF;EGYPTIAN HIEROGLYPH-13FEF;Lo;0;L;;;;;N;;;;; +13FF0;EGYPTIAN HIEROGLYPH-13FF0;Lo;0;L;;;;;N;;;;; +13FF1;EGYPTIAN HIEROGLYPH-13FF1;Lo;0;L;;;;;N;;;;; +13FF2;EGYPTIAN HIEROGLYPH-13FF2;Lo;0;L;;;;;N;;;;; +13FF3;EGYPTIAN HIEROGLYPH-13FF3;Lo;0;L;;;;;N;;;;; +13FF4;EGYPTIAN HIEROGLYPH-13FF4;Lo;0;L;;;;;N;;;;; +13FF5;EGYPTIAN HIEROGLYPH-13FF5;Lo;0;L;;;;;N;;;;; +13FF6;EGYPTIAN HIEROGLYPH-13FF6;Lo;0;L;;;;;N;;;;; +13FF7;EGYPTIAN HIEROGLYPH-13FF7;Lo;0;L;;;;;N;;;;; +13FF8;EGYPTIAN HIEROGLYPH-13FF8;Lo;0;L;;;;;N;;;;; +13FF9;EGYPTIAN HIEROGLYPH-13FF9;Lo;0;L;;;;;N;;;;; +13FFA;EGYPTIAN HIEROGLYPH-13FFA;Lo;0;L;;;;;N;;;;; +13FFB;EGYPTIAN HIEROGLYPH-13FFB;Lo;0;L;;;;;N;;;;; +13FFC;EGYPTIAN HIEROGLYPH-13FFC;Lo;0;L;;;;;N;;;;; +13FFD;EGYPTIAN HIEROGLYPH-13FFD;Lo;0;L;;;;;N;;;;; +13FFE;EGYPTIAN HIEROGLYPH-13FFE;Lo;0;L;;;;;N;;;;; +13FFF;EGYPTIAN HIEROGLYPH-13FFF;Lo;0;L;;;;;N;;;;; +14000;EGYPTIAN HIEROGLYPH-14000;Lo;0;L;;;;;N;;;;; +14001;EGYPTIAN HIEROGLYPH-14001;Lo;0;L;;;;;N;;;;; +14002;EGYPTIAN HIEROGLYPH-14002;Lo;0;L;;;;;N;;;;; +14003;EGYPTIAN HIEROGLYPH-14003;Lo;0;L;;;;;N;;;;; +14004;EGYPTIAN HIEROGLYPH-14004;Lo;0;L;;;;;N;;;;; +14005;EGYPTIAN HIEROGLYPH-14005;Lo;0;L;;;;;N;;;;; +14006;EGYPTIAN HIEROGLYPH-14006;Lo;0;L;;;;;N;;;;; +14007;EGYPTIAN HIEROGLYPH-14007;Lo;0;L;;;;;N;;;;; +14008;EGYPTIAN HIEROGLYPH-14008;Lo;0;L;;;;;N;;;;; +14009;EGYPTIAN HIEROGLYPH-14009;Lo;0;L;;;;;N;;;;; +1400A;EGYPTIAN HIEROGLYPH-1400A;Lo;0;L;;;;;N;;;;; +1400B;EGYPTIAN HIEROGLYPH-1400B;Lo;0;L;;;;;N;;;;; +1400C;EGYPTIAN HIEROGLYPH-1400C;Lo;0;L;;;;;N;;;;; +1400D;EGYPTIAN HIEROGLYPH-1400D;Lo;0;L;;;;;N;;;;; +1400E;EGYPTIAN HIEROGLYPH-1400E;Lo;0;L;;;;;N;;;;; +1400F;EGYPTIAN HIEROGLYPH-1400F;Lo;0;L;;;;;N;;;;; +14010;EGYPTIAN HIEROGLYPH-14010;Lo;0;L;;;;;N;;;;; +14011;EGYPTIAN HIEROGLYPH-14011;Lo;0;L;;;;;N;;;;; +14012;EGYPTIAN HIEROGLYPH-14012;Lo;0;L;;;;;N;;;;; +14013;EGYPTIAN HIEROGLYPH-14013;Lo;0;L;;;;;N;;;;; +14014;EGYPTIAN HIEROGLYPH-14014;Lo;0;L;;;;;N;;;;; +14015;EGYPTIAN HIEROGLYPH-14015;Lo;0;L;;;;;N;;;;; +14016;EGYPTIAN HIEROGLYPH-14016;Lo;0;L;;;;;N;;;;; +14017;EGYPTIAN HIEROGLYPH-14017;Lo;0;L;;;;;N;;;;; +14018;EGYPTIAN HIEROGLYPH-14018;Lo;0;L;;;;;N;;;;; +14019;EGYPTIAN HIEROGLYPH-14019;Lo;0;L;;;;;N;;;;; +1401A;EGYPTIAN HIEROGLYPH-1401A;Lo;0;L;;;;;N;;;;; +1401B;EGYPTIAN HIEROGLYPH-1401B;Lo;0;L;;;;;N;;;;; +1401C;EGYPTIAN HIEROGLYPH-1401C;Lo;0;L;;;;;N;;;;; +1401D;EGYPTIAN HIEROGLYPH-1401D;Lo;0;L;;;;;N;;;;; +1401E;EGYPTIAN HIEROGLYPH-1401E;Lo;0;L;;;;;N;;;;; +1401F;EGYPTIAN HIEROGLYPH-1401F;Lo;0;L;;;;;N;;;;; +14020;EGYPTIAN HIEROGLYPH-14020;Lo;0;L;;;;;N;;;;; +14021;EGYPTIAN HIEROGLYPH-14021;Lo;0;L;;;;;N;;;;; +14022;EGYPTIAN HIEROGLYPH-14022;Lo;0;L;;;;;N;;;;; +14023;EGYPTIAN HIEROGLYPH-14023;Lo;0;L;;;;;N;;;;; +14024;EGYPTIAN HIEROGLYPH-14024;Lo;0;L;;;;;N;;;;; +14025;EGYPTIAN HIEROGLYPH-14025;Lo;0;L;;;;;N;;;;; +14026;EGYPTIAN HIEROGLYPH-14026;Lo;0;L;;;;;N;;;;; +14027;EGYPTIAN HIEROGLYPH-14027;Lo;0;L;;;;;N;;;;; +14028;EGYPTIAN HIEROGLYPH-14028;Lo;0;L;;;;;N;;;;; +14029;EGYPTIAN HIEROGLYPH-14029;Lo;0;L;;;;;N;;;;; +1402A;EGYPTIAN HIEROGLYPH-1402A;Lo;0;L;;;;;N;;;;; +1402B;EGYPTIAN HIEROGLYPH-1402B;Lo;0;L;;;;;N;;;;; +1402C;EGYPTIAN HIEROGLYPH-1402C;Lo;0;L;;;;;N;;;;; +1402D;EGYPTIAN HIEROGLYPH-1402D;Lo;0;L;;;;;N;;;;; +1402E;EGYPTIAN HIEROGLYPH-1402E;Lo;0;L;;;;;N;;;;; +1402F;EGYPTIAN HIEROGLYPH-1402F;Lo;0;L;;;;;N;;;;; +14030;EGYPTIAN HIEROGLYPH-14030;Lo;0;L;;;;;N;;;;; +14031;EGYPTIAN HIEROGLYPH-14031;Lo;0;L;;;;;N;;;;; +14032;EGYPTIAN HIEROGLYPH-14032;Lo;0;L;;;;;N;;;;; +14033;EGYPTIAN HIEROGLYPH-14033;Lo;0;L;;;;;N;;;;; +14034;EGYPTIAN HIEROGLYPH-14034;Lo;0;L;;;;;N;;;;; +14035;EGYPTIAN HIEROGLYPH-14035;Lo;0;L;;;;;N;;;;; +14036;EGYPTIAN HIEROGLYPH-14036;Lo;0;L;;;;;N;;;;; +14037;EGYPTIAN HIEROGLYPH-14037;Lo;0;L;;;;;N;;;;; +14038;EGYPTIAN HIEROGLYPH-14038;Lo;0;L;;;;;N;;;;; +14039;EGYPTIAN HIEROGLYPH-14039;Lo;0;L;;;;;N;;;;; +1403A;EGYPTIAN HIEROGLYPH-1403A;Lo;0;L;;;;;N;;;;; +1403B;EGYPTIAN HIEROGLYPH-1403B;Lo;0;L;;;;;N;;;;; +1403C;EGYPTIAN HIEROGLYPH-1403C;Lo;0;L;;;;;N;;;;; +1403D;EGYPTIAN HIEROGLYPH-1403D;Lo;0;L;;;;;N;;;;; +1403E;EGYPTIAN HIEROGLYPH-1403E;Lo;0;L;;;;;N;;;;; +1403F;EGYPTIAN HIEROGLYPH-1403F;Lo;0;L;;;;;N;;;;; +14040;EGYPTIAN HIEROGLYPH-14040;Lo;0;L;;;;;N;;;;; +14041;EGYPTIAN HIEROGLYPH-14041;Lo;0;L;;;;;N;;;;; +14042;EGYPTIAN HIEROGLYPH-14042;Lo;0;L;;;;;N;;;;; +14043;EGYPTIAN HIEROGLYPH-14043;Lo;0;L;;;;;N;;;;; +14044;EGYPTIAN HIEROGLYPH-14044;Lo;0;L;;;;;N;;;;; +14045;EGYPTIAN HIEROGLYPH-14045;Lo;0;L;;;;;N;;;;; +14046;EGYPTIAN HIEROGLYPH-14046;Lo;0;L;;;;;N;;;;; +14047;EGYPTIAN HIEROGLYPH-14047;Lo;0;L;;;;;N;;;;; +14048;EGYPTIAN HIEROGLYPH-14048;Lo;0;L;;;;;N;;;;; +14049;EGYPTIAN HIEROGLYPH-14049;Lo;0;L;;;;;N;;;;; +1404A;EGYPTIAN HIEROGLYPH-1404A;Lo;0;L;;;;;N;;;;; +1404B;EGYPTIAN HIEROGLYPH-1404B;Lo;0;L;;;;;N;;;;; +1404C;EGYPTIAN HIEROGLYPH-1404C;Lo;0;L;;;;;N;;;;; +1404D;EGYPTIAN HIEROGLYPH-1404D;Lo;0;L;;;;;N;;;;; +1404E;EGYPTIAN HIEROGLYPH-1404E;Lo;0;L;;;;;N;;;;; +1404F;EGYPTIAN HIEROGLYPH-1404F;Lo;0;L;;;;;N;;;;; +14050;EGYPTIAN HIEROGLYPH-14050;Lo;0;L;;;;;N;;;;; +14051;EGYPTIAN HIEROGLYPH-14051;Lo;0;L;;;;;N;;;;; +14052;EGYPTIAN HIEROGLYPH-14052;Lo;0;L;;;;;N;;;;; +14053;EGYPTIAN HIEROGLYPH-14053;Lo;0;L;;;;;N;;;;; +14054;EGYPTIAN HIEROGLYPH-14054;Lo;0;L;;;;;N;;;;; +14055;EGYPTIAN HIEROGLYPH-14055;Lo;0;L;;;;;N;;;;; +14056;EGYPTIAN HIEROGLYPH-14056;Lo;0;L;;;;;N;;;;; +14057;EGYPTIAN HIEROGLYPH-14057;Lo;0;L;;;;;N;;;;; +14058;EGYPTIAN HIEROGLYPH-14058;Lo;0;L;;;;;N;;;;; +14059;EGYPTIAN HIEROGLYPH-14059;Lo;0;L;;;;;N;;;;; +1405A;EGYPTIAN HIEROGLYPH-1405A;Lo;0;L;;;;;N;;;;; +1405B;EGYPTIAN HIEROGLYPH-1405B;Lo;0;L;;;;;N;;;;; +1405C;EGYPTIAN HIEROGLYPH-1405C;Lo;0;L;;;;;N;;;;; +1405D;EGYPTIAN HIEROGLYPH-1405D;Lo;0;L;;;;;N;;;;; +1405E;EGYPTIAN HIEROGLYPH-1405E;Lo;0;L;;;;;N;;;;; +1405F;EGYPTIAN HIEROGLYPH-1405F;Lo;0;L;;;;;N;;;;; +14060;EGYPTIAN HIEROGLYPH-14060;Lo;0;L;;;;;N;;;;; +14061;EGYPTIAN HIEROGLYPH-14061;Lo;0;L;;;;;N;;;;; +14062;EGYPTIAN HIEROGLYPH-14062;Lo;0;L;;;;;N;;;;; +14063;EGYPTIAN HIEROGLYPH-14063;Lo;0;L;;;;;N;;;;; +14064;EGYPTIAN HIEROGLYPH-14064;Lo;0;L;;;;;N;;;;; +14065;EGYPTIAN HIEROGLYPH-14065;Lo;0;L;;;;;N;;;;; +14066;EGYPTIAN HIEROGLYPH-14066;Lo;0;L;;;;;N;;;;; +14067;EGYPTIAN HIEROGLYPH-14067;Lo;0;L;;;;;N;;;;; +14068;EGYPTIAN HIEROGLYPH-14068;Lo;0;L;;;;;N;;;;; +14069;EGYPTIAN HIEROGLYPH-14069;Lo;0;L;;;;;N;;;;; +1406A;EGYPTIAN HIEROGLYPH-1406A;Lo;0;L;;;;;N;;;;; +1406B;EGYPTIAN HIEROGLYPH-1406B;Lo;0;L;;;;;N;;;;; +1406C;EGYPTIAN HIEROGLYPH-1406C;Lo;0;L;;;;;N;;;;; +1406D;EGYPTIAN HIEROGLYPH-1406D;Lo;0;L;;;;;N;;;;; +1406E;EGYPTIAN HIEROGLYPH-1406E;Lo;0;L;;;;;N;;;;; +1406F;EGYPTIAN HIEROGLYPH-1406F;Lo;0;L;;;;;N;;;;; +14070;EGYPTIAN HIEROGLYPH-14070;Lo;0;L;;;;;N;;;;; +14071;EGYPTIAN HIEROGLYPH-14071;Lo;0;L;;;;;N;;;;; +14072;EGYPTIAN HIEROGLYPH-14072;Lo;0;L;;;;;N;;;;; +14073;EGYPTIAN HIEROGLYPH-14073;Lo;0;L;;;;;N;;;;; +14074;EGYPTIAN HIEROGLYPH-14074;Lo;0;L;;;;;N;;;;; +14075;EGYPTIAN HIEROGLYPH-14075;Lo;0;L;;;;;N;;;;; +14076;EGYPTIAN HIEROGLYPH-14076;Lo;0;L;;;;;N;;;;; +14077;EGYPTIAN HIEROGLYPH-14077;Lo;0;L;;;;;N;;;;; +14078;EGYPTIAN HIEROGLYPH-14078;Lo;0;L;;;;;N;;;;; +14079;EGYPTIAN HIEROGLYPH-14079;Lo;0;L;;;;;N;;;;; +1407A;EGYPTIAN HIEROGLYPH-1407A;Lo;0;L;;;;;N;;;;; +1407B;EGYPTIAN HIEROGLYPH-1407B;Lo;0;L;;;;;N;;;;; +1407C;EGYPTIAN HIEROGLYPH-1407C;Lo;0;L;;;;;N;;;;; +1407D;EGYPTIAN HIEROGLYPH-1407D;Lo;0;L;;;;;N;;;;; +1407E;EGYPTIAN HIEROGLYPH-1407E;Lo;0;L;;;;;N;;;;; +1407F;EGYPTIAN HIEROGLYPH-1407F;Lo;0;L;;;;;N;;;;; +14080;EGYPTIAN HIEROGLYPH-14080;Lo;0;L;;;;;N;;;;; +14081;EGYPTIAN HIEROGLYPH-14081;Lo;0;L;;;;;N;;;;; +14082;EGYPTIAN HIEROGLYPH-14082;Lo;0;L;;;;;N;;;;; +14083;EGYPTIAN HIEROGLYPH-14083;Lo;0;L;;;;;N;;;;; +14084;EGYPTIAN HIEROGLYPH-14084;Lo;0;L;;;;;N;;;;; +14085;EGYPTIAN HIEROGLYPH-14085;Lo;0;L;;;;;N;;;;; +14086;EGYPTIAN HIEROGLYPH-14086;Lo;0;L;;;;;N;;;;; +14087;EGYPTIAN HIEROGLYPH-14087;Lo;0;L;;;;;N;;;;; +14088;EGYPTIAN HIEROGLYPH-14088;Lo;0;L;;;;;N;;;;; +14089;EGYPTIAN HIEROGLYPH-14089;Lo;0;L;;;;;N;;;;; +1408A;EGYPTIAN HIEROGLYPH-1408A;Lo;0;L;;;;;N;;;;; +1408B;EGYPTIAN HIEROGLYPH-1408B;Lo;0;L;;;;;N;;;;; +1408C;EGYPTIAN HIEROGLYPH-1408C;Lo;0;L;;;;;N;;;;; +1408D;EGYPTIAN HIEROGLYPH-1408D;Lo;0;L;;;;;N;;;;; +1408E;EGYPTIAN HIEROGLYPH-1408E;Lo;0;L;;;;;N;;;;; +1408F;EGYPTIAN HIEROGLYPH-1408F;Lo;0;L;;;;;N;;;;; +14090;EGYPTIAN HIEROGLYPH-14090;Lo;0;L;;;;;N;;;;; +14091;EGYPTIAN HIEROGLYPH-14091;Lo;0;L;;;;;N;;;;; +14092;EGYPTIAN HIEROGLYPH-14092;Lo;0;L;;;;;N;;;;; +14093;EGYPTIAN HIEROGLYPH-14093;Lo;0;L;;;;;N;;;;; +14094;EGYPTIAN HIEROGLYPH-14094;Lo;0;L;;;;;N;;;;; +14095;EGYPTIAN HIEROGLYPH-14095;Lo;0;L;;;;;N;;;;; +14096;EGYPTIAN HIEROGLYPH-14096;Lo;0;L;;;;;N;;;;; +14097;EGYPTIAN HIEROGLYPH-14097;Lo;0;L;;;;;N;;;;; +14098;EGYPTIAN HIEROGLYPH-14098;Lo;0;L;;;;;N;;;;; +14099;EGYPTIAN HIEROGLYPH-14099;Lo;0;L;;;;;N;;;;; +1409A;EGYPTIAN HIEROGLYPH-1409A;Lo;0;L;;;;;N;;;;; +1409B;EGYPTIAN HIEROGLYPH-1409B;Lo;0;L;;;;;N;;;;; +1409C;EGYPTIAN HIEROGLYPH-1409C;Lo;0;L;;;;;N;;;;; +1409D;EGYPTIAN HIEROGLYPH-1409D;Lo;0;L;;;;;N;;;;; +1409E;EGYPTIAN HIEROGLYPH-1409E;Lo;0;L;;;;;N;;;;; +1409F;EGYPTIAN HIEROGLYPH-1409F;Lo;0;L;;;;;N;;;;; +140A0;EGYPTIAN HIEROGLYPH-140A0;Lo;0;L;;;;;N;;;;; +140A1;EGYPTIAN HIEROGLYPH-140A1;Lo;0;L;;;;;N;;;;; +140A2;EGYPTIAN HIEROGLYPH-140A2;Lo;0;L;;;;;N;;;;; +140A3;EGYPTIAN HIEROGLYPH-140A3;Lo;0;L;;;;;N;;;;; +140A4;EGYPTIAN HIEROGLYPH-140A4;Lo;0;L;;;;;N;;;;; +140A5;EGYPTIAN HIEROGLYPH-140A5;Lo;0;L;;;;;N;;;;; +140A6;EGYPTIAN HIEROGLYPH-140A6;Lo;0;L;;;;;N;;;;; +140A7;EGYPTIAN HIEROGLYPH-140A7;Lo;0;L;;;;;N;;;;; +140A8;EGYPTIAN HIEROGLYPH-140A8;Lo;0;L;;;;;N;;;;; +140A9;EGYPTIAN HIEROGLYPH-140A9;Lo;0;L;;;;;N;;;;; +140AA;EGYPTIAN HIEROGLYPH-140AA;Lo;0;L;;;;;N;;;;; +140AB;EGYPTIAN HIEROGLYPH-140AB;Lo;0;L;;;;;N;;;;; +140AC;EGYPTIAN HIEROGLYPH-140AC;Lo;0;L;;;;;N;;;;; +140AD;EGYPTIAN HIEROGLYPH-140AD;Lo;0;L;;;;;N;;;;; +140AE;EGYPTIAN HIEROGLYPH-140AE;Lo;0;L;;;;;N;;;;; +140AF;EGYPTIAN HIEROGLYPH-140AF;Lo;0;L;;;;;N;;;;; +140B0;EGYPTIAN HIEROGLYPH-140B0;Lo;0;L;;;;;N;;;;; +140B1;EGYPTIAN HIEROGLYPH-140B1;Lo;0;L;;;;;N;;;;; +140B2;EGYPTIAN HIEROGLYPH-140B2;Lo;0;L;;;;;N;;;;; +140B3;EGYPTIAN HIEROGLYPH-140B3;Lo;0;L;;;;;N;;;;; +140B4;EGYPTIAN HIEROGLYPH-140B4;Lo;0;L;;;;;N;;;;; +140B5;EGYPTIAN HIEROGLYPH-140B5;Lo;0;L;;;;;N;;;;; +140B6;EGYPTIAN HIEROGLYPH-140B6;Lo;0;L;;;;;N;;;;; +140B7;EGYPTIAN HIEROGLYPH-140B7;Lo;0;L;;;;;N;;;;; +140B8;EGYPTIAN HIEROGLYPH-140B8;Lo;0;L;;;;;N;;;;; +140B9;EGYPTIAN HIEROGLYPH-140B9;Lo;0;L;;;;;N;;;;; +140BA;EGYPTIAN HIEROGLYPH-140BA;Lo;0;L;;;;;N;;;;; +140BB;EGYPTIAN HIEROGLYPH-140BB;Lo;0;L;;;;;N;;;;; +140BC;EGYPTIAN HIEROGLYPH-140BC;Lo;0;L;;;;;N;;;;; +140BD;EGYPTIAN HIEROGLYPH-140BD;Lo;0;L;;;;;N;;;;; +140BE;EGYPTIAN HIEROGLYPH-140BE;Lo;0;L;;;;;N;;;;; +140BF;EGYPTIAN HIEROGLYPH-140BF;Lo;0;L;;;;;N;;;;; +140C0;EGYPTIAN HIEROGLYPH-140C0;Lo;0;L;;;;;N;;;;; +140C1;EGYPTIAN HIEROGLYPH-140C1;Lo;0;L;;;;;N;;;;; +140C2;EGYPTIAN HIEROGLYPH-140C2;Lo;0;L;;;;;N;;;;; +140C3;EGYPTIAN HIEROGLYPH-140C3;Lo;0;L;;;;;N;;;;; +140C4;EGYPTIAN HIEROGLYPH-140C4;Lo;0;L;;;;;N;;;;; +140C5;EGYPTIAN HIEROGLYPH-140C5;Lo;0;L;;;;;N;;;;; +140C6;EGYPTIAN HIEROGLYPH-140C6;Lo;0;L;;;;;N;;;;; +140C7;EGYPTIAN HIEROGLYPH-140C7;Lo;0;L;;;;;N;;;;; +140C8;EGYPTIAN HIEROGLYPH-140C8;Lo;0;L;;;;;N;;;;; +140C9;EGYPTIAN HIEROGLYPH-140C9;Lo;0;L;;;;;N;;;;; +140CA;EGYPTIAN HIEROGLYPH-140CA;Lo;0;L;;;;;N;;;;; +140CB;EGYPTIAN HIEROGLYPH-140CB;Lo;0;L;;;;;N;;;;; +140CC;EGYPTIAN HIEROGLYPH-140CC;Lo;0;L;;;;;N;;;;; +140CD;EGYPTIAN HIEROGLYPH-140CD;Lo;0;L;;;;;N;;;;; +140CE;EGYPTIAN HIEROGLYPH-140CE;Lo;0;L;;;;;N;;;;; +140CF;EGYPTIAN HIEROGLYPH-140CF;Lo;0;L;;;;;N;;;;; +140D0;EGYPTIAN HIEROGLYPH-140D0;Lo;0;L;;;;;N;;;;; +140D1;EGYPTIAN HIEROGLYPH-140D1;Lo;0;L;;;;;N;;;;; +140D2;EGYPTIAN HIEROGLYPH-140D2;Lo;0;L;;;;;N;;;;; +140D3;EGYPTIAN HIEROGLYPH-140D3;Lo;0;L;;;;;N;;;;; +140D4;EGYPTIAN HIEROGLYPH-140D4;Lo;0;L;;;;;N;;;;; +140D5;EGYPTIAN HIEROGLYPH-140D5;Lo;0;L;;;;;N;;;;; +140D6;EGYPTIAN HIEROGLYPH-140D6;Lo;0;L;;;;;N;;;;; +140D7;EGYPTIAN HIEROGLYPH-140D7;Lo;0;L;;;;;N;;;;; +140D8;EGYPTIAN HIEROGLYPH-140D8;Lo;0;L;;;;;N;;;;; +140D9;EGYPTIAN HIEROGLYPH-140D9;Lo;0;L;;;;;N;;;;; +140DA;EGYPTIAN HIEROGLYPH-140DA;Lo;0;L;;;;;N;;;;; +140DB;EGYPTIAN HIEROGLYPH-140DB;Lo;0;L;;;;;N;;;;; +140DC;EGYPTIAN HIEROGLYPH-140DC;Lo;0;L;;;;;N;;;;; +140DD;EGYPTIAN HIEROGLYPH-140DD;Lo;0;L;;;;;N;;;;; +140DE;EGYPTIAN HIEROGLYPH-140DE;Lo;0;L;;;;;N;;;;; +140DF;EGYPTIAN HIEROGLYPH-140DF;Lo;0;L;;;;;N;;;;; +140E0;EGYPTIAN HIEROGLYPH-140E0;Lo;0;L;;;;;N;;;;; +140E1;EGYPTIAN HIEROGLYPH-140E1;Lo;0;L;;;;;N;;;;; +140E2;EGYPTIAN HIEROGLYPH-140E2;Lo;0;L;;;;;N;;;;; +140E3;EGYPTIAN HIEROGLYPH-140E3;Lo;0;L;;;;;N;;;;; +140E4;EGYPTIAN HIEROGLYPH-140E4;Lo;0;L;;;;;N;;;;; +140E5;EGYPTIAN HIEROGLYPH-140E5;Lo;0;L;;;;;N;;;;; +140E6;EGYPTIAN HIEROGLYPH-140E6;Lo;0;L;;;;;N;;;;; +140E7;EGYPTIAN HIEROGLYPH-140E7;Lo;0;L;;;;;N;;;;; +140E8;EGYPTIAN HIEROGLYPH-140E8;Lo;0;L;;;;;N;;;;; +140E9;EGYPTIAN HIEROGLYPH-140E9;Lo;0;L;;;;;N;;;;; +140EA;EGYPTIAN HIEROGLYPH-140EA;Lo;0;L;;;;;N;;;;; +140EB;EGYPTIAN HIEROGLYPH-140EB;Lo;0;L;;;;;N;;;;; +140EC;EGYPTIAN HIEROGLYPH-140EC;Lo;0;L;;;;;N;;;;; +140ED;EGYPTIAN HIEROGLYPH-140ED;Lo;0;L;;;;;N;;;;; +140EE;EGYPTIAN HIEROGLYPH-140EE;Lo;0;L;;;;;N;;;;; +140EF;EGYPTIAN HIEROGLYPH-140EF;Lo;0;L;;;;;N;;;;; +140F0;EGYPTIAN HIEROGLYPH-140F0;Lo;0;L;;;;;N;;;;; +140F1;EGYPTIAN HIEROGLYPH-140F1;Lo;0;L;;;;;N;;;;; +140F2;EGYPTIAN HIEROGLYPH-140F2;Lo;0;L;;;;;N;;;;; +140F3;EGYPTIAN HIEROGLYPH-140F3;Lo;0;L;;;;;N;;;;; +140F4;EGYPTIAN HIEROGLYPH-140F4;Lo;0;L;;;;;N;;;;; +140F5;EGYPTIAN HIEROGLYPH-140F5;Lo;0;L;;;;;N;;;;; +140F6;EGYPTIAN HIEROGLYPH-140F6;Lo;0;L;;;;;N;;;;; +140F7;EGYPTIAN HIEROGLYPH-140F7;Lo;0;L;;;;;N;;;;; +140F8;EGYPTIAN HIEROGLYPH-140F8;Lo;0;L;;;;;N;;;;; +140F9;EGYPTIAN HIEROGLYPH-140F9;Lo;0;L;;;;;N;;;;; +140FA;EGYPTIAN HIEROGLYPH-140FA;Lo;0;L;;;;;N;;;;; +140FB;EGYPTIAN HIEROGLYPH-140FB;Lo;0;L;;;;;N;;;;; +140FC;EGYPTIAN HIEROGLYPH-140FC;Lo;0;L;;;;;N;;;;; +140FD;EGYPTIAN HIEROGLYPH-140FD;Lo;0;L;;;;;N;;;;; +140FE;EGYPTIAN HIEROGLYPH-140FE;Lo;0;L;;;;;N;;;;; +140FF;EGYPTIAN HIEROGLYPH-140FF;Lo;0;L;;;;;N;;;;; +14100;EGYPTIAN HIEROGLYPH-14100;Lo;0;L;;;;;N;;;;; +14101;EGYPTIAN HIEROGLYPH-14101;Lo;0;L;;;;;N;;;;; +14102;EGYPTIAN HIEROGLYPH-14102;Lo;0;L;;;;;N;;;;; +14103;EGYPTIAN HIEROGLYPH-14103;Lo;0;L;;;;;N;;;;; +14104;EGYPTIAN HIEROGLYPH-14104;Lo;0;L;;;;;N;;;;; +14105;EGYPTIAN HIEROGLYPH-14105;Lo;0;L;;;;;N;;;;; +14106;EGYPTIAN HIEROGLYPH-14106;Lo;0;L;;;;;N;;;;; +14107;EGYPTIAN HIEROGLYPH-14107;Lo;0;L;;;;;N;;;;; +14108;EGYPTIAN HIEROGLYPH-14108;Lo;0;L;;;;;N;;;;; +14109;EGYPTIAN HIEROGLYPH-14109;Lo;0;L;;;;;N;;;;; +1410A;EGYPTIAN HIEROGLYPH-1410A;Lo;0;L;;;;;N;;;;; +1410B;EGYPTIAN HIEROGLYPH-1410B;Lo;0;L;;;;;N;;;;; +1410C;EGYPTIAN HIEROGLYPH-1410C;Lo;0;L;;;;;N;;;;; +1410D;EGYPTIAN HIEROGLYPH-1410D;Lo;0;L;;;;;N;;;;; +1410E;EGYPTIAN HIEROGLYPH-1410E;Lo;0;L;;;;;N;;;;; +1410F;EGYPTIAN HIEROGLYPH-1410F;Lo;0;L;;;;;N;;;;; +14110;EGYPTIAN HIEROGLYPH-14110;Lo;0;L;;;;;N;;;;; +14111;EGYPTIAN HIEROGLYPH-14111;Lo;0;L;;;;;N;;;;; +14112;EGYPTIAN HIEROGLYPH-14112;Lo;0;L;;;;;N;;;;; +14113;EGYPTIAN HIEROGLYPH-14113;Lo;0;L;;;;;N;;;;; +14114;EGYPTIAN HIEROGLYPH-14114;Lo;0;L;;;;;N;;;;; +14115;EGYPTIAN HIEROGLYPH-14115;Lo;0;L;;;;;N;;;;; +14116;EGYPTIAN HIEROGLYPH-14116;Lo;0;L;;;;;N;;;;; +14117;EGYPTIAN HIEROGLYPH-14117;Lo;0;L;;;;;N;;;;; +14118;EGYPTIAN HIEROGLYPH-14118;Lo;0;L;;;;;N;;;;; +14119;EGYPTIAN HIEROGLYPH-14119;Lo;0;L;;;;;N;;;;; +1411A;EGYPTIAN HIEROGLYPH-1411A;Lo;0;L;;;;;N;;;;; +1411B;EGYPTIAN HIEROGLYPH-1411B;Lo;0;L;;;;;N;;;;; +1411C;EGYPTIAN HIEROGLYPH-1411C;Lo;0;L;;;;;N;;;;; +1411D;EGYPTIAN HIEROGLYPH-1411D;Lo;0;L;;;;;N;;;;; +1411E;EGYPTIAN HIEROGLYPH-1411E;Lo;0;L;;;;;N;;;;; +1411F;EGYPTIAN HIEROGLYPH-1411F;Lo;0;L;;;;;N;;;;; +14120;EGYPTIAN HIEROGLYPH-14120;Lo;0;L;;;;;N;;;;; +14121;EGYPTIAN HIEROGLYPH-14121;Lo;0;L;;;;;N;;;;; +14122;EGYPTIAN HIEROGLYPH-14122;Lo;0;L;;;;;N;;;;; +14123;EGYPTIAN HIEROGLYPH-14123;Lo;0;L;;;;;N;;;;; +14124;EGYPTIAN HIEROGLYPH-14124;Lo;0;L;;;;;N;;;;; +14125;EGYPTIAN HIEROGLYPH-14125;Lo;0;L;;;;;N;;;;; +14126;EGYPTIAN HIEROGLYPH-14126;Lo;0;L;;;;;N;;;;; +14127;EGYPTIAN HIEROGLYPH-14127;Lo;0;L;;;;;N;;;;; +14128;EGYPTIAN HIEROGLYPH-14128;Lo;0;L;;;;;N;;;;; +14129;EGYPTIAN HIEROGLYPH-14129;Lo;0;L;;;;;N;;;;; +1412A;EGYPTIAN HIEROGLYPH-1412A;Lo;0;L;;;;;N;;;;; +1412B;EGYPTIAN HIEROGLYPH-1412B;Lo;0;L;;;;;N;;;;; +1412C;EGYPTIAN HIEROGLYPH-1412C;Lo;0;L;;;;;N;;;;; +1412D;EGYPTIAN HIEROGLYPH-1412D;Lo;0;L;;;;;N;;;;; +1412E;EGYPTIAN HIEROGLYPH-1412E;Lo;0;L;;;;;N;;;;; +1412F;EGYPTIAN HIEROGLYPH-1412F;Lo;0;L;;;;;N;;;;; +14130;EGYPTIAN HIEROGLYPH-14130;Lo;0;L;;;;;N;;;;; +14131;EGYPTIAN HIEROGLYPH-14131;Lo;0;L;;;;;N;;;;; +14132;EGYPTIAN HIEROGLYPH-14132;Lo;0;L;;;;;N;;;;; +14133;EGYPTIAN HIEROGLYPH-14133;Lo;0;L;;;;;N;;;;; +14134;EGYPTIAN HIEROGLYPH-14134;Lo;0;L;;;;;N;;;;; +14135;EGYPTIAN HIEROGLYPH-14135;Lo;0;L;;;;;N;;;;; +14136;EGYPTIAN HIEROGLYPH-14136;Lo;0;L;;;;;N;;;;; +14137;EGYPTIAN HIEROGLYPH-14137;Lo;0;L;;;;;N;;;;; +14138;EGYPTIAN HIEROGLYPH-14138;Lo;0;L;;;;;N;;;;; +14139;EGYPTIAN HIEROGLYPH-14139;Lo;0;L;;;;;N;;;;; +1413A;EGYPTIAN HIEROGLYPH-1413A;Lo;0;L;;;;;N;;;;; +1413B;EGYPTIAN HIEROGLYPH-1413B;Lo;0;L;;;;;N;;;;; +1413C;EGYPTIAN HIEROGLYPH-1413C;Lo;0;L;;;;;N;;;;; +1413D;EGYPTIAN HIEROGLYPH-1413D;Lo;0;L;;;;;N;;;;; +1413E;EGYPTIAN HIEROGLYPH-1413E;Lo;0;L;;;;;N;;;;; +1413F;EGYPTIAN HIEROGLYPH-1413F;Lo;0;L;;;;;N;;;;; +14140;EGYPTIAN HIEROGLYPH-14140;Lo;0;L;;;;;N;;;;; +14141;EGYPTIAN HIEROGLYPH-14141;Lo;0;L;;;;;N;;;;; +14142;EGYPTIAN HIEROGLYPH-14142;Lo;0;L;;;;;N;;;;; +14143;EGYPTIAN HIEROGLYPH-14143;Lo;0;L;;;;;N;;;;; +14144;EGYPTIAN HIEROGLYPH-14144;Lo;0;L;;;;;N;;;;; +14145;EGYPTIAN HIEROGLYPH-14145;Lo;0;L;;;;;N;;;;; +14146;EGYPTIAN HIEROGLYPH-14146;Lo;0;L;;;;;N;;;;; +14147;EGYPTIAN HIEROGLYPH-14147;Lo;0;L;;;;;N;;;;; +14148;EGYPTIAN HIEROGLYPH-14148;Lo;0;L;;;;;N;;;;; +14149;EGYPTIAN HIEROGLYPH-14149;Lo;0;L;;;;;N;;;;; +1414A;EGYPTIAN HIEROGLYPH-1414A;Lo;0;L;;;;;N;;;;; +1414B;EGYPTIAN HIEROGLYPH-1414B;Lo;0;L;;;;;N;;;;; +1414C;EGYPTIAN HIEROGLYPH-1414C;Lo;0;L;;;;;N;;;;; +1414D;EGYPTIAN HIEROGLYPH-1414D;Lo;0;L;;;;;N;;;;; +1414E;EGYPTIAN HIEROGLYPH-1414E;Lo;0;L;;;;;N;;;;; +1414F;EGYPTIAN HIEROGLYPH-1414F;Lo;0;L;;;;;N;;;;; +14150;EGYPTIAN HIEROGLYPH-14150;Lo;0;L;;;;;N;;;;; +14151;EGYPTIAN HIEROGLYPH-14151;Lo;0;L;;;;;N;;;;; +14152;EGYPTIAN HIEROGLYPH-14152;Lo;0;L;;;;;N;;;;; +14153;EGYPTIAN HIEROGLYPH-14153;Lo;0;L;;;;;N;;;;; +14154;EGYPTIAN HIEROGLYPH-14154;Lo;0;L;;;;;N;;;;; +14155;EGYPTIAN HIEROGLYPH-14155;Lo;0;L;;;;;N;;;;; +14156;EGYPTIAN HIEROGLYPH-14156;Lo;0;L;;;;;N;;;;; +14157;EGYPTIAN HIEROGLYPH-14157;Lo;0;L;;;;;N;;;;; +14158;EGYPTIAN HIEROGLYPH-14158;Lo;0;L;;;;;N;;;;; +14159;EGYPTIAN HIEROGLYPH-14159;Lo;0;L;;;;;N;;;;; +1415A;EGYPTIAN HIEROGLYPH-1415A;Lo;0;L;;;;;N;;;;; +1415B;EGYPTIAN HIEROGLYPH-1415B;Lo;0;L;;;;;N;;;;; +1415C;EGYPTIAN HIEROGLYPH-1415C;Lo;0;L;;;;;N;;;;; +1415D;EGYPTIAN HIEROGLYPH-1415D;Lo;0;L;;;;;N;;;;; +1415E;EGYPTIAN HIEROGLYPH-1415E;Lo;0;L;;;;;N;;;;; +1415F;EGYPTIAN HIEROGLYPH-1415F;Lo;0;L;;;;;N;;;;; +14160;EGYPTIAN HIEROGLYPH-14160;Lo;0;L;;;;;N;;;;; +14161;EGYPTIAN HIEROGLYPH-14161;Lo;0;L;;;;;N;;;;; +14162;EGYPTIAN HIEROGLYPH-14162;Lo;0;L;;;;;N;;;;; +14163;EGYPTIAN HIEROGLYPH-14163;Lo;0;L;;;;;N;;;;; +14164;EGYPTIAN HIEROGLYPH-14164;Lo;0;L;;;;;N;;;;; +14165;EGYPTIAN HIEROGLYPH-14165;Lo;0;L;;;;;N;;;;; +14166;EGYPTIAN HIEROGLYPH-14166;Lo;0;L;;;;;N;;;;; +14167;EGYPTIAN HIEROGLYPH-14167;Lo;0;L;;;;;N;;;;; +14168;EGYPTIAN HIEROGLYPH-14168;Lo;0;L;;;;;N;;;;; +14169;EGYPTIAN HIEROGLYPH-14169;Lo;0;L;;;;;N;;;;; +1416A;EGYPTIAN HIEROGLYPH-1416A;Lo;0;L;;;;;N;;;;; +1416B;EGYPTIAN HIEROGLYPH-1416B;Lo;0;L;;;;;N;;;;; +1416C;EGYPTIAN HIEROGLYPH-1416C;Lo;0;L;;;;;N;;;;; +1416D;EGYPTIAN HIEROGLYPH-1416D;Lo;0;L;;;;;N;;;;; +1416E;EGYPTIAN HIEROGLYPH-1416E;Lo;0;L;;;;;N;;;;; +1416F;EGYPTIAN HIEROGLYPH-1416F;Lo;0;L;;;;;N;;;;; +14170;EGYPTIAN HIEROGLYPH-14170;Lo;0;L;;;;;N;;;;; +14171;EGYPTIAN HIEROGLYPH-14171;Lo;0;L;;;;;N;;;;; +14172;EGYPTIAN HIEROGLYPH-14172;Lo;0;L;;;;;N;;;;; +14173;EGYPTIAN HIEROGLYPH-14173;Lo;0;L;;;;;N;;;;; +14174;EGYPTIAN HIEROGLYPH-14174;Lo;0;L;;;;;N;;;;; +14175;EGYPTIAN HIEROGLYPH-14175;Lo;0;L;;;;;N;;;;; +14176;EGYPTIAN HIEROGLYPH-14176;Lo;0;L;;;;;N;;;;; +14177;EGYPTIAN HIEROGLYPH-14177;Lo;0;L;;;;;N;;;;; +14178;EGYPTIAN HIEROGLYPH-14178;Lo;0;L;;;;;N;;;;; +14179;EGYPTIAN HIEROGLYPH-14179;Lo;0;L;;;;;N;;;;; +1417A;EGYPTIAN HIEROGLYPH-1417A;Lo;0;L;;;;;N;;;;; +1417B;EGYPTIAN HIEROGLYPH-1417B;Lo;0;L;;;;;N;;;;; +1417C;EGYPTIAN HIEROGLYPH-1417C;Lo;0;L;;;;;N;;;;; +1417D;EGYPTIAN HIEROGLYPH-1417D;Lo;0;L;;;;;N;;;;; +1417E;EGYPTIAN HIEROGLYPH-1417E;Lo;0;L;;;;;N;;;;; +1417F;EGYPTIAN HIEROGLYPH-1417F;Lo;0;L;;;;;N;;;;; +14180;EGYPTIAN HIEROGLYPH-14180;Lo;0;L;;;;;N;;;;; +14181;EGYPTIAN HIEROGLYPH-14181;Lo;0;L;;;;;N;;;;; +14182;EGYPTIAN HIEROGLYPH-14182;Lo;0;L;;;;;N;;;;; +14183;EGYPTIAN HIEROGLYPH-14183;Lo;0;L;;;;;N;;;;; +14184;EGYPTIAN HIEROGLYPH-14184;Lo;0;L;;;;;N;;;;; +14185;EGYPTIAN HIEROGLYPH-14185;Lo;0;L;;;;;N;;;;; +14186;EGYPTIAN HIEROGLYPH-14186;Lo;0;L;;;;;N;;;;; +14187;EGYPTIAN HIEROGLYPH-14187;Lo;0;L;;;;;N;;;;; +14188;EGYPTIAN HIEROGLYPH-14188;Lo;0;L;;;;;N;;;;; +14189;EGYPTIAN HIEROGLYPH-14189;Lo;0;L;;;;;N;;;;; +1418A;EGYPTIAN HIEROGLYPH-1418A;Lo;0;L;;;;;N;;;;; +1418B;EGYPTIAN HIEROGLYPH-1418B;Lo;0;L;;;;;N;;;;; +1418C;EGYPTIAN HIEROGLYPH-1418C;Lo;0;L;;;;;N;;;;; +1418D;EGYPTIAN HIEROGLYPH-1418D;Lo;0;L;;;;;N;;;;; +1418E;EGYPTIAN HIEROGLYPH-1418E;Lo;0;L;;;;;N;;;;; +1418F;EGYPTIAN HIEROGLYPH-1418F;Lo;0;L;;;;;N;;;;; +14190;EGYPTIAN HIEROGLYPH-14190;Lo;0;L;;;;;N;;;;; +14191;EGYPTIAN HIEROGLYPH-14191;Lo;0;L;;;;;N;;;;; +14192;EGYPTIAN HIEROGLYPH-14192;Lo;0;L;;;;;N;;;;; +14193;EGYPTIAN HIEROGLYPH-14193;Lo;0;L;;;;;N;;;;; +14194;EGYPTIAN HIEROGLYPH-14194;Lo;0;L;;;;;N;;;;; +14195;EGYPTIAN HIEROGLYPH-14195;Lo;0;L;;;;;N;;;;; +14196;EGYPTIAN HIEROGLYPH-14196;Lo;0;L;;;;;N;;;;; +14197;EGYPTIAN HIEROGLYPH-14197;Lo;0;L;;;;;N;;;;; +14198;EGYPTIAN HIEROGLYPH-14198;Lo;0;L;;;;;N;;;;; +14199;EGYPTIAN HIEROGLYPH-14199;Lo;0;L;;;;;N;;;;; +1419A;EGYPTIAN HIEROGLYPH-1419A;Lo;0;L;;;;;N;;;;; +1419B;EGYPTIAN HIEROGLYPH-1419B;Lo;0;L;;;;;N;;;;; +1419C;EGYPTIAN HIEROGLYPH-1419C;Lo;0;L;;;;;N;;;;; +1419D;EGYPTIAN HIEROGLYPH-1419D;Lo;0;L;;;;;N;;;;; +1419E;EGYPTIAN HIEROGLYPH-1419E;Lo;0;L;;;;;N;;;;; +1419F;EGYPTIAN HIEROGLYPH-1419F;Lo;0;L;;;;;N;;;;; +141A0;EGYPTIAN HIEROGLYPH-141A0;Lo;0;L;;;;;N;;;;; +141A1;EGYPTIAN HIEROGLYPH-141A1;Lo;0;L;;;;;N;;;;; +141A2;EGYPTIAN HIEROGLYPH-141A2;Lo;0;L;;;;;N;;;;; +141A3;EGYPTIAN HIEROGLYPH-141A3;Lo;0;L;;;;;N;;;;; +141A4;EGYPTIAN HIEROGLYPH-141A4;Lo;0;L;;;;;N;;;;; +141A5;EGYPTIAN HIEROGLYPH-141A5;Lo;0;L;;;;;N;;;;; +141A6;EGYPTIAN HIEROGLYPH-141A6;Lo;0;L;;;;;N;;;;; +141A7;EGYPTIAN HIEROGLYPH-141A7;Lo;0;L;;;;;N;;;;; +141A8;EGYPTIAN HIEROGLYPH-141A8;Lo;0;L;;;;;N;;;;; +141A9;EGYPTIAN HIEROGLYPH-141A9;Lo;0;L;;;;;N;;;;; +141AA;EGYPTIAN HIEROGLYPH-141AA;Lo;0;L;;;;;N;;;;; +141AB;EGYPTIAN HIEROGLYPH-141AB;Lo;0;L;;;;;N;;;;; +141AC;EGYPTIAN HIEROGLYPH-141AC;Lo;0;L;;;;;N;;;;; +141AD;EGYPTIAN HIEROGLYPH-141AD;Lo;0;L;;;;;N;;;;; +141AE;EGYPTIAN HIEROGLYPH-141AE;Lo;0;L;;;;;N;;;;; +141AF;EGYPTIAN HIEROGLYPH-141AF;Lo;0;L;;;;;N;;;;; +141B0;EGYPTIAN HIEROGLYPH-141B0;Lo;0;L;;;;;N;;;;; +141B1;EGYPTIAN HIEROGLYPH-141B1;Lo;0;L;;;;;N;;;;; +141B2;EGYPTIAN HIEROGLYPH-141B2;Lo;0;L;;;;;N;;;;; +141B3;EGYPTIAN HIEROGLYPH-141B3;Lo;0;L;;;;;N;;;;; +141B4;EGYPTIAN HIEROGLYPH-141B4;Lo;0;L;;;;;N;;;;; +141B5;EGYPTIAN HIEROGLYPH-141B5;Lo;0;L;;;;;N;;;;; +141B6;EGYPTIAN HIEROGLYPH-141B6;Lo;0;L;;;;;N;;;;; +141B7;EGYPTIAN HIEROGLYPH-141B7;Lo;0;L;;;;;N;;;;; +141B8;EGYPTIAN HIEROGLYPH-141B8;Lo;0;L;;;;;N;;;;; +141B9;EGYPTIAN HIEROGLYPH-141B9;Lo;0;L;;;;;N;;;;; +141BA;EGYPTIAN HIEROGLYPH-141BA;Lo;0;L;;;;;N;;;;; +141BB;EGYPTIAN HIEROGLYPH-141BB;Lo;0;L;;;;;N;;;;; +141BC;EGYPTIAN HIEROGLYPH-141BC;Lo;0;L;;;;;N;;;;; +141BD;EGYPTIAN HIEROGLYPH-141BD;Lo;0;L;;;;;N;;;;; +141BE;EGYPTIAN HIEROGLYPH-141BE;Lo;0;L;;;;;N;;;;; +141BF;EGYPTIAN HIEROGLYPH-141BF;Lo;0;L;;;;;N;;;;; +141C0;EGYPTIAN HIEROGLYPH-141C0;Lo;0;L;;;;;N;;;;; +141C1;EGYPTIAN HIEROGLYPH-141C1;Lo;0;L;;;;;N;;;;; +141C2;EGYPTIAN HIEROGLYPH-141C2;Lo;0;L;;;;;N;;;;; +141C3;EGYPTIAN HIEROGLYPH-141C3;Lo;0;L;;;;;N;;;;; +141C4;EGYPTIAN HIEROGLYPH-141C4;Lo;0;L;;;;;N;;;;; +141C5;EGYPTIAN HIEROGLYPH-141C5;Lo;0;L;;;;;N;;;;; +141C6;EGYPTIAN HIEROGLYPH-141C6;Lo;0;L;;;;;N;;;;; +141C7;EGYPTIAN HIEROGLYPH-141C7;Lo;0;L;;;;;N;;;;; +141C8;EGYPTIAN HIEROGLYPH-141C8;Lo;0;L;;;;;N;;;;; +141C9;EGYPTIAN HIEROGLYPH-141C9;Lo;0;L;;;;;N;;;;; +141CA;EGYPTIAN HIEROGLYPH-141CA;Lo;0;L;;;;;N;;;;; +141CB;EGYPTIAN HIEROGLYPH-141CB;Lo;0;L;;;;;N;;;;; +141CC;EGYPTIAN HIEROGLYPH-141CC;Lo;0;L;;;;;N;;;;; +141CD;EGYPTIAN HIEROGLYPH-141CD;Lo;0;L;;;;;N;;;;; +141CE;EGYPTIAN HIEROGLYPH-141CE;Lo;0;L;;;;;N;;;;; +141CF;EGYPTIAN HIEROGLYPH-141CF;Lo;0;L;;;;;N;;;;; +141D0;EGYPTIAN HIEROGLYPH-141D0;Lo;0;L;;;;;N;;;;; +141D1;EGYPTIAN HIEROGLYPH-141D1;Lo;0;L;;;;;N;;;;; +141D2;EGYPTIAN HIEROGLYPH-141D2;Lo;0;L;;;;;N;;;;; +141D3;EGYPTIAN HIEROGLYPH-141D3;Lo;0;L;;;;;N;;;;; +141D4;EGYPTIAN HIEROGLYPH-141D4;Lo;0;L;;;;;N;;;;; +141D5;EGYPTIAN HIEROGLYPH-141D5;Lo;0;L;;;;;N;;;;; +141D6;EGYPTIAN HIEROGLYPH-141D6;Lo;0;L;;;;;N;;;;; +141D7;EGYPTIAN HIEROGLYPH-141D7;Lo;0;L;;;;;N;;;;; +141D8;EGYPTIAN HIEROGLYPH-141D8;Lo;0;L;;;;;N;;;;; +141D9;EGYPTIAN HIEROGLYPH-141D9;Lo;0;L;;;;;N;;;;; +141DA;EGYPTIAN HIEROGLYPH-141DA;Lo;0;L;;;;;N;;;;; +141DB;EGYPTIAN HIEROGLYPH-141DB;Lo;0;L;;;;;N;;;;; +141DC;EGYPTIAN HIEROGLYPH-141DC;Lo;0;L;;;;;N;;;;; +141DD;EGYPTIAN HIEROGLYPH-141DD;Lo;0;L;;;;;N;;;;; +141DE;EGYPTIAN HIEROGLYPH-141DE;Lo;0;L;;;;;N;;;;; +141DF;EGYPTIAN HIEROGLYPH-141DF;Lo;0;L;;;;;N;;;;; +141E0;EGYPTIAN HIEROGLYPH-141E0;Lo;0;L;;;;;N;;;;; +141E1;EGYPTIAN HIEROGLYPH-141E1;Lo;0;L;;;;;N;;;;; +141E2;EGYPTIAN HIEROGLYPH-141E2;Lo;0;L;;;;;N;;;;; +141E3;EGYPTIAN HIEROGLYPH-141E3;Lo;0;L;;;;;N;;;;; +141E4;EGYPTIAN HIEROGLYPH-141E4;Lo;0;L;;;;;N;;;;; +141E5;EGYPTIAN HIEROGLYPH-141E5;Lo;0;L;;;;;N;;;;; +141E6;EGYPTIAN HIEROGLYPH-141E6;Lo;0;L;;;;;N;;;;; +141E7;EGYPTIAN HIEROGLYPH-141E7;Lo;0;L;;;;;N;;;;; +141E8;EGYPTIAN HIEROGLYPH-141E8;Lo;0;L;;;;;N;;;;; +141E9;EGYPTIAN HIEROGLYPH-141E9;Lo;0;L;;;;;N;;;;; +141EA;EGYPTIAN HIEROGLYPH-141EA;Lo;0;L;;;;;N;;;;; +141EB;EGYPTIAN HIEROGLYPH-141EB;Lo;0;L;;;;;N;;;;; +141EC;EGYPTIAN HIEROGLYPH-141EC;Lo;0;L;;;;;N;;;;; +141ED;EGYPTIAN HIEROGLYPH-141ED;Lo;0;L;;;;;N;;;;; +141EE;EGYPTIAN HIEROGLYPH-141EE;Lo;0;L;;;;;N;;;;; +141EF;EGYPTIAN HIEROGLYPH-141EF;Lo;0;L;;;;;N;;;;; +141F0;EGYPTIAN HIEROGLYPH-141F0;Lo;0;L;;;;;N;;;;; +141F1;EGYPTIAN HIEROGLYPH-141F1;Lo;0;L;;;;;N;;;;; +141F2;EGYPTIAN HIEROGLYPH-141F2;Lo;0;L;;;;;N;;;;; +141F3;EGYPTIAN HIEROGLYPH-141F3;Lo;0;L;;;;;N;;;;; +141F4;EGYPTIAN HIEROGLYPH-141F4;Lo;0;L;;;;;N;;;;; +141F5;EGYPTIAN HIEROGLYPH-141F5;Lo;0;L;;;;;N;;;;; +141F6;EGYPTIAN HIEROGLYPH-141F6;Lo;0;L;;;;;N;;;;; +141F7;EGYPTIAN HIEROGLYPH-141F7;Lo;0;L;;;;;N;;;;; +141F8;EGYPTIAN HIEROGLYPH-141F8;Lo;0;L;;;;;N;;;;; +141F9;EGYPTIAN HIEROGLYPH-141F9;Lo;0;L;;;;;N;;;;; +141FA;EGYPTIAN HIEROGLYPH-141FA;Lo;0;L;;;;;N;;;;; +141FB;EGYPTIAN HIEROGLYPH-141FB;Lo;0;L;;;;;N;;;;; +141FC;EGYPTIAN HIEROGLYPH-141FC;Lo;0;L;;;;;N;;;;; +141FD;EGYPTIAN HIEROGLYPH-141FD;Lo;0;L;;;;;N;;;;; +141FE;EGYPTIAN HIEROGLYPH-141FE;Lo;0;L;;;;;N;;;;; +141FF;EGYPTIAN HIEROGLYPH-141FF;Lo;0;L;;;;;N;;;;; +14200;EGYPTIAN HIEROGLYPH-14200;Lo;0;L;;;;;N;;;;; +14201;EGYPTIAN HIEROGLYPH-14201;Lo;0;L;;;;;N;;;;; +14202;EGYPTIAN HIEROGLYPH-14202;Lo;0;L;;;;;N;;;;; +14203;EGYPTIAN HIEROGLYPH-14203;Lo;0;L;;;;;N;;;;; +14204;EGYPTIAN HIEROGLYPH-14204;Lo;0;L;;;;;N;;;;; +14205;EGYPTIAN HIEROGLYPH-14205;Lo;0;L;;;;;N;;;;; +14206;EGYPTIAN HIEROGLYPH-14206;Lo;0;L;;;;;N;;;;; +14207;EGYPTIAN HIEROGLYPH-14207;Lo;0;L;;;;;N;;;;; +14208;EGYPTIAN HIEROGLYPH-14208;Lo;0;L;;;;;N;;;;; +14209;EGYPTIAN HIEROGLYPH-14209;Lo;0;L;;;;;N;;;;; +1420A;EGYPTIAN HIEROGLYPH-1420A;Lo;0;L;;;;;N;;;;; +1420B;EGYPTIAN HIEROGLYPH-1420B;Lo;0;L;;;;;N;;;;; +1420C;EGYPTIAN HIEROGLYPH-1420C;Lo;0;L;;;;;N;;;;; +1420D;EGYPTIAN HIEROGLYPH-1420D;Lo;0;L;;;;;N;;;;; +1420E;EGYPTIAN HIEROGLYPH-1420E;Lo;0;L;;;;;N;;;;; +1420F;EGYPTIAN HIEROGLYPH-1420F;Lo;0;L;;;;;N;;;;; +14210;EGYPTIAN HIEROGLYPH-14210;Lo;0;L;;;;;N;;;;; +14211;EGYPTIAN HIEROGLYPH-14211;Lo;0;L;;;;;N;;;;; +14212;EGYPTIAN HIEROGLYPH-14212;Lo;0;L;;;;;N;;;;; +14213;EGYPTIAN HIEROGLYPH-14213;Lo;0;L;;;;;N;;;;; +14214;EGYPTIAN HIEROGLYPH-14214;Lo;0;L;;;;;N;;;;; +14215;EGYPTIAN HIEROGLYPH-14215;Lo;0;L;;;;;N;;;;; +14216;EGYPTIAN HIEROGLYPH-14216;Lo;0;L;;;;;N;;;;; +14217;EGYPTIAN HIEROGLYPH-14217;Lo;0;L;;;;;N;;;;; +14218;EGYPTIAN HIEROGLYPH-14218;Lo;0;L;;;;;N;;;;; +14219;EGYPTIAN HIEROGLYPH-14219;Lo;0;L;;;;;N;;;;; +1421A;EGYPTIAN HIEROGLYPH-1421A;Lo;0;L;;;;;N;;;;; +1421B;EGYPTIAN HIEROGLYPH-1421B;Lo;0;L;;;;;N;;;;; +1421C;EGYPTIAN HIEROGLYPH-1421C;Lo;0;L;;;;;N;;;;; +1421D;EGYPTIAN HIEROGLYPH-1421D;Lo;0;L;;;;;N;;;;; +1421E;EGYPTIAN HIEROGLYPH-1421E;Lo;0;L;;;;;N;;;;; +1421F;EGYPTIAN HIEROGLYPH-1421F;Lo;0;L;;;;;N;;;;; +14220;EGYPTIAN HIEROGLYPH-14220;Lo;0;L;;;;;N;;;;; +14221;EGYPTIAN HIEROGLYPH-14221;Lo;0;L;;;;;N;;;;; +14222;EGYPTIAN HIEROGLYPH-14222;Lo;0;L;;;;;N;;;;; +14223;EGYPTIAN HIEROGLYPH-14223;Lo;0;L;;;;;N;;;;; +14224;EGYPTIAN HIEROGLYPH-14224;Lo;0;L;;;;;N;;;;; +14225;EGYPTIAN HIEROGLYPH-14225;Lo;0;L;;;;;N;;;;; +14226;EGYPTIAN HIEROGLYPH-14226;Lo;0;L;;;;;N;;;;; +14227;EGYPTIAN HIEROGLYPH-14227;Lo;0;L;;;;;N;;;;; +14228;EGYPTIAN HIEROGLYPH-14228;Lo;0;L;;;;;N;;;;; +14229;EGYPTIAN HIEROGLYPH-14229;Lo;0;L;;;;;N;;;;; +1422A;EGYPTIAN HIEROGLYPH-1422A;Lo;0;L;;;;;N;;;;; +1422B;EGYPTIAN HIEROGLYPH-1422B;Lo;0;L;;;;;N;;;;; +1422C;EGYPTIAN HIEROGLYPH-1422C;Lo;0;L;;;;;N;;;;; +1422D;EGYPTIAN HIEROGLYPH-1422D;Lo;0;L;;;;;N;;;;; +1422E;EGYPTIAN HIEROGLYPH-1422E;Lo;0;L;;;;;N;;;;; +1422F;EGYPTIAN HIEROGLYPH-1422F;Lo;0;L;;;;;N;;;;; +14230;EGYPTIAN HIEROGLYPH-14230;Lo;0;L;;;;;N;;;;; +14231;EGYPTIAN HIEROGLYPH-14231;Lo;0;L;;;;;N;;;;; +14232;EGYPTIAN HIEROGLYPH-14232;Lo;0;L;;;;;N;;;;; +14233;EGYPTIAN HIEROGLYPH-14233;Lo;0;L;;;;;N;;;;; +14234;EGYPTIAN HIEROGLYPH-14234;Lo;0;L;;;;;N;;;;; +14235;EGYPTIAN HIEROGLYPH-14235;Lo;0;L;;;;;N;;;;; +14236;EGYPTIAN HIEROGLYPH-14236;Lo;0;L;;;;;N;;;;; +14237;EGYPTIAN HIEROGLYPH-14237;Lo;0;L;;;;;N;;;;; +14238;EGYPTIAN HIEROGLYPH-14238;Lo;0;L;;;;;N;;;;; +14239;EGYPTIAN HIEROGLYPH-14239;Lo;0;L;;;;;N;;;;; +1423A;EGYPTIAN HIEROGLYPH-1423A;Lo;0;L;;;;;N;;;;; +1423B;EGYPTIAN HIEROGLYPH-1423B;Lo;0;L;;;;;N;;;;; +1423C;EGYPTIAN HIEROGLYPH-1423C;Lo;0;L;;;;;N;;;;; +1423D;EGYPTIAN HIEROGLYPH-1423D;Lo;0;L;;;;;N;;;;; +1423E;EGYPTIAN HIEROGLYPH-1423E;Lo;0;L;;;;;N;;;;; +1423F;EGYPTIAN HIEROGLYPH-1423F;Lo;0;L;;;;;N;;;;; +14240;EGYPTIAN HIEROGLYPH-14240;Lo;0;L;;;;;N;;;;; +14241;EGYPTIAN HIEROGLYPH-14241;Lo;0;L;;;;;N;;;;; +14242;EGYPTIAN HIEROGLYPH-14242;Lo;0;L;;;;;N;;;;; +14243;EGYPTIAN HIEROGLYPH-14243;Lo;0;L;;;;;N;;;;; +14244;EGYPTIAN HIEROGLYPH-14244;Lo;0;L;;;;;N;;;;; +14245;EGYPTIAN HIEROGLYPH-14245;Lo;0;L;;;;;N;;;;; +14246;EGYPTIAN HIEROGLYPH-14246;Lo;0;L;;;;;N;;;;; +14247;EGYPTIAN HIEROGLYPH-14247;Lo;0;L;;;;;N;;;;; +14248;EGYPTIAN HIEROGLYPH-14248;Lo;0;L;;;;;N;;;;; +14249;EGYPTIAN HIEROGLYPH-14249;Lo;0;L;;;;;N;;;;; +1424A;EGYPTIAN HIEROGLYPH-1424A;Lo;0;L;;;;;N;;;;; +1424B;EGYPTIAN HIEROGLYPH-1424B;Lo;0;L;;;;;N;;;;; +1424C;EGYPTIAN HIEROGLYPH-1424C;Lo;0;L;;;;;N;;;;; +1424D;EGYPTIAN HIEROGLYPH-1424D;Lo;0;L;;;;;N;;;;; +1424E;EGYPTIAN HIEROGLYPH-1424E;Lo;0;L;;;;;N;;;;; +1424F;EGYPTIAN HIEROGLYPH-1424F;Lo;0;L;;;;;N;;;;; +14250;EGYPTIAN HIEROGLYPH-14250;Lo;0;L;;;;;N;;;;; +14251;EGYPTIAN HIEROGLYPH-14251;Lo;0;L;;;;;N;;;;; +14252;EGYPTIAN HIEROGLYPH-14252;Lo;0;L;;;;;N;;;;; +14253;EGYPTIAN HIEROGLYPH-14253;Lo;0;L;;;;;N;;;;; +14254;EGYPTIAN HIEROGLYPH-14254;Lo;0;L;;;;;N;;;;; +14255;EGYPTIAN HIEROGLYPH-14255;Lo;0;L;;;;;N;;;;; +14256;EGYPTIAN HIEROGLYPH-14256;Lo;0;L;;;;;N;;;;; +14257;EGYPTIAN HIEROGLYPH-14257;Lo;0;L;;;;;N;;;;; +14258;EGYPTIAN HIEROGLYPH-14258;Lo;0;L;;;;;N;;;;; +14259;EGYPTIAN HIEROGLYPH-14259;Lo;0;L;;;;;N;;;;; +1425A;EGYPTIAN HIEROGLYPH-1425A;Lo;0;L;;;;;N;;;;; +1425B;EGYPTIAN HIEROGLYPH-1425B;Lo;0;L;;;;;N;;;;; +1425C;EGYPTIAN HIEROGLYPH-1425C;Lo;0;L;;;;;N;;;;; +1425D;EGYPTIAN HIEROGLYPH-1425D;Lo;0;L;;;;;N;;;;; +1425E;EGYPTIAN HIEROGLYPH-1425E;Lo;0;L;;;;;N;;;;; +1425F;EGYPTIAN HIEROGLYPH-1425F;Lo;0;L;;;;;N;;;;; +14260;EGYPTIAN HIEROGLYPH-14260;Lo;0;L;;;;;N;;;;; +14261;EGYPTIAN HIEROGLYPH-14261;Lo;0;L;;;;;N;;;;; +14262;EGYPTIAN HIEROGLYPH-14262;Lo;0;L;;;;;N;;;;; +14263;EGYPTIAN HIEROGLYPH-14263;Lo;0;L;;;;;N;;;;; +14264;EGYPTIAN HIEROGLYPH-14264;Lo;0;L;;;;;N;;;;; +14265;EGYPTIAN HIEROGLYPH-14265;Lo;0;L;;;;;N;;;;; +14266;EGYPTIAN HIEROGLYPH-14266;Lo;0;L;;;;;N;;;;; +14267;EGYPTIAN HIEROGLYPH-14267;Lo;0;L;;;;;N;;;;; +14268;EGYPTIAN HIEROGLYPH-14268;Lo;0;L;;;;;N;;;;; +14269;EGYPTIAN HIEROGLYPH-14269;Lo;0;L;;;;;N;;;;; +1426A;EGYPTIAN HIEROGLYPH-1426A;Lo;0;L;;;;;N;;;;; +1426B;EGYPTIAN HIEROGLYPH-1426B;Lo;0;L;;;;;N;;;;; +1426C;EGYPTIAN HIEROGLYPH-1426C;Lo;0;L;;;;;N;;;;; +1426D;EGYPTIAN HIEROGLYPH-1426D;Lo;0;L;;;;;N;;;;; +1426E;EGYPTIAN HIEROGLYPH-1426E;Lo;0;L;;;;;N;;;;; +1426F;EGYPTIAN HIEROGLYPH-1426F;Lo;0;L;;;;;N;;;;; +14270;EGYPTIAN HIEROGLYPH-14270;Lo;0;L;;;;;N;;;;; +14271;EGYPTIAN HIEROGLYPH-14271;Lo;0;L;;;;;N;;;;; +14272;EGYPTIAN HIEROGLYPH-14272;Lo;0;L;;;;;N;;;;; +14273;EGYPTIAN HIEROGLYPH-14273;Lo;0;L;;;;;N;;;;; +14274;EGYPTIAN HIEROGLYPH-14274;Lo;0;L;;;;;N;;;;; +14275;EGYPTIAN HIEROGLYPH-14275;Lo;0;L;;;;;N;;;;; +14276;EGYPTIAN HIEROGLYPH-14276;Lo;0;L;;;;;N;;;;; +14277;EGYPTIAN HIEROGLYPH-14277;Lo;0;L;;;;;N;;;;; +14278;EGYPTIAN HIEROGLYPH-14278;Lo;0;L;;;;;N;;;;; +14279;EGYPTIAN HIEROGLYPH-14279;Lo;0;L;;;;;N;;;;; +1427A;EGYPTIAN HIEROGLYPH-1427A;Lo;0;L;;;;;N;;;;; +1427B;EGYPTIAN HIEROGLYPH-1427B;Lo;0;L;;;;;N;;;;; +1427C;EGYPTIAN HIEROGLYPH-1427C;Lo;0;L;;;;;N;;;;; +1427D;EGYPTIAN HIEROGLYPH-1427D;Lo;0;L;;;;;N;;;;; +1427E;EGYPTIAN HIEROGLYPH-1427E;Lo;0;L;;;;;N;;;;; +1427F;EGYPTIAN HIEROGLYPH-1427F;Lo;0;L;;;;;N;;;;; +14280;EGYPTIAN HIEROGLYPH-14280;Lo;0;L;;;;;N;;;;; +14281;EGYPTIAN HIEROGLYPH-14281;Lo;0;L;;;;;N;;;;; +14282;EGYPTIAN HIEROGLYPH-14282;Lo;0;L;;;;;N;;;;; +14283;EGYPTIAN HIEROGLYPH-14283;Lo;0;L;;;;;N;;;;; +14284;EGYPTIAN HIEROGLYPH-14284;Lo;0;L;;;;;N;;;;; +14285;EGYPTIAN HIEROGLYPH-14285;Lo;0;L;;;;;N;;;;; +14286;EGYPTIAN HIEROGLYPH-14286;Lo;0;L;;;;;N;;;;; +14287;EGYPTIAN HIEROGLYPH-14287;Lo;0;L;;;;;N;;;;; +14288;EGYPTIAN HIEROGLYPH-14288;Lo;0;L;;;;;N;;;;; +14289;EGYPTIAN HIEROGLYPH-14289;Lo;0;L;;;;;N;;;;; +1428A;EGYPTIAN HIEROGLYPH-1428A;Lo;0;L;;;;;N;;;;; +1428B;EGYPTIAN HIEROGLYPH-1428B;Lo;0;L;;;;;N;;;;; +1428C;EGYPTIAN HIEROGLYPH-1428C;Lo;0;L;;;;;N;;;;; +1428D;EGYPTIAN HIEROGLYPH-1428D;Lo;0;L;;;;;N;;;;; +1428E;EGYPTIAN HIEROGLYPH-1428E;Lo;0;L;;;;;N;;;;; +1428F;EGYPTIAN HIEROGLYPH-1428F;Lo;0;L;;;;;N;;;;; +14290;EGYPTIAN HIEROGLYPH-14290;Lo;0;L;;;;;N;;;;; +14291;EGYPTIAN HIEROGLYPH-14291;Lo;0;L;;;;;N;;;;; +14292;EGYPTIAN HIEROGLYPH-14292;Lo;0;L;;;;;N;;;;; +14293;EGYPTIAN HIEROGLYPH-14293;Lo;0;L;;;;;N;;;;; +14294;EGYPTIAN HIEROGLYPH-14294;Lo;0;L;;;;;N;;;;; +14295;EGYPTIAN HIEROGLYPH-14295;Lo;0;L;;;;;N;;;;; +14296;EGYPTIAN HIEROGLYPH-14296;Lo;0;L;;;;;N;;;;; +14297;EGYPTIAN HIEROGLYPH-14297;Lo;0;L;;;;;N;;;;; +14298;EGYPTIAN HIEROGLYPH-14298;Lo;0;L;;;;;N;;;;; +14299;EGYPTIAN HIEROGLYPH-14299;Lo;0;L;;;;;N;;;;; +1429A;EGYPTIAN HIEROGLYPH-1429A;Lo;0;L;;;;;N;;;;; +1429B;EGYPTIAN HIEROGLYPH-1429B;Lo;0;L;;;;;N;;;;; +1429C;EGYPTIAN HIEROGLYPH-1429C;Lo;0;L;;;;;N;;;;; +1429D;EGYPTIAN HIEROGLYPH-1429D;Lo;0;L;;;;;N;;;;; +1429E;EGYPTIAN HIEROGLYPH-1429E;Lo;0;L;;;;;N;;;;; +1429F;EGYPTIAN HIEROGLYPH-1429F;Lo;0;L;;;;;N;;;;; +142A0;EGYPTIAN HIEROGLYPH-142A0;Lo;0;L;;;;;N;;;;; +142A1;EGYPTIAN HIEROGLYPH-142A1;Lo;0;L;;;;;N;;;;; +142A2;EGYPTIAN HIEROGLYPH-142A2;Lo;0;L;;;;;N;;;;; +142A3;EGYPTIAN HIEROGLYPH-142A3;Lo;0;L;;;;;N;;;;; +142A4;EGYPTIAN HIEROGLYPH-142A4;Lo;0;L;;;;;N;;;;; +142A5;EGYPTIAN HIEROGLYPH-142A5;Lo;0;L;;;;;N;;;;; +142A6;EGYPTIAN HIEROGLYPH-142A6;Lo;0;L;;;;;N;;;;; +142A7;EGYPTIAN HIEROGLYPH-142A7;Lo;0;L;;;;;N;;;;; +142A8;EGYPTIAN HIEROGLYPH-142A8;Lo;0;L;;;;;N;;;;; +142A9;EGYPTIAN HIEROGLYPH-142A9;Lo;0;L;;;;;N;;;;; +142AA;EGYPTIAN HIEROGLYPH-142AA;Lo;0;L;;;;;N;;;;; +142AB;EGYPTIAN HIEROGLYPH-142AB;Lo;0;L;;;;;N;;;;; +142AC;EGYPTIAN HIEROGLYPH-142AC;Lo;0;L;;;;;N;;;;; +142AD;EGYPTIAN HIEROGLYPH-142AD;Lo;0;L;;;;;N;;;;; +142AE;EGYPTIAN HIEROGLYPH-142AE;Lo;0;L;;;;;N;;;;; +142AF;EGYPTIAN HIEROGLYPH-142AF;Lo;0;L;;;;;N;;;;; +142B0;EGYPTIAN HIEROGLYPH-142B0;Lo;0;L;;;;;N;;;;; +142B1;EGYPTIAN HIEROGLYPH-142B1;Lo;0;L;;;;;N;;;;; +142B2;EGYPTIAN HIEROGLYPH-142B2;Lo;0;L;;;;;N;;;;; +142B3;EGYPTIAN HIEROGLYPH-142B3;Lo;0;L;;;;;N;;;;; +142B4;EGYPTIAN HIEROGLYPH-142B4;Lo;0;L;;;;;N;;;;; +142B5;EGYPTIAN HIEROGLYPH-142B5;Lo;0;L;;;;;N;;;;; +142B6;EGYPTIAN HIEROGLYPH-142B6;Lo;0;L;;;;;N;;;;; +142B7;EGYPTIAN HIEROGLYPH-142B7;Lo;0;L;;;;;N;;;;; +142B8;EGYPTIAN HIEROGLYPH-142B8;Lo;0;L;;;;;N;;;;; +142B9;EGYPTIAN HIEROGLYPH-142B9;Lo;0;L;;;;;N;;;;; +142BA;EGYPTIAN HIEROGLYPH-142BA;Lo;0;L;;;;;N;;;;; +142BB;EGYPTIAN HIEROGLYPH-142BB;Lo;0;L;;;;;N;;;;; +142BC;EGYPTIAN HIEROGLYPH-142BC;Lo;0;L;;;;;N;;;;; +142BD;EGYPTIAN HIEROGLYPH-142BD;Lo;0;L;;;;;N;;;;; +142BE;EGYPTIAN HIEROGLYPH-142BE;Lo;0;L;;;;;N;;;;; +142BF;EGYPTIAN HIEROGLYPH-142BF;Lo;0;L;;;;;N;;;;; +142C0;EGYPTIAN HIEROGLYPH-142C0;Lo;0;L;;;;;N;;;;; +142C1;EGYPTIAN HIEROGLYPH-142C1;Lo;0;L;;;;;N;;;;; +142C2;EGYPTIAN HIEROGLYPH-142C2;Lo;0;L;;;;;N;;;;; +142C3;EGYPTIAN HIEROGLYPH-142C3;Lo;0;L;;;;;N;;;;; +142C4;EGYPTIAN HIEROGLYPH-142C4;Lo;0;L;;;;;N;;;;; +142C5;EGYPTIAN HIEROGLYPH-142C5;Lo;0;L;;;;;N;;;;; +142C6;EGYPTIAN HIEROGLYPH-142C6;Lo;0;L;;;;;N;;;;; +142C7;EGYPTIAN HIEROGLYPH-142C7;Lo;0;L;;;;;N;;;;; +142C8;EGYPTIAN HIEROGLYPH-142C8;Lo;0;L;;;;;N;;;;; +142C9;EGYPTIAN HIEROGLYPH-142C9;Lo;0;L;;;;;N;;;;; +142CA;EGYPTIAN HIEROGLYPH-142CA;Lo;0;L;;;;;N;;;;; +142CB;EGYPTIAN HIEROGLYPH-142CB;Lo;0;L;;;;;N;;;;; +142CC;EGYPTIAN HIEROGLYPH-142CC;Lo;0;L;;;;;N;;;;; +142CD;EGYPTIAN HIEROGLYPH-142CD;Lo;0;L;;;;;N;;;;; +142CE;EGYPTIAN HIEROGLYPH-142CE;Lo;0;L;;;;;N;;;;; +142CF;EGYPTIAN HIEROGLYPH-142CF;Lo;0;L;;;;;N;;;;; +142D0;EGYPTIAN HIEROGLYPH-142D0;Lo;0;L;;;;;N;;;;; +142D1;EGYPTIAN HIEROGLYPH-142D1;Lo;0;L;;;;;N;;;;; +142D2;EGYPTIAN HIEROGLYPH-142D2;Lo;0;L;;;;;N;;;;; +142D3;EGYPTIAN HIEROGLYPH-142D3;Lo;0;L;;;;;N;;;;; +142D4;EGYPTIAN HIEROGLYPH-142D4;Lo;0;L;;;;;N;;;;; +142D5;EGYPTIAN HIEROGLYPH-142D5;Lo;0;L;;;;;N;;;;; +142D6;EGYPTIAN HIEROGLYPH-142D6;Lo;0;L;;;;;N;;;;; +142D7;EGYPTIAN HIEROGLYPH-142D7;Lo;0;L;;;;;N;;;;; +142D8;EGYPTIAN HIEROGLYPH-142D8;Lo;0;L;;;;;N;;;;; +142D9;EGYPTIAN HIEROGLYPH-142D9;Lo;0;L;;;;;N;;;;; +142DA;EGYPTIAN HIEROGLYPH-142DA;Lo;0;L;;;;;N;;;;; +142DB;EGYPTIAN HIEROGLYPH-142DB;Lo;0;L;;;;;N;;;;; +142DC;EGYPTIAN HIEROGLYPH-142DC;Lo;0;L;;;;;N;;;;; +142DD;EGYPTIAN HIEROGLYPH-142DD;Lo;0;L;;;;;N;;;;; +142DE;EGYPTIAN HIEROGLYPH-142DE;Lo;0;L;;;;;N;;;;; +142DF;EGYPTIAN HIEROGLYPH-142DF;Lo;0;L;;;;;N;;;;; +142E0;EGYPTIAN HIEROGLYPH-142E0;Lo;0;L;;;;;N;;;;; +142E1;EGYPTIAN HIEROGLYPH-142E1;Lo;0;L;;;;;N;;;;; +142E2;EGYPTIAN HIEROGLYPH-142E2;Lo;0;L;;;;;N;;;;; +142E3;EGYPTIAN HIEROGLYPH-142E3;Lo;0;L;;;;;N;;;;; +142E4;EGYPTIAN HIEROGLYPH-142E4;Lo;0;L;;;;;N;;;;; +142E5;EGYPTIAN HIEROGLYPH-142E5;Lo;0;L;;;;;N;;;;; +142E6;EGYPTIAN HIEROGLYPH-142E6;Lo;0;L;;;;;N;;;;; +142E7;EGYPTIAN HIEROGLYPH-142E7;Lo;0;L;;;;;N;;;;; +142E8;EGYPTIAN HIEROGLYPH-142E8;Lo;0;L;;;;;N;;;;; +142E9;EGYPTIAN HIEROGLYPH-142E9;Lo;0;L;;;;;N;;;;; +142EA;EGYPTIAN HIEROGLYPH-142EA;Lo;0;L;;;;;N;;;;; +142EB;EGYPTIAN HIEROGLYPH-142EB;Lo;0;L;;;;;N;;;;; +142EC;EGYPTIAN HIEROGLYPH-142EC;Lo;0;L;;;;;N;;;;; +142ED;EGYPTIAN HIEROGLYPH-142ED;Lo;0;L;;;;;N;;;;; +142EE;EGYPTIAN HIEROGLYPH-142EE;Lo;0;L;;;;;N;;;;; +142EF;EGYPTIAN HIEROGLYPH-142EF;Lo;0;L;;;;;N;;;;; +142F0;EGYPTIAN HIEROGLYPH-142F0;Lo;0;L;;;;;N;;;;; +142F1;EGYPTIAN HIEROGLYPH-142F1;Lo;0;L;;;;;N;;;;; +142F2;EGYPTIAN HIEROGLYPH-142F2;Lo;0;L;;;;;N;;;;; +142F3;EGYPTIAN HIEROGLYPH-142F3;Lo;0;L;;;;;N;;;;; +142F4;EGYPTIAN HIEROGLYPH-142F4;Lo;0;L;;;;;N;;;;; +142F5;EGYPTIAN HIEROGLYPH-142F5;Lo;0;L;;;;;N;;;;; +142F6;EGYPTIAN HIEROGLYPH-142F6;Lo;0;L;;;;;N;;;;; +142F7;EGYPTIAN HIEROGLYPH-142F7;Lo;0;L;;;;;N;;;;; +142F8;EGYPTIAN HIEROGLYPH-142F8;Lo;0;L;;;;;N;;;;; +142F9;EGYPTIAN HIEROGLYPH-142F9;Lo;0;L;;;;;N;;;;; +142FA;EGYPTIAN HIEROGLYPH-142FA;Lo;0;L;;;;;N;;;;; +142FB;EGYPTIAN HIEROGLYPH-142FB;Lo;0;L;;;;;N;;;;; +142FC;EGYPTIAN HIEROGLYPH-142FC;Lo;0;L;;;;;N;;;;; +142FD;EGYPTIAN HIEROGLYPH-142FD;Lo;0;L;;;;;N;;;;; +142FE;EGYPTIAN HIEROGLYPH-142FE;Lo;0;L;;;;;N;;;;; +142FF;EGYPTIAN HIEROGLYPH-142FF;Lo;0;L;;;;;N;;;;; +14300;EGYPTIAN HIEROGLYPH-14300;Lo;0;L;;;;;N;;;;; +14301;EGYPTIAN HIEROGLYPH-14301;Lo;0;L;;;;;N;;;;; +14302;EGYPTIAN HIEROGLYPH-14302;Lo;0;L;;;;;N;;;;; +14303;EGYPTIAN HIEROGLYPH-14303;Lo;0;L;;;;;N;;;;; +14304;EGYPTIAN HIEROGLYPH-14304;Lo;0;L;;;;;N;;;;; +14305;EGYPTIAN HIEROGLYPH-14305;Lo;0;L;;;;;N;;;;; +14306;EGYPTIAN HIEROGLYPH-14306;Lo;0;L;;;;;N;;;;; +14307;EGYPTIAN HIEROGLYPH-14307;Lo;0;L;;;;;N;;;;; +14308;EGYPTIAN HIEROGLYPH-14308;Lo;0;L;;;;;N;;;;; +14309;EGYPTIAN HIEROGLYPH-14309;Lo;0;L;;;;;N;;;;; +1430A;EGYPTIAN HIEROGLYPH-1430A;Lo;0;L;;;;;N;;;;; +1430B;EGYPTIAN HIEROGLYPH-1430B;Lo;0;L;;;;;N;;;;; +1430C;EGYPTIAN HIEROGLYPH-1430C;Lo;0;L;;;;;N;;;;; +1430D;EGYPTIAN HIEROGLYPH-1430D;Lo;0;L;;;;;N;;;;; +1430E;EGYPTIAN HIEROGLYPH-1430E;Lo;0;L;;;;;N;;;;; +1430F;EGYPTIAN HIEROGLYPH-1430F;Lo;0;L;;;;;N;;;;; +14310;EGYPTIAN HIEROGLYPH-14310;Lo;0;L;;;;;N;;;;; +14311;EGYPTIAN HIEROGLYPH-14311;Lo;0;L;;;;;N;;;;; +14312;EGYPTIAN HIEROGLYPH-14312;Lo;0;L;;;;;N;;;;; +14313;EGYPTIAN HIEROGLYPH-14313;Lo;0;L;;;;;N;;;;; +14314;EGYPTIAN HIEROGLYPH-14314;Lo;0;L;;;;;N;;;;; +14315;EGYPTIAN HIEROGLYPH-14315;Lo;0;L;;;;;N;;;;; +14316;EGYPTIAN HIEROGLYPH-14316;Lo;0;L;;;;;N;;;;; +14317;EGYPTIAN HIEROGLYPH-14317;Lo;0;L;;;;;N;;;;; +14318;EGYPTIAN HIEROGLYPH-14318;Lo;0;L;;;;;N;;;;; +14319;EGYPTIAN HIEROGLYPH-14319;Lo;0;L;;;;;N;;;;; +1431A;EGYPTIAN HIEROGLYPH-1431A;Lo;0;L;;;;;N;;;;; +1431B;EGYPTIAN HIEROGLYPH-1431B;Lo;0;L;;;;;N;;;;; +1431C;EGYPTIAN HIEROGLYPH-1431C;Lo;0;L;;;;;N;;;;; +1431D;EGYPTIAN HIEROGLYPH-1431D;Lo;0;L;;;;;N;;;;; +1431E;EGYPTIAN HIEROGLYPH-1431E;Lo;0;L;;;;;N;;;;; +1431F;EGYPTIAN HIEROGLYPH-1431F;Lo;0;L;;;;;N;;;;; +14320;EGYPTIAN HIEROGLYPH-14320;Lo;0;L;;;;;N;;;;; +14321;EGYPTIAN HIEROGLYPH-14321;Lo;0;L;;;;;N;;;;; +14322;EGYPTIAN HIEROGLYPH-14322;Lo;0;L;;;;;N;;;;; +14323;EGYPTIAN HIEROGLYPH-14323;Lo;0;L;;;;;N;;;;; +14324;EGYPTIAN HIEROGLYPH-14324;Lo;0;L;;;;;N;;;;; +14325;EGYPTIAN HIEROGLYPH-14325;Lo;0;L;;;;;N;;;;; +14326;EGYPTIAN HIEROGLYPH-14326;Lo;0;L;;;;;N;;;;; +14327;EGYPTIAN HIEROGLYPH-14327;Lo;0;L;;;;;N;;;;; +14328;EGYPTIAN HIEROGLYPH-14328;Lo;0;L;;;;;N;;;;; +14329;EGYPTIAN HIEROGLYPH-14329;Lo;0;L;;;;;N;;;;; +1432A;EGYPTIAN HIEROGLYPH-1432A;Lo;0;L;;;;;N;;;;; +1432B;EGYPTIAN HIEROGLYPH-1432B;Lo;0;L;;;;;N;;;;; +1432C;EGYPTIAN HIEROGLYPH-1432C;Lo;0;L;;;;;N;;;;; +1432D;EGYPTIAN HIEROGLYPH-1432D;Lo;0;L;;;;;N;;;;; +1432E;EGYPTIAN HIEROGLYPH-1432E;Lo;0;L;;;;;N;;;;; +1432F;EGYPTIAN HIEROGLYPH-1432F;Lo;0;L;;;;;N;;;;; +14330;EGYPTIAN HIEROGLYPH-14330;Lo;0;L;;;;;N;;;;; +14331;EGYPTIAN HIEROGLYPH-14331;Lo;0;L;;;;;N;;;;; +14332;EGYPTIAN HIEROGLYPH-14332;Lo;0;L;;;;;N;;;;; +14333;EGYPTIAN HIEROGLYPH-14333;Lo;0;L;;;;;N;;;;; +14334;EGYPTIAN HIEROGLYPH-14334;Lo;0;L;;;;;N;;;;; +14335;EGYPTIAN HIEROGLYPH-14335;Lo;0;L;;;;;N;;;;; +14336;EGYPTIAN HIEROGLYPH-14336;Lo;0;L;;;;;N;;;;; +14337;EGYPTIAN HIEROGLYPH-14337;Lo;0;L;;;;;N;;;;; +14338;EGYPTIAN HIEROGLYPH-14338;Lo;0;L;;;;;N;;;;; +14339;EGYPTIAN HIEROGLYPH-14339;Lo;0;L;;;;;N;;;;; +1433A;EGYPTIAN HIEROGLYPH-1433A;Lo;0;L;;;;;N;;;;; +1433B;EGYPTIAN HIEROGLYPH-1433B;Lo;0;L;;;;;N;;;;; +1433C;EGYPTIAN HIEROGLYPH-1433C;Lo;0;L;;;;;N;;;;; +1433D;EGYPTIAN HIEROGLYPH-1433D;Lo;0;L;;;;;N;;;;; +1433E;EGYPTIAN HIEROGLYPH-1433E;Lo;0;L;;;;;N;;;;; +1433F;EGYPTIAN HIEROGLYPH-1433F;Lo;0;L;;;;;N;;;;; +14340;EGYPTIAN HIEROGLYPH-14340;Lo;0;L;;;;;N;;;;; +14341;EGYPTIAN HIEROGLYPH-14341;Lo;0;L;;;;;N;;;;; +14342;EGYPTIAN HIEROGLYPH-14342;Lo;0;L;;;;;N;;;;; +14343;EGYPTIAN HIEROGLYPH-14343;Lo;0;L;;;;;N;;;;; +14344;EGYPTIAN HIEROGLYPH-14344;Lo;0;L;;;;;N;;;;; +14345;EGYPTIAN HIEROGLYPH-14345;Lo;0;L;;;;;N;;;;; +14346;EGYPTIAN HIEROGLYPH-14346;Lo;0;L;;;;;N;;;;; +14347;EGYPTIAN HIEROGLYPH-14347;Lo;0;L;;;;;N;;;;; +14348;EGYPTIAN HIEROGLYPH-14348;Lo;0;L;;;;;N;;;;; +14349;EGYPTIAN HIEROGLYPH-14349;Lo;0;L;;;;;N;;;;; +1434A;EGYPTIAN HIEROGLYPH-1434A;Lo;0;L;;;;;N;;;;; +1434B;EGYPTIAN HIEROGLYPH-1434B;Lo;0;L;;;;;N;;;;; +1434C;EGYPTIAN HIEROGLYPH-1434C;Lo;0;L;;;;;N;;;;; +1434D;EGYPTIAN HIEROGLYPH-1434D;Lo;0;L;;;;;N;;;;; +1434E;EGYPTIAN HIEROGLYPH-1434E;Lo;0;L;;;;;N;;;;; +1434F;EGYPTIAN HIEROGLYPH-1434F;Lo;0;L;;;;;N;;;;; +14350;EGYPTIAN HIEROGLYPH-14350;Lo;0;L;;;;;N;;;;; +14351;EGYPTIAN HIEROGLYPH-14351;Lo;0;L;;;;;N;;;;; +14352;EGYPTIAN HIEROGLYPH-14352;Lo;0;L;;;;;N;;;;; +14353;EGYPTIAN HIEROGLYPH-14353;Lo;0;L;;;;;N;;;;; +14354;EGYPTIAN HIEROGLYPH-14354;Lo;0;L;;;;;N;;;;; +14355;EGYPTIAN HIEROGLYPH-14355;Lo;0;L;;;;;N;;;;; +14356;EGYPTIAN HIEROGLYPH-14356;Lo;0;L;;;;;N;;;;; +14357;EGYPTIAN HIEROGLYPH-14357;Lo;0;L;;;;;N;;;;; +14358;EGYPTIAN HIEROGLYPH-14358;Lo;0;L;;;;;N;;;;; +14359;EGYPTIAN HIEROGLYPH-14359;Lo;0;L;;;;;N;;;;; +1435A;EGYPTIAN HIEROGLYPH-1435A;Lo;0;L;;;;;N;;;;; +1435B;EGYPTIAN HIEROGLYPH-1435B;Lo;0;L;;;;;N;;;;; +1435C;EGYPTIAN HIEROGLYPH-1435C;Lo;0;L;;;;;N;;;;; +1435D;EGYPTIAN HIEROGLYPH-1435D;Lo;0;L;;;;;N;;;;; +1435E;EGYPTIAN HIEROGLYPH-1435E;Lo;0;L;;;;;N;;;;; +1435F;EGYPTIAN HIEROGLYPH-1435F;Lo;0;L;;;;;N;;;;; +14360;EGYPTIAN HIEROGLYPH-14360;Lo;0;L;;;;;N;;;;; +14361;EGYPTIAN HIEROGLYPH-14361;Lo;0;L;;;;;N;;;;; +14362;EGYPTIAN HIEROGLYPH-14362;Lo;0;L;;;;;N;;;;; +14363;EGYPTIAN HIEROGLYPH-14363;Lo;0;L;;;;;N;;;;; +14364;EGYPTIAN HIEROGLYPH-14364;Lo;0;L;;;;;N;;;;; +14365;EGYPTIAN HIEROGLYPH-14365;Lo;0;L;;;;;N;;;;; +14366;EGYPTIAN HIEROGLYPH-14366;Lo;0;L;;;;;N;;;;; +14367;EGYPTIAN HIEROGLYPH-14367;Lo;0;L;;;;;N;;;;; +14368;EGYPTIAN HIEROGLYPH-14368;Lo;0;L;;;;;N;;;;; +14369;EGYPTIAN HIEROGLYPH-14369;Lo;0;L;;;;;N;;;;; +1436A;EGYPTIAN HIEROGLYPH-1436A;Lo;0;L;;;;;N;;;;; +1436B;EGYPTIAN HIEROGLYPH-1436B;Lo;0;L;;;;;N;;;;; +1436C;EGYPTIAN HIEROGLYPH-1436C;Lo;0;L;;;;;N;;;;; +1436D;EGYPTIAN HIEROGLYPH-1436D;Lo;0;L;;;;;N;;;;; +1436E;EGYPTIAN HIEROGLYPH-1436E;Lo;0;L;;;;;N;;;;; +1436F;EGYPTIAN HIEROGLYPH-1436F;Lo;0;L;;;;;N;;;;; +14370;EGYPTIAN HIEROGLYPH-14370;Lo;0;L;;;;;N;;;;; +14371;EGYPTIAN HIEROGLYPH-14371;Lo;0;L;;;;;N;;;;; +14372;EGYPTIAN HIEROGLYPH-14372;Lo;0;L;;;;;N;;;;; +14373;EGYPTIAN HIEROGLYPH-14373;Lo;0;L;;;;;N;;;;; +14374;EGYPTIAN HIEROGLYPH-14374;Lo;0;L;;;;;N;;;;; +14375;EGYPTIAN HIEROGLYPH-14375;Lo;0;L;;;;;N;;;;; +14376;EGYPTIAN HIEROGLYPH-14376;Lo;0;L;;;;;N;;;;; +14377;EGYPTIAN HIEROGLYPH-14377;Lo;0;L;;;;;N;;;;; +14378;EGYPTIAN HIEROGLYPH-14378;Lo;0;L;;;;;N;;;;; +14379;EGYPTIAN HIEROGLYPH-14379;Lo;0;L;;;;;N;;;;; +1437A;EGYPTIAN HIEROGLYPH-1437A;Lo;0;L;;;;;N;;;;; +1437B;EGYPTIAN HIEROGLYPH-1437B;Lo;0;L;;;;;N;;;;; +1437C;EGYPTIAN HIEROGLYPH-1437C;Lo;0;L;;;;;N;;;;; +1437D;EGYPTIAN HIEROGLYPH-1437D;Lo;0;L;;;;;N;;;;; +1437E;EGYPTIAN HIEROGLYPH-1437E;Lo;0;L;;;;;N;;;;; +1437F;EGYPTIAN HIEROGLYPH-1437F;Lo;0;L;;;;;N;;;;; +14380;EGYPTIAN HIEROGLYPH-14380;Lo;0;L;;;;;N;;;;; +14381;EGYPTIAN HIEROGLYPH-14381;Lo;0;L;;;;;N;;;;; +14382;EGYPTIAN HIEROGLYPH-14382;Lo;0;L;;;;;N;;;;; +14383;EGYPTIAN HIEROGLYPH-14383;Lo;0;L;;;;;N;;;;; +14384;EGYPTIAN HIEROGLYPH-14384;Lo;0;L;;;;;N;;;;; +14385;EGYPTIAN HIEROGLYPH-14385;Lo;0;L;;;;;N;;;;; +14386;EGYPTIAN HIEROGLYPH-14386;Lo;0;L;;;;;N;;;;; +14387;EGYPTIAN HIEROGLYPH-14387;Lo;0;L;;;;;N;;;;; +14388;EGYPTIAN HIEROGLYPH-14388;Lo;0;L;;;;;N;;;;; +14389;EGYPTIAN HIEROGLYPH-14389;Lo;0;L;;;;;N;;;;; +1438A;EGYPTIAN HIEROGLYPH-1438A;Lo;0;L;;;;;N;;;;; +1438B;EGYPTIAN HIEROGLYPH-1438B;Lo;0;L;;;;;N;;;;; +1438C;EGYPTIAN HIEROGLYPH-1438C;Lo;0;L;;;;;N;;;;; +1438D;EGYPTIAN HIEROGLYPH-1438D;Lo;0;L;;;;;N;;;;; +1438E;EGYPTIAN HIEROGLYPH-1438E;Lo;0;L;;;;;N;;;;; +1438F;EGYPTIAN HIEROGLYPH-1438F;Lo;0;L;;;;;N;;;;; +14390;EGYPTIAN HIEROGLYPH-14390;Lo;0;L;;;;;N;;;;; +14391;EGYPTIAN HIEROGLYPH-14391;Lo;0;L;;;;;N;;;;; +14392;EGYPTIAN HIEROGLYPH-14392;Lo;0;L;;;;;N;;;;; +14393;EGYPTIAN HIEROGLYPH-14393;Lo;0;L;;;;;N;;;;; +14394;EGYPTIAN HIEROGLYPH-14394;Lo;0;L;;;;;N;;;;; +14395;EGYPTIAN HIEROGLYPH-14395;Lo;0;L;;;;;N;;;;; +14396;EGYPTIAN HIEROGLYPH-14396;Lo;0;L;;;;;N;;;;; +14397;EGYPTIAN HIEROGLYPH-14397;Lo;0;L;;;;;N;;;;; +14398;EGYPTIAN HIEROGLYPH-14398;Lo;0;L;;;;;N;;;;; +14399;EGYPTIAN HIEROGLYPH-14399;Lo;0;L;;;;;N;;;;; +1439A;EGYPTIAN HIEROGLYPH-1439A;Lo;0;L;;;;;N;;;;; +1439B;EGYPTIAN HIEROGLYPH-1439B;Lo;0;L;;;;;N;;;;; +1439C;EGYPTIAN HIEROGLYPH-1439C;Lo;0;L;;;;;N;;;;; +1439D;EGYPTIAN HIEROGLYPH-1439D;Lo;0;L;;;;;N;;;;; +1439E;EGYPTIAN HIEROGLYPH-1439E;Lo;0;L;;;;;N;;;;; +1439F;EGYPTIAN HIEROGLYPH-1439F;Lo;0;L;;;;;N;;;;; +143A0;EGYPTIAN HIEROGLYPH-143A0;Lo;0;L;;;;;N;;;;; +143A1;EGYPTIAN HIEROGLYPH-143A1;Lo;0;L;;;;;N;;;;; +143A2;EGYPTIAN HIEROGLYPH-143A2;Lo;0;L;;;;;N;;;;; +143A3;EGYPTIAN HIEROGLYPH-143A3;Lo;0;L;;;;;N;;;;; +143A4;EGYPTIAN HIEROGLYPH-143A4;Lo;0;L;;;;;N;;;;; +143A5;EGYPTIAN HIEROGLYPH-143A5;Lo;0;L;;;;;N;;;;; +143A6;EGYPTIAN HIEROGLYPH-143A6;Lo;0;L;;;;;N;;;;; +143A7;EGYPTIAN HIEROGLYPH-143A7;Lo;0;L;;;;;N;;;;; +143A8;EGYPTIAN HIEROGLYPH-143A8;Lo;0;L;;;;;N;;;;; +143A9;EGYPTIAN HIEROGLYPH-143A9;Lo;0;L;;;;;N;;;;; +143AA;EGYPTIAN HIEROGLYPH-143AA;Lo;0;L;;;;;N;;;;; +143AB;EGYPTIAN HIEROGLYPH-143AB;Lo;0;L;;;;;N;;;;; +143AC;EGYPTIAN HIEROGLYPH-143AC;Lo;0;L;;;;;N;;;;; +143AD;EGYPTIAN HIEROGLYPH-143AD;Lo;0;L;;;;;N;;;;; +143AE;EGYPTIAN HIEROGLYPH-143AE;Lo;0;L;;;;;N;;;;; +143AF;EGYPTIAN HIEROGLYPH-143AF;Lo;0;L;;;;;N;;;;; +143B0;EGYPTIAN HIEROGLYPH-143B0;Lo;0;L;;;;;N;;;;; +143B1;EGYPTIAN HIEROGLYPH-143B1;Lo;0;L;;;;;N;;;;; +143B2;EGYPTIAN HIEROGLYPH-143B2;Lo;0;L;;;;;N;;;;; +143B3;EGYPTIAN HIEROGLYPH-143B3;Lo;0;L;;;;;N;;;;; +143B4;EGYPTIAN HIEROGLYPH-143B4;Lo;0;L;;;;;N;;;;; +143B5;EGYPTIAN HIEROGLYPH-143B5;Lo;0;L;;;;;N;;;;; +143B6;EGYPTIAN HIEROGLYPH-143B6;Lo;0;L;;;;;N;;;;; +143B7;EGYPTIAN HIEROGLYPH-143B7;Lo;0;L;;;;;N;;;;; +143B8;EGYPTIAN HIEROGLYPH-143B8;Lo;0;L;;;;;N;;;;; +143B9;EGYPTIAN HIEROGLYPH-143B9;Lo;0;L;;;;;N;;;;; +143BA;EGYPTIAN HIEROGLYPH-143BA;Lo;0;L;;;;;N;;;;; +143BB;EGYPTIAN HIEROGLYPH-143BB;Lo;0;L;;;;;N;;;;; +143BC;EGYPTIAN HIEROGLYPH-143BC;Lo;0;L;;;;;N;;;;; +143BD;EGYPTIAN HIEROGLYPH-143BD;Lo;0;L;;;;;N;;;;; +143BE;EGYPTIAN HIEROGLYPH-143BE;Lo;0;L;;;;;N;;;;; +143BF;EGYPTIAN HIEROGLYPH-143BF;Lo;0;L;;;;;N;;;;; +143C0;EGYPTIAN HIEROGLYPH-143C0;Lo;0;L;;;;;N;;;;; +143C1;EGYPTIAN HIEROGLYPH-143C1;Lo;0;L;;;;;N;;;;; +143C2;EGYPTIAN HIEROGLYPH-143C2;Lo;0;L;;;;;N;;;;; +143C3;EGYPTIAN HIEROGLYPH-143C3;Lo;0;L;;;;;N;;;;; +143C4;EGYPTIAN HIEROGLYPH-143C4;Lo;0;L;;;;;N;;;;; +143C5;EGYPTIAN HIEROGLYPH-143C5;Lo;0;L;;;;;N;;;;; +143C6;EGYPTIAN HIEROGLYPH-143C6;Lo;0;L;;;;;N;;;;; +143C7;EGYPTIAN HIEROGLYPH-143C7;Lo;0;L;;;;;N;;;;; +143C8;EGYPTIAN HIEROGLYPH-143C8;Lo;0;L;;;;;N;;;;; +143C9;EGYPTIAN HIEROGLYPH-143C9;Lo;0;L;;;;;N;;;;; +143CA;EGYPTIAN HIEROGLYPH-143CA;Lo;0;L;;;;;N;;;;; +143CB;EGYPTIAN HIEROGLYPH-143CB;Lo;0;L;;;;;N;;;;; +143CC;EGYPTIAN HIEROGLYPH-143CC;Lo;0;L;;;;;N;;;;; +143CD;EGYPTIAN HIEROGLYPH-143CD;Lo;0;L;;;;;N;;;;; +143CE;EGYPTIAN HIEROGLYPH-143CE;Lo;0;L;;;;;N;;;;; +143CF;EGYPTIAN HIEROGLYPH-143CF;Lo;0;L;;;;;N;;;;; +143D0;EGYPTIAN HIEROGLYPH-143D0;Lo;0;L;;;;;N;;;;; +143D1;EGYPTIAN HIEROGLYPH-143D1;Lo;0;L;;;;;N;;;;; +143D2;EGYPTIAN HIEROGLYPH-143D2;Lo;0;L;;;;;N;;;;; +143D3;EGYPTIAN HIEROGLYPH-143D3;Lo;0;L;;;;;N;;;;; +143D4;EGYPTIAN HIEROGLYPH-143D4;Lo;0;L;;;;;N;;;;; +143D5;EGYPTIAN HIEROGLYPH-143D5;Lo;0;L;;;;;N;;;;; +143D6;EGYPTIAN HIEROGLYPH-143D6;Lo;0;L;;;;;N;;;;; +143D7;EGYPTIAN HIEROGLYPH-143D7;Lo;0;L;;;;;N;;;;; +143D8;EGYPTIAN HIEROGLYPH-143D8;Lo;0;L;;;;;N;;;;; +143D9;EGYPTIAN HIEROGLYPH-143D9;Lo;0;L;;;;;N;;;;; +143DA;EGYPTIAN HIEROGLYPH-143DA;Lo;0;L;;;;;N;;;;; +143DB;EGYPTIAN HIEROGLYPH-143DB;Lo;0;L;;;;;N;;;;; +143DC;EGYPTIAN HIEROGLYPH-143DC;Lo;0;L;;;;;N;;;;; +143DD;EGYPTIAN HIEROGLYPH-143DD;Lo;0;L;;;;;N;;;;; +143DE;EGYPTIAN HIEROGLYPH-143DE;Lo;0;L;;;;;N;;;;; +143DF;EGYPTIAN HIEROGLYPH-143DF;Lo;0;L;;;;;N;;;;; +143E0;EGYPTIAN HIEROGLYPH-143E0;Lo;0;L;;;;;N;;;;; +143E1;EGYPTIAN HIEROGLYPH-143E1;Lo;0;L;;;;;N;;;;; +143E2;EGYPTIAN HIEROGLYPH-143E2;Lo;0;L;;;;;N;;;;; +143E3;EGYPTIAN HIEROGLYPH-143E3;Lo;0;L;;;;;N;;;;; +143E4;EGYPTIAN HIEROGLYPH-143E4;Lo;0;L;;;;;N;;;;; +143E5;EGYPTIAN HIEROGLYPH-143E5;Lo;0;L;;;;;N;;;;; +143E6;EGYPTIAN HIEROGLYPH-143E6;Lo;0;L;;;;;N;;;;; +143E7;EGYPTIAN HIEROGLYPH-143E7;Lo;0;L;;;;;N;;;;; +143E8;EGYPTIAN HIEROGLYPH-143E8;Lo;0;L;;;;;N;;;;; +143E9;EGYPTIAN HIEROGLYPH-143E9;Lo;0;L;;;;;N;;;;; +143EA;EGYPTIAN HIEROGLYPH-143EA;Lo;0;L;;;;;N;;;;; +143EB;EGYPTIAN HIEROGLYPH-143EB;Lo;0;L;;;;;N;;;;; +143EC;EGYPTIAN HIEROGLYPH-143EC;Lo;0;L;;;;;N;;;;; +143ED;EGYPTIAN HIEROGLYPH-143ED;Lo;0;L;;;;;N;;;;; +143EE;EGYPTIAN HIEROGLYPH-143EE;Lo;0;L;;;;;N;;;;; +143EF;EGYPTIAN HIEROGLYPH-143EF;Lo;0;L;;;;;N;;;;; +143F0;EGYPTIAN HIEROGLYPH-143F0;Lo;0;L;;;;;N;;;;; +143F1;EGYPTIAN HIEROGLYPH-143F1;Lo;0;L;;;;;N;;;;; +143F2;EGYPTIAN HIEROGLYPH-143F2;Lo;0;L;;;;;N;;;;; +143F3;EGYPTIAN HIEROGLYPH-143F3;Lo;0;L;;;;;N;;;;; +143F4;EGYPTIAN HIEROGLYPH-143F4;Lo;0;L;;;;;N;;;;; +143F5;EGYPTIAN HIEROGLYPH-143F5;Lo;0;L;;;;;N;;;;; +143F6;EGYPTIAN HIEROGLYPH-143F6;Lo;0;L;;;;;N;;;;; +143F7;EGYPTIAN HIEROGLYPH-143F7;Lo;0;L;;;;;N;;;;; +143F8;EGYPTIAN HIEROGLYPH-143F8;Lo;0;L;;;;;N;;;;; +143F9;EGYPTIAN HIEROGLYPH-143F9;Lo;0;L;;;;;N;;;;; +143FA;EGYPTIAN HIEROGLYPH-143FA;Lo;0;L;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -24771,6 +29053,64 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 14644;ANATOLIAN HIEROGLYPH A528;Lo;0;L;;;;;N;;;;; 14645;ANATOLIAN HIEROGLYPH A529;Lo;0;L;;;;;N;;;;; 14646;ANATOLIAN HIEROGLYPH A530;Lo;0;L;;;;;N;;;;; +16100;GURUNG KHEMA LETTER A;Lo;0;L;;;;;N;;;;; +16101;GURUNG KHEMA LETTER KA;Lo;0;L;;;;;N;;;;; +16102;GURUNG KHEMA LETTER KHA;Lo;0;L;;;;;N;;;;; +16103;GURUNG KHEMA LETTER GA;Lo;0;L;;;;;N;;;;; +16104;GURUNG KHEMA LETTER GHA;Lo;0;L;;;;;N;;;;; +16105;GURUNG KHEMA LETTER NGA;Lo;0;L;;;;;N;;;;; +16106;GURUNG KHEMA LETTER CA;Lo;0;L;;;;;N;;;;; +16107;GURUNG KHEMA LETTER CHA;Lo;0;L;;;;;N;;;;; +16108;GURUNG KHEMA LETTER JA;Lo;0;L;;;;;N;;;;; +16109;GURUNG KHEMA LETTER JHA;Lo;0;L;;;;;N;;;;; +1610A;GURUNG KHEMA LETTER HA;Lo;0;L;;;;;N;;;;; +1610B;GURUNG KHEMA LETTER TTA;Lo;0;L;;;;;N;;;;; +1610C;GURUNG KHEMA LETTER TTHA;Lo;0;L;;;;;N;;;;; +1610D;GURUNG KHEMA LETTER DDA;Lo;0;L;;;;;N;;;;; +1610E;GURUNG KHEMA LETTER DDHA;Lo;0;L;;;;;N;;;;; +1610F;GURUNG KHEMA LETTER VA;Lo;0;L;;;;;N;;;;; +16110;GURUNG KHEMA LETTER TA;Lo;0;L;;;;;N;;;;; +16111;GURUNG KHEMA LETTER THA;Lo;0;L;;;;;N;;;;; +16112;GURUNG KHEMA LETTER DA;Lo;0;L;;;;;N;;;;; +16113;GURUNG KHEMA LETTER DHA;Lo;0;L;;;;;N;;;;; +16114;GURUNG KHEMA LETTER NA;Lo;0;L;;;;;N;;;;; +16115;GURUNG KHEMA LETTER PA;Lo;0;L;;;;;N;;;;; +16116;GURUNG KHEMA LETTER PHA;Lo;0;L;;;;;N;;;;; +16117;GURUNG KHEMA LETTER BA;Lo;0;L;;;;;N;;;;; +16118;GURUNG KHEMA LETTER BHA;Lo;0;L;;;;;N;;;;; +16119;GURUNG KHEMA LETTER MA;Lo;0;L;;;;;N;;;;; +1611A;GURUNG KHEMA LETTER YA;Lo;0;L;;;;;N;;;;; +1611B;GURUNG KHEMA LETTER RA;Lo;0;L;;;;;N;;;;; +1611C;GURUNG KHEMA LETTER LA;Lo;0;L;;;;;N;;;;; +1611D;GURUNG KHEMA LETTER SA;Lo;0;L;;;;;N;;;;; +1611E;GURUNG KHEMA VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +1611F;GURUNG KHEMA VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +16120;GURUNG KHEMA VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +16121;GURUNG KHEMA VOWEL SIGN U;Mn;0;NSM;1611E 1611E;;;;N;;;;; +16122;GURUNG KHEMA VOWEL SIGN UU;Mn;0;NSM;1611E 16129;;;;N;;;;; +16123;GURUNG KHEMA VOWEL SIGN E;Mn;0;NSM;1611E 1611F;;;;N;;;;; +16124;GURUNG KHEMA VOWEL SIGN EE;Mn;0;NSM;16129 1611F;;;;N;;;;; +16125;GURUNG KHEMA VOWEL SIGN AI;Mn;0;NSM;1611E 16120;;;;N;;;;; +16126;GURUNG KHEMA VOWEL SIGN O;Mn;0;NSM;16121 1611F;;;;N;;;;; +16127;GURUNG KHEMA VOWEL SIGN OO;Mn;0;NSM;16122 1611F;;;;N;;;;; +16128;GURUNG KHEMA VOWEL SIGN AU;Mn;0;NSM;16121 16120;;;;N;;;;; +16129;GURUNG KHEMA VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;; +1612A;GURUNG KHEMA CONSONANT SIGN MEDIAL YA;Mc;0;L;;;;;N;;;;; +1612B;GURUNG KHEMA CONSONANT SIGN MEDIAL VA;Mc;0;L;;;;;N;;;;; +1612C;GURUNG KHEMA CONSONANT SIGN MEDIAL HA;Mc;0;L;;;;;N;;;;; +1612D;GURUNG KHEMA SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +1612E;GURUNG KHEMA CONSONANT SIGN MEDIAL RA;Mn;0;NSM;;;;;N;;;;; +1612F;GURUNG KHEMA SIGN THOLHOMA;Mn;9;NSM;;;;;N;;;;; +16130;GURUNG KHEMA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +16131;GURUNG KHEMA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +16132;GURUNG KHEMA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +16133;GURUNG KHEMA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +16134;GURUNG KHEMA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +16135;GURUNG KHEMA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +16136;GURUNG KHEMA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +16137;GURUNG KHEMA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +16138;GURUNG KHEMA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +16139;GURUNG KHEMA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 16800;BAMUM LETTER PHASE-A NGKUE MFON;Lo;0;L;;;;;N;;;;; 16801;BAMUM LETTER PHASE-A GBIEE FON;Lo;0;L;;;;;N;;;;; 16802;BAMUM LETTER PHASE-A PON MFON PIPAEMGBIEE;Lo;0;L;;;;;N;;;;; @@ -25635,6 +29975,64 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16B8D;PAHAWH HMONG CLAN SIGN TSWB;Lo;0;L;;;;;N;;;;; 16B8E;PAHAWH HMONG CLAN SIGN KWM;Lo;0;L;;;;;N;;;;; 16B8F;PAHAWH HMONG CLAN SIGN VWJ;Lo;0;L;;;;;N;;;;; +16D40;KIRAT RAI SIGN ANUSVARA;Lm;0;L;;;;;N;;;;; +16D41;KIRAT RAI SIGN TONPI;Lm;0;L;;;;;N;;;;; +16D42;KIRAT RAI SIGN VISARGA;Lm;0;L;;;;;N;;;;; +16D43;KIRAT RAI LETTER A;Lo;0;L;;;;;N;;;;; +16D44;KIRAT RAI LETTER KA;Lo;0;L;;;;;N;;;;; +16D45;KIRAT RAI LETTER KHA;Lo;0;L;;;;;N;;;;; +16D46;KIRAT RAI LETTER GA;Lo;0;L;;;;;N;;;;; +16D47;KIRAT RAI LETTER GHA;Lo;0;L;;;;;N;;;;; +16D48;KIRAT RAI LETTER NGA;Lo;0;L;;;;;N;;;;; +16D49;KIRAT RAI LETTER CA;Lo;0;L;;;;;N;;;;; +16D4A;KIRAT RAI LETTER CHA;Lo;0;L;;;;;N;;;;; +16D4B;KIRAT RAI LETTER JA;Lo;0;L;;;;;N;;;;; +16D4C;KIRAT RAI LETTER JHA;Lo;0;L;;;;;N;;;;; +16D4D;KIRAT RAI LETTER NYA;Lo;0;L;;;;;N;;;;; +16D4E;KIRAT RAI LETTER TTA;Lo;0;L;;;;;N;;;;; +16D4F;KIRAT RAI LETTER TTHA;Lo;0;L;;;;;N;;;;; +16D50;KIRAT RAI LETTER DDA;Lo;0;L;;;;;N;;;;; +16D51;KIRAT RAI LETTER DDHA;Lo;0;L;;;;;N;;;;; +16D52;KIRAT RAI LETTER TA;Lo;0;L;;;;;N;;;;; +16D53;KIRAT RAI LETTER THA;Lo;0;L;;;;;N;;;;; +16D54;KIRAT RAI LETTER DA;Lo;0;L;;;;;N;;;;; +16D55;KIRAT RAI LETTER DHA;Lo;0;L;;;;;N;;;;; +16D56;KIRAT RAI LETTER NA;Lo;0;L;;;;;N;;;;; +16D57;KIRAT RAI LETTER PA;Lo;0;L;;;;;N;;;;; +16D58;KIRAT RAI LETTER PHA;Lo;0;L;;;;;N;;;;; +16D59;KIRAT RAI LETTER BA;Lo;0;L;;;;;N;;;;; +16D5A;KIRAT RAI LETTER BHA;Lo;0;L;;;;;N;;;;; +16D5B;KIRAT RAI LETTER MA;Lo;0;L;;;;;N;;;;; +16D5C;KIRAT RAI LETTER YA;Lo;0;L;;;;;N;;;;; +16D5D;KIRAT RAI LETTER RA;Lo;0;L;;;;;N;;;;; +16D5E;KIRAT RAI LETTER LA;Lo;0;L;;;;;N;;;;; +16D5F;KIRAT RAI LETTER VA;Lo;0;L;;;;;N;;;;; +16D60;KIRAT RAI LETTER SA;Lo;0;L;;;;;N;;;;; +16D61;KIRAT RAI LETTER SHA;Lo;0;L;;;;;N;;;;; +16D62;KIRAT RAI LETTER HA;Lo;0;L;;;;;N;;;;; +16D63;KIRAT RAI VOWEL SIGN AA;Lo;0;L;;;;;N;;;;; +16D64;KIRAT RAI VOWEL SIGN I;Lo;0;L;;;;;N;;;;; +16D65;KIRAT RAI VOWEL SIGN U;Lo;0;L;;;;;N;;;;; +16D66;KIRAT RAI VOWEL SIGN UE;Lo;0;L;;;;;N;;;;; +16D67;KIRAT RAI VOWEL SIGN E;Lo;0;L;;;;;N;;;;; +16D68;KIRAT RAI VOWEL SIGN AI;Lo;0;L;16D67 16D67;;;;N;;;;; +16D69;KIRAT RAI VOWEL SIGN O;Lo;0;L;16D63 16D67;;;;N;;;;; +16D6A;KIRAT RAI VOWEL SIGN AU;Lo;0;L;16D69 16D67;;;;N;;;;; +16D6B;KIRAT RAI SIGN VIRAMA;Lm;0;L;;;;;N;;;;; +16D6C;KIRAT RAI SIGN SAAT;Lm;0;L;;;;;N;;;;; +16D6D;KIRAT RAI SIGN YUPI;Po;0;L;;;;;N;;;;; +16D6E;KIRAT RAI DANDA;Po;0;L;;;;;N;;;;; +16D6F;KIRAT RAI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +16D70;KIRAT RAI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +16D71;KIRAT RAI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +16D72;KIRAT RAI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +16D73;KIRAT RAI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +16D74;KIRAT RAI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +16D75;KIRAT RAI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +16D76;KIRAT RAI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +16D77;KIRAT RAI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +16D78;KIRAT RAI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +16D79;KIRAT RAI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 16E40;MEDEFAIDRIN CAPITAL LETTER M;Lu;0;L;;;;;N;;;;16E60; 16E41;MEDEFAIDRIN CAPITAL LETTER S;Lu;0;L;;;;;N;;;;16E61; 16E42;MEDEFAIDRIN CAPITAL LETTER V;Lu;0;L;;;;;N;;;;16E62; @@ -27122,6 +31520,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 18CD3;KHITAN SMALL SCRIPT CHARACTER-18CD3;Lo;0;L;;;;;N;;;;; 18CD4;KHITAN SMALL SCRIPT CHARACTER-18CD4;Lo;0;L;;;;;N;;;;; 18CD5;KHITAN SMALL SCRIPT CHARACTER-18CD5;Lo;0;L;;;;;N;;;;; +18CFF;KHITAN SMALL SCRIPT CHARACTER-18CFF;Lo;0;L;;;;;N;;;;; 18D00;;Lo;0;L;;;;;N;;;;; 18D08;;Lo;0;L;;;;;N;;;;; 1AFF0;KATAKANA LETTER MINNAN TONE-2;Lm;0;L;;;;;N;;;;; @@ -27980,6 +32379,692 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1BCA1;SHORTHAND FORMAT CONTINUING OVERLAP;Cf;0;BN;;;;;N;;;;; 1BCA2;SHORTHAND FORMAT DOWN STEP;Cf;0;BN;;;;;N;;;;; 1BCA3;SHORTHAND FORMAT UP STEP;Cf;0;BN;;;;;N;;;;; +1CC00;UP-POINTING GO-KART;So;0;ON;;;;;N;;;;; +1CC01;RIGHT-POINTING GO-KART;So;0;ON;;;;;N;;;;; +1CC02;LEFT-POINTING STICK FIGURE;So;0;ON;;;;;N;;;;; +1CC03;RIGHT-POINTING STICK FIGURE;So;0;ON;;;;;N;;;;; +1CC04;DOWN-POINTING STICK FIGURE;So;0;ON;;;;;N;;;;; +1CC05;LOWER HORIZONTAL RULER SEGMENT;So;0;ON;;;;;N;;;;; +1CC06;RIGHT VERTICAL RULER SEGMENT;So;0;ON;;;;;N;;;;; +1CC07;LOWER RIGHT RULER SEGMENT;So;0;ON;;;;;N;;;;; +1CC08;ANTENNA;So;0;ON;;;;;N;;;;; +1CC09;HORIZONTAL RESISTOR SEGMENT;So;0;ON;;;;;N;;;;; +1CC0A;VERTICAL RESISTOR SEGMENT;So;0;ON;;;;;N;;;;; +1CC0B;LEFT THIRD INDUCTOR;So;0;ON;;;;;N;;;;; +1CC0C;MIDDLE THIRD INDUCTOR;So;0;ON;;;;;N;;;;; +1CC0D;RIGHT THIRD INDUCTOR;So;0;ON;;;;;N;;;;; +1CC0E;LEFT-POINTING DIODE;So;0;ON;;;;;N;;;;; +1CC0F;RIGHT-POINTING DIODE;So;0;ON;;;;;N;;;;; +1CC10;NPN TRANSISTOR;So;0;ON;;;;;N;;;;; +1CC11;PNP TRANSISTOR;So;0;ON;;;;;N;;;;; +1CC12;RECEPTACLE;So;0;ON;;;;;N;;;;; +1CC13;HORIZONTAL CAPACITOR;So;0;ON;;;;;N;;;;; +1CC14;VERTICAL CAPACITOR;So;0;ON;;;;;N;;;;; +1CC15;LOGIC GATE OR;So;0;ON;;;;;N;;;;; +1CC16;LOGIC GATE AND;So;0;ON;;;;;N;;;;; +1CC17;LOGIC GATE INVERTED INPUTS;So;0;ON;;;;;N;;;;; +1CC18;LOGIC GATE INVERTED OUTPUT;So;0;ON;;;;;N;;;;; +1CC19;LOGIC GATE BUFFER;So;0;ON;;;;;N;;;;; +1CC1A;LOGIC GATE BUFFER WITH INVERTED INPUT;So;0;ON;;;;;N;;;;; +1CC1B;BOX DRAWINGS LIGHT HORIZONTAL AND UPPER RIGHT;So;0;ON;;;;;N;;;;; +1CC1C;BOX DRAWINGS LIGHT HORIZONTAL AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CC1D;BOX DRAWINGS LIGHT TOP AND UPPER LEFT;So;0;ON;;;;;N;;;;; +1CC1E;BOX DRAWINGS LIGHT BOTTOM AND LOWER LEFT;So;0;ON;;;;;N;;;;; +1CC1F;BOX DRAWINGS DOUBLE DIAGONAL UPPER RIGHT TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1CC20;BOX DRAWINGS DOUBLE DIAGONAL UPPER LEFT TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CC21;SEPARATED BLOCK QUADRANT-1;So;0;ON;;;;;N;;;;; +1CC22;SEPARATED BLOCK QUADRANT-2;So;0;ON;;;;;N;;;;; +1CC23;SEPARATED BLOCK QUADRANT-12;So;0;ON;;;;;N;;;;; +1CC24;SEPARATED BLOCK QUADRANT-3;So;0;ON;;;;;N;;;;; +1CC25;SEPARATED BLOCK QUADRANT-13;So;0;ON;;;;;N;;;;; +1CC26;SEPARATED BLOCK QUADRANT-23;So;0;ON;;;;;N;;;;; +1CC27;SEPARATED BLOCK QUADRANT-123;So;0;ON;;;;;N;;;;; +1CC28;SEPARATED BLOCK QUADRANT-4;So;0;ON;;;;;N;;;;; +1CC29;SEPARATED BLOCK QUADRANT-14;So;0;ON;;;;;N;;;;; +1CC2A;SEPARATED BLOCK QUADRANT-24;So;0;ON;;;;;N;;;;; +1CC2B;SEPARATED BLOCK QUADRANT-124;So;0;ON;;;;;N;;;;; +1CC2C;SEPARATED BLOCK QUADRANT-34;So;0;ON;;;;;N;;;;; +1CC2D;SEPARATED BLOCK QUADRANT-134;So;0;ON;;;;;N;;;;; +1CC2E;SEPARATED BLOCK QUADRANT-234;So;0;ON;;;;;N;;;;; +1CC2F;SEPARATED BLOCK QUADRANT-1234;So;0;ON;;;;;N;;;;; +1CC30;UPPER LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC31;UPPER CENTRE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC32;UPPER CENTRE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC33;UPPER RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC34;UPPER MIDDLE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC35;UPPER LEFT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC36;UPPER RIGHT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC37;UPPER MIDDLE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC38;LOWER MIDDLE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC39;LOWER LEFT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC3A;LOWER RIGHT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC3B;LOWER MIDDLE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3C;LOWER LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3D;LOWER CENTRE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3E;LOWER CENTRE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3F;LOWER RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC40;SPARSE HORIZONTAL FILL;So;0;ON;;;;;N;;;;; +1CC41;SPARSE VERTICAL FILL;So;0;ON;;;;;N;;;;; +1CC42;ORTHOGONAL CROSSHATCH FILL;So;0;ON;;;;;N;;;;; +1CC43;DIAGONAL CROSSHATCH FILL;So;0;ON;;;;;N;;;;; +1CC44;DENSE VERTICAL FILL;So;0;ON;;;;;N;;;;; +1CC45;DENSE HORIZONTAL FILL;So;0;ON;;;;;N;;;;; +1CC46;SPECKLE FILL FRAME-1;So;0;ON;;;;;N;;;;; +1CC47;SPECKLE FILL FRAME-2;So;0;ON;;;;;N;;;;; +1CC48;LEFT-FACING BASSINET;So;0;ON;;;;;N;;;;; +1CC49;RIGHT-FACING BASSINET;So;0;ON;;;;;N;;;;; +1CC4A;FLYING SAUCER WITH BEAMS;So;0;ON;;;;;N;;;;; +1CC4B;FLYING SAUCER WITHOUT BEAMS;So;0;ON;;;;;N;;;;; +1CC4C;ALIEN MONSTER OPEN JAWS;So;0;ON;;;;;N;;;;; +1CC4D;ALIEN MONSTER CLOSED JAWS;So;0;ON;;;;;N;;;;; +1CC4E;ALIEN SQUID OPEN TENTACLES;So;0;ON;;;;;N;;;;; +1CC4F;ALIEN SQUID CLOSED TENTACLES;So;0;ON;;;;;N;;;;; +1CC50;ALIEN CRAB STEPPING RIGHT;So;0;ON;;;;;N;;;;; +1CC51;ALIEN CRAB STEPPING LEFT;So;0;ON;;;;;N;;;;; +1CC52;ALIEN SPIDER CROUCHING;So;0;ON;;;;;N;;;;; +1CC53;ALIEN SPIDER SPREAD;So;0;ON;;;;;N;;;;; +1CC54;ALIEN MONSTER STEP-1;So;0;ON;;;;;N;;;;; +1CC55;ALIEN MONSTER STEP-2;So;0;ON;;;;;N;;;;; +1CC56;LEFT-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC57;UP-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC58;RIGHT-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC59;DOWN-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC5A;TOP HALF LEFT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5B;TOP HALF FORWARD-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5C;TOP HALF RIGHT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5D;BOTTOM HALF LEFT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5E;BOTTOM HALF FORWARD-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5F;BOTTOM HALF RIGHT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC60;LEFT-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC61;UP-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC62;RIGHT-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC63;DOWN-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC64;MUSHROOM CLOUD;So;0;ON;;;;;N;;;;; +1CC65;LEFT-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC66;UP-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC67;RIGHT-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC68;DOWN-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC69;EIGHT RAYS INWARD;So;0;ON;;;;;N;;;;; +1CC6A;EIGHT RAYS OUTWARD;So;0;ON;;;;;N;;;;; +1CC6B;BLACK LARGE CIRCLE MINUS LEFT QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6C;BLACK LARGE CIRCLE MINUS UPPER QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6D;BLACK LARGE CIRCLE MINUS RIGHT QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6E;BLACK LARGE CIRCLE MINUS LOWER QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6F;BLACK NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1CC70;LEFT-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC71;UP-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC72;RIGHT-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC73;DOWN-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC74;LEFT-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC75;UP-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC76;RIGHT-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC77;DOWN-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC78;LEFT-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC79;UP-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC7A;RIGHT-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC7B;DOWN-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC7C;SQUARE SPIRAL FROM TOP LEFT;So;0;ON;;;;;N;;;;; +1CC7D;SQUARE SPIRAL FROM TOP RIGHT;So;0;ON;;;;;N;;;;; +1CC7E;SQUARE SPIRAL FROM BOTTOM RIGHT;So;0;ON;;;;;N;;;;; +1CC7F;SQUARE SPIRAL FROM BOTTOM LEFT;So;0;ON;;;;;N;;;;; +1CC80;STRIPED LEFT-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC81;STRIPED UP-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC82;STRIPED RIGHT-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC83;STRIPED DOWN-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC84;VERTICAL LADDER;So;0;ON;;;;;N;;;;; +1CC85;HORIZONTAL LADDER;So;0;ON;;;;;N;;;;; +1CC86;WHITE LOWER LEFT POINTER;So;0;ON;;;;;N;;;;; +1CC87;WHITE LOWER RIGHT POINTER;So;0;ON;;;;;N;;;;; +1CC88;TWO RINGS ALIGNED HORIZONTALLY;So;0;ON;;;;;N;;;;; +1CC89;SQUARE FOUR CORNER SALTIRES;So;0;ON;;;;;N;;;;; +1CC8A;SQUARE FOUR CORNER DIAGONALS;So;0;ON;;;;;N;;;;; +1CC8B;SQUARE FOUR CORNER BLACK TRIANGLES;So;0;ON;;;;;N;;;;; +1CC8C;SQUARE APERTURE;So;0;ON;;;;;N;;;;; +1CC8D;INVERSE BLACK DIAMOND;So;0;ON;;;;;N;;;;; +1CC8E;LEFT AND UPPER ONE EIGHTH BLOCK CONTAINING BLACK SMALL SQUARE;So;0;ON;;;;;N;;;;; +1CC8F;INVERSE BLACK SMALL SQUARE;So;0;ON;;;;;N;;;;; +1CC90;VERTICAL LINE WITH FOUR TICK MARKS;So;0;ON;;;;;N;;;;; +1CC91;HORIZONTAL LINE WITH FOUR TICK MARKS;So;0;ON;;;;;N;;;;; +1CC92;LEFT-FACING FISH;So;0;ON;;;;;N;;;;; +1CC93;RIGHT-FACING FISH;So;0;ON;;;;;N;;;;; +1CC94;LEFT-FACING FISH WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC95;RIGHT-FACING FISH WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC96;FLAPPING BIRD;So;0;ON;;;;;N;;;;; +1CC97;LEFT-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC98;UP-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC99;RIGHT-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC9A;DOWN-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC9B;HORIZONTAL RACING CAR;So;0;ON;;;;;N;;;;; +1CC9C;VERTICAL RACING CAR;So;0;ON;;;;;N;;;;; +1CC9D;VERTICAL GO-KART;So;0;ON;;;;;N;;;;; +1CC9E;LEFT-POINTING TANK;So;0;ON;;;;;N;;;;; +1CC9F;RIGHT-POINTING TANK;So;0;ON;;;;;N;;;;; +1CCA0;LEFT-POINTING ROCKET BOOSTER;So;0;ON;;;;;N;;;;; +1CCA1;RIGHT-POINTING ROCKET BOOSTER;So;0;ON;;;;;N;;;;; +1CCA2;LEFT-POINTING ROLLER COASTER CAR;So;0;ON;;;;;N;;;;; +1CCA3;RIGHT-POINTING ROLLER COASTER CAR;So;0;ON;;;;;N;;;;; +1CCA4;LEFT HALF FLYING SAUCER;So;0;ON;;;;;N;;;;; +1CCA5;RIGHT HALF FLYING SAUCER;So;0;ON;;;;;N;;;;; +1CCA6;UPPER LEFT QUADRANT FACE WITH OPEN EYES;So;0;ON;;;;;N;;;;; +1CCA7;UPPER RIGHT QUADRANT FACE WITH OPEN EYES;So;0;ON;;;;;N;;;;; +1CCA8;UPPER LEFT QUADRANT FACE WITH CLOSED EYES;So;0;ON;;;;;N;;;;; +1CCA9;UPPER RIGHT QUADRANT FACE WITH CLOSED EYES;So;0;ON;;;;;N;;;;; +1CCAA;LOWER LEFT QUADRANT SMILING FACE;So;0;ON;;;;;N;;;;; +1CCAB;LOWER RIGHT QUADRANT SMILING FACE;So;0;ON;;;;;N;;;;; +1CCAC;LOWER LEFT QUADRANT NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1CCAD;LOWER RIGHT QUADRANT NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1CCAE;LOWER LEFT QUADRANT FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CCAF;LOWER RIGHT QUADRANT FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CCB0;LOWER LEFT QUADRANT FROWNING FACE;So;0;ON;;;;;N;;;;; +1CCB1;LOWER RIGHT QUADRANT FROWNING FACE;So;0;ON;;;;;N;;;;; +1CCB2;UPPER LEFT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB3;UPPER RIGHT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB4;LOWER LEFT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB5;LOWER RIGHT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB6;UPPER LEFT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCB7;UPPER RIGHT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCB8;LOWER LEFT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCB9;LOWER RIGHT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCBA;UPPER LEFT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBB;UPPER RIGHT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBC;LOWER LEFT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBD;LOWER RIGHT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBE;UPPER LEFT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCBF;UPPER RIGHT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCC0;LOWER LEFT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCC1;LOWER RIGHT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCC2;UPPER LEFT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC3;UPPER RIGHT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC4;LOWER LEFT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC5;LOWER RIGHT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC6;UPPER LEFT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCC7;UPPER RIGHT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCC8;LOWER LEFT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCC9;LOWER RIGHT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCCA;UPPER LEFT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCB;UPPER RIGHT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCC;LOWER LEFT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCD;LOWER RIGHT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCE;UPPER LEFT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCCF;UPPER RIGHT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCD0;LOWER LEFT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCD1;LOWER RIGHT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCD2;UPPER LEFT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD3;UPPER RIGHT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD4;LOWER LEFT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD5;LOWER RIGHT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD6;OUTLINED LATIN CAPITAL LETTER A;So;0;L; 0041;;;;N;;;;; +1CCD7;OUTLINED LATIN CAPITAL LETTER B;So;0;L; 0042;;;;N;;;;; +1CCD8;OUTLINED LATIN CAPITAL LETTER C;So;0;L; 0043;;;;N;;;;; +1CCD9;OUTLINED LATIN CAPITAL LETTER D;So;0;L; 0044;;;;N;;;;; +1CCDA;OUTLINED LATIN CAPITAL LETTER E;So;0;L; 0045;;;;N;;;;; +1CCDB;OUTLINED LATIN CAPITAL LETTER F;So;0;L; 0046;;;;N;;;;; +1CCDC;OUTLINED LATIN CAPITAL LETTER G;So;0;L; 0047;;;;N;;;;; +1CCDD;OUTLINED LATIN CAPITAL LETTER H;So;0;L; 0048;;;;N;;;;; +1CCDE;OUTLINED LATIN CAPITAL LETTER I;So;0;L; 0049;;;;N;;;;; +1CCDF;OUTLINED LATIN CAPITAL LETTER J;So;0;L; 004A;;;;N;;;;; +1CCE0;OUTLINED LATIN CAPITAL LETTER K;So;0;L; 004B;;;;N;;;;; +1CCE1;OUTLINED LATIN CAPITAL LETTER L;So;0;L; 004C;;;;N;;;;; +1CCE2;OUTLINED LATIN CAPITAL LETTER M;So;0;L; 004D;;;;N;;;;; +1CCE3;OUTLINED LATIN CAPITAL LETTER N;So;0;L; 004E;;;;N;;;;; +1CCE4;OUTLINED LATIN CAPITAL LETTER O;So;0;L; 004F;;;;N;;;;; +1CCE5;OUTLINED LATIN CAPITAL LETTER P;So;0;L; 0050;;;;N;;;;; +1CCE6;OUTLINED LATIN CAPITAL LETTER Q;So;0;L; 0051;;;;N;;;;; +1CCE7;OUTLINED LATIN CAPITAL LETTER R;So;0;L; 0052;;;;N;;;;; +1CCE8;OUTLINED LATIN CAPITAL LETTER S;So;0;L; 0053;;;;N;;;;; +1CCE9;OUTLINED LATIN CAPITAL LETTER T;So;0;L; 0054;;;;N;;;;; +1CCEA;OUTLINED LATIN CAPITAL LETTER U;So;0;L; 0055;;;;N;;;;; +1CCEB;OUTLINED LATIN CAPITAL LETTER V;So;0;L; 0056;;;;N;;;;; +1CCEC;OUTLINED LATIN CAPITAL LETTER W;So;0;L; 0057;;;;N;;;;; +1CCED;OUTLINED LATIN CAPITAL LETTER X;So;0;L; 0058;;;;N;;;;; +1CCEE;OUTLINED LATIN CAPITAL LETTER Y;So;0;L; 0059;;;;N;;;;; +1CCEF;OUTLINED LATIN CAPITAL LETTER Z;So;0;L; 005A;;;;N;;;;; +1CCF0;OUTLINED DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1CCF1;OUTLINED DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1CCF2;OUTLINED DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1CCF3;OUTLINED DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1CCF4;OUTLINED DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1CCF5;OUTLINED DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1CCF6;OUTLINED DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1CCF7;OUTLINED DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1CCF8;OUTLINED DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1CCF9;OUTLINED DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1CD00;BLOCK OCTANT-3;So;0;ON;;;;;N;;;;; +1CD01;BLOCK OCTANT-23;So;0;ON;;;;;N;;;;; +1CD02;BLOCK OCTANT-123;So;0;ON;;;;;N;;;;; +1CD03;BLOCK OCTANT-4;So;0;ON;;;;;N;;;;; +1CD04;BLOCK OCTANT-14;So;0;ON;;;;;N;;;;; +1CD05;BLOCK OCTANT-124;So;0;ON;;;;;N;;;;; +1CD06;BLOCK OCTANT-34;So;0;ON;;;;;N;;;;; +1CD07;BLOCK OCTANT-134;So;0;ON;;;;;N;;;;; +1CD08;BLOCK OCTANT-234;So;0;ON;;;;;N;;;;; +1CD09;BLOCK OCTANT-5;So;0;ON;;;;;N;;;;; +1CD0A;BLOCK OCTANT-15;So;0;ON;;;;;N;;;;; +1CD0B;BLOCK OCTANT-25;So;0;ON;;;;;N;;;;; +1CD0C;BLOCK OCTANT-125;So;0;ON;;;;;N;;;;; +1CD0D;BLOCK OCTANT-135;So;0;ON;;;;;N;;;;; +1CD0E;BLOCK OCTANT-235;So;0;ON;;;;;N;;;;; +1CD0F;BLOCK OCTANT-1235;So;0;ON;;;;;N;;;;; +1CD10;BLOCK OCTANT-45;So;0;ON;;;;;N;;;;; +1CD11;BLOCK OCTANT-145;So;0;ON;;;;;N;;;;; +1CD12;BLOCK OCTANT-245;So;0;ON;;;;;N;;;;; +1CD13;BLOCK OCTANT-1245;So;0;ON;;;;;N;;;;; +1CD14;BLOCK OCTANT-345;So;0;ON;;;;;N;;;;; +1CD15;BLOCK OCTANT-1345;So;0;ON;;;;;N;;;;; +1CD16;BLOCK OCTANT-2345;So;0;ON;;;;;N;;;;; +1CD17;BLOCK OCTANT-12345;So;0;ON;;;;;N;;;;; +1CD18;BLOCK OCTANT-6;So;0;ON;;;;;N;;;;; +1CD19;BLOCK OCTANT-16;So;0;ON;;;;;N;;;;; +1CD1A;BLOCK OCTANT-26;So;0;ON;;;;;N;;;;; +1CD1B;BLOCK OCTANT-126;So;0;ON;;;;;N;;;;; +1CD1C;BLOCK OCTANT-36;So;0;ON;;;;;N;;;;; +1CD1D;BLOCK OCTANT-136;So;0;ON;;;;;N;;;;; +1CD1E;BLOCK OCTANT-236;So;0;ON;;;;;N;;;;; +1CD1F;BLOCK OCTANT-1236;So;0;ON;;;;;N;;;;; +1CD20;BLOCK OCTANT-146;So;0;ON;;;;;N;;;;; +1CD21;BLOCK OCTANT-246;So;0;ON;;;;;N;;;;; +1CD22;BLOCK OCTANT-1246;So;0;ON;;;;;N;;;;; +1CD23;BLOCK OCTANT-346;So;0;ON;;;;;N;;;;; +1CD24;BLOCK OCTANT-1346;So;0;ON;;;;;N;;;;; +1CD25;BLOCK OCTANT-2346;So;0;ON;;;;;N;;;;; +1CD26;BLOCK OCTANT-12346;So;0;ON;;;;;N;;;;; +1CD27;BLOCK OCTANT-56;So;0;ON;;;;;N;;;;; +1CD28;BLOCK OCTANT-156;So;0;ON;;;;;N;;;;; +1CD29;BLOCK OCTANT-256;So;0;ON;;;;;N;;;;; +1CD2A;BLOCK OCTANT-1256;So;0;ON;;;;;N;;;;; +1CD2B;BLOCK OCTANT-356;So;0;ON;;;;;N;;;;; +1CD2C;BLOCK OCTANT-1356;So;0;ON;;;;;N;;;;; +1CD2D;BLOCK OCTANT-2356;So;0;ON;;;;;N;;;;; +1CD2E;BLOCK OCTANT-12356;So;0;ON;;;;;N;;;;; +1CD2F;BLOCK OCTANT-456;So;0;ON;;;;;N;;;;; +1CD30;BLOCK OCTANT-1456;So;0;ON;;;;;N;;;;; +1CD31;BLOCK OCTANT-2456;So;0;ON;;;;;N;;;;; +1CD32;BLOCK OCTANT-12456;So;0;ON;;;;;N;;;;; +1CD33;BLOCK OCTANT-3456;So;0;ON;;;;;N;;;;; +1CD34;BLOCK OCTANT-13456;So;0;ON;;;;;N;;;;; +1CD35;BLOCK OCTANT-23456;So;0;ON;;;;;N;;;;; +1CD36;BLOCK OCTANT-17;So;0;ON;;;;;N;;;;; +1CD37;BLOCK OCTANT-27;So;0;ON;;;;;N;;;;; +1CD38;BLOCK OCTANT-127;So;0;ON;;;;;N;;;;; +1CD39;BLOCK OCTANT-37;So;0;ON;;;;;N;;;;; +1CD3A;BLOCK OCTANT-137;So;0;ON;;;;;N;;;;; +1CD3B;BLOCK OCTANT-237;So;0;ON;;;;;N;;;;; +1CD3C;BLOCK OCTANT-1237;So;0;ON;;;;;N;;;;; +1CD3D;BLOCK OCTANT-47;So;0;ON;;;;;N;;;;; +1CD3E;BLOCK OCTANT-147;So;0;ON;;;;;N;;;;; +1CD3F;BLOCK OCTANT-247;So;0;ON;;;;;N;;;;; +1CD40;BLOCK OCTANT-1247;So;0;ON;;;;;N;;;;; +1CD41;BLOCK OCTANT-347;So;0;ON;;;;;N;;;;; +1CD42;BLOCK OCTANT-1347;So;0;ON;;;;;N;;;;; +1CD43;BLOCK OCTANT-2347;So;0;ON;;;;;N;;;;; +1CD44;BLOCK OCTANT-12347;So;0;ON;;;;;N;;;;; +1CD45;BLOCK OCTANT-157;So;0;ON;;;;;N;;;;; +1CD46;BLOCK OCTANT-257;So;0;ON;;;;;N;;;;; +1CD47;BLOCK OCTANT-1257;So;0;ON;;;;;N;;;;; +1CD48;BLOCK OCTANT-357;So;0;ON;;;;;N;;;;; +1CD49;BLOCK OCTANT-2357;So;0;ON;;;;;N;;;;; +1CD4A;BLOCK OCTANT-12357;So;0;ON;;;;;N;;;;; +1CD4B;BLOCK OCTANT-457;So;0;ON;;;;;N;;;;; +1CD4C;BLOCK OCTANT-1457;So;0;ON;;;;;N;;;;; +1CD4D;BLOCK OCTANT-12457;So;0;ON;;;;;N;;;;; +1CD4E;BLOCK OCTANT-3457;So;0;ON;;;;;N;;;;; +1CD4F;BLOCK OCTANT-13457;So;0;ON;;;;;N;;;;; +1CD50;BLOCK OCTANT-23457;So;0;ON;;;;;N;;;;; +1CD51;BLOCK OCTANT-67;So;0;ON;;;;;N;;;;; +1CD52;BLOCK OCTANT-167;So;0;ON;;;;;N;;;;; +1CD53;BLOCK OCTANT-267;So;0;ON;;;;;N;;;;; +1CD54;BLOCK OCTANT-1267;So;0;ON;;;;;N;;;;; +1CD55;BLOCK OCTANT-367;So;0;ON;;;;;N;;;;; +1CD56;BLOCK OCTANT-1367;So;0;ON;;;;;N;;;;; +1CD57;BLOCK OCTANT-2367;So;0;ON;;;;;N;;;;; +1CD58;BLOCK OCTANT-12367;So;0;ON;;;;;N;;;;; +1CD59;BLOCK OCTANT-467;So;0;ON;;;;;N;;;;; +1CD5A;BLOCK OCTANT-1467;So;0;ON;;;;;N;;;;; +1CD5B;BLOCK OCTANT-2467;So;0;ON;;;;;N;;;;; +1CD5C;BLOCK OCTANT-12467;So;0;ON;;;;;N;;;;; +1CD5D;BLOCK OCTANT-3467;So;0;ON;;;;;N;;;;; +1CD5E;BLOCK OCTANT-13467;So;0;ON;;;;;N;;;;; +1CD5F;BLOCK OCTANT-23467;So;0;ON;;;;;N;;;;; +1CD60;BLOCK OCTANT-123467;So;0;ON;;;;;N;;;;; +1CD61;BLOCK OCTANT-567;So;0;ON;;;;;N;;;;; +1CD62;BLOCK OCTANT-1567;So;0;ON;;;;;N;;;;; +1CD63;BLOCK OCTANT-2567;So;0;ON;;;;;N;;;;; +1CD64;BLOCK OCTANT-12567;So;0;ON;;;;;N;;;;; +1CD65;BLOCK OCTANT-3567;So;0;ON;;;;;N;;;;; +1CD66;BLOCK OCTANT-13567;So;0;ON;;;;;N;;;;; +1CD67;BLOCK OCTANT-23567;So;0;ON;;;;;N;;;;; +1CD68;BLOCK OCTANT-123567;So;0;ON;;;;;N;;;;; +1CD69;BLOCK OCTANT-4567;So;0;ON;;;;;N;;;;; +1CD6A;BLOCK OCTANT-14567;So;0;ON;;;;;N;;;;; +1CD6B;BLOCK OCTANT-24567;So;0;ON;;;;;N;;;;; +1CD6C;BLOCK OCTANT-124567;So;0;ON;;;;;N;;;;; +1CD6D;BLOCK OCTANT-34567;So;0;ON;;;;;N;;;;; +1CD6E;BLOCK OCTANT-134567;So;0;ON;;;;;N;;;;; +1CD6F;BLOCK OCTANT-234567;So;0;ON;;;;;N;;;;; +1CD70;BLOCK OCTANT-1234567;So;0;ON;;;;;N;;;;; +1CD71;BLOCK OCTANT-18;So;0;ON;;;;;N;;;;; +1CD72;BLOCK OCTANT-28;So;0;ON;;;;;N;;;;; +1CD73;BLOCK OCTANT-128;So;0;ON;;;;;N;;;;; +1CD74;BLOCK OCTANT-38;So;0;ON;;;;;N;;;;; +1CD75;BLOCK OCTANT-138;So;0;ON;;;;;N;;;;; +1CD76;BLOCK OCTANT-238;So;0;ON;;;;;N;;;;; +1CD77;BLOCK OCTANT-1238;So;0;ON;;;;;N;;;;; +1CD78;BLOCK OCTANT-48;So;0;ON;;;;;N;;;;; +1CD79;BLOCK OCTANT-148;So;0;ON;;;;;N;;;;; +1CD7A;BLOCK OCTANT-248;So;0;ON;;;;;N;;;;; +1CD7B;BLOCK OCTANT-1248;So;0;ON;;;;;N;;;;; +1CD7C;BLOCK OCTANT-348;So;0;ON;;;;;N;;;;; +1CD7D;BLOCK OCTANT-1348;So;0;ON;;;;;N;;;;; +1CD7E;BLOCK OCTANT-2348;So;0;ON;;;;;N;;;;; +1CD7F;BLOCK OCTANT-12348;So;0;ON;;;;;N;;;;; +1CD80;BLOCK OCTANT-58;So;0;ON;;;;;N;;;;; +1CD81;BLOCK OCTANT-158;So;0;ON;;;;;N;;;;; +1CD82;BLOCK OCTANT-258;So;0;ON;;;;;N;;;;; +1CD83;BLOCK OCTANT-1258;So;0;ON;;;;;N;;;;; +1CD84;BLOCK OCTANT-358;So;0;ON;;;;;N;;;;; +1CD85;BLOCK OCTANT-1358;So;0;ON;;;;;N;;;;; +1CD86;BLOCK OCTANT-2358;So;0;ON;;;;;N;;;;; +1CD87;BLOCK OCTANT-12358;So;0;ON;;;;;N;;;;; +1CD88;BLOCK OCTANT-458;So;0;ON;;;;;N;;;;; +1CD89;BLOCK OCTANT-1458;So;0;ON;;;;;N;;;;; +1CD8A;BLOCK OCTANT-2458;So;0;ON;;;;;N;;;;; +1CD8B;BLOCK OCTANT-12458;So;0;ON;;;;;N;;;;; +1CD8C;BLOCK OCTANT-3458;So;0;ON;;;;;N;;;;; +1CD8D;BLOCK OCTANT-13458;So;0;ON;;;;;N;;;;; +1CD8E;BLOCK OCTANT-23458;So;0;ON;;;;;N;;;;; +1CD8F;BLOCK OCTANT-123458;So;0;ON;;;;;N;;;;; +1CD90;BLOCK OCTANT-168;So;0;ON;;;;;N;;;;; +1CD91;BLOCK OCTANT-268;So;0;ON;;;;;N;;;;; +1CD92;BLOCK OCTANT-1268;So;0;ON;;;;;N;;;;; +1CD93;BLOCK OCTANT-368;So;0;ON;;;;;N;;;;; +1CD94;BLOCK OCTANT-2368;So;0;ON;;;;;N;;;;; +1CD95;BLOCK OCTANT-12368;So;0;ON;;;;;N;;;;; +1CD96;BLOCK OCTANT-468;So;0;ON;;;;;N;;;;; +1CD97;BLOCK OCTANT-1468;So;0;ON;;;;;N;;;;; +1CD98;BLOCK OCTANT-12468;So;0;ON;;;;;N;;;;; +1CD99;BLOCK OCTANT-3468;So;0;ON;;;;;N;;;;; +1CD9A;BLOCK OCTANT-13468;So;0;ON;;;;;N;;;;; +1CD9B;BLOCK OCTANT-23468;So;0;ON;;;;;N;;;;; +1CD9C;BLOCK OCTANT-568;So;0;ON;;;;;N;;;;; +1CD9D;BLOCK OCTANT-1568;So;0;ON;;;;;N;;;;; +1CD9E;BLOCK OCTANT-2568;So;0;ON;;;;;N;;;;; +1CD9F;BLOCK OCTANT-12568;So;0;ON;;;;;N;;;;; +1CDA0;BLOCK OCTANT-3568;So;0;ON;;;;;N;;;;; +1CDA1;BLOCK OCTANT-13568;So;0;ON;;;;;N;;;;; +1CDA2;BLOCK OCTANT-23568;So;0;ON;;;;;N;;;;; +1CDA3;BLOCK OCTANT-123568;So;0;ON;;;;;N;;;;; +1CDA4;BLOCK OCTANT-4568;So;0;ON;;;;;N;;;;; +1CDA5;BLOCK OCTANT-14568;So;0;ON;;;;;N;;;;; +1CDA6;BLOCK OCTANT-24568;So;0;ON;;;;;N;;;;; +1CDA7;BLOCK OCTANT-124568;So;0;ON;;;;;N;;;;; +1CDA8;BLOCK OCTANT-34568;So;0;ON;;;;;N;;;;; +1CDA9;BLOCK OCTANT-134568;So;0;ON;;;;;N;;;;; +1CDAA;BLOCK OCTANT-234568;So;0;ON;;;;;N;;;;; +1CDAB;BLOCK OCTANT-1234568;So;0;ON;;;;;N;;;;; +1CDAC;BLOCK OCTANT-178;So;0;ON;;;;;N;;;;; +1CDAD;BLOCK OCTANT-278;So;0;ON;;;;;N;;;;; +1CDAE;BLOCK OCTANT-1278;So;0;ON;;;;;N;;;;; +1CDAF;BLOCK OCTANT-378;So;0;ON;;;;;N;;;;; +1CDB0;BLOCK OCTANT-1378;So;0;ON;;;;;N;;;;; +1CDB1;BLOCK OCTANT-2378;So;0;ON;;;;;N;;;;; +1CDB2;BLOCK OCTANT-12378;So;0;ON;;;;;N;;;;; +1CDB3;BLOCK OCTANT-478;So;0;ON;;;;;N;;;;; +1CDB4;BLOCK OCTANT-1478;So;0;ON;;;;;N;;;;; +1CDB5;BLOCK OCTANT-2478;So;0;ON;;;;;N;;;;; +1CDB6;BLOCK OCTANT-12478;So;0;ON;;;;;N;;;;; +1CDB7;BLOCK OCTANT-3478;So;0;ON;;;;;N;;;;; +1CDB8;BLOCK OCTANT-13478;So;0;ON;;;;;N;;;;; +1CDB9;BLOCK OCTANT-23478;So;0;ON;;;;;N;;;;; +1CDBA;BLOCK OCTANT-123478;So;0;ON;;;;;N;;;;; +1CDBB;BLOCK OCTANT-578;So;0;ON;;;;;N;;;;; +1CDBC;BLOCK OCTANT-1578;So;0;ON;;;;;N;;;;; +1CDBD;BLOCK OCTANT-2578;So;0;ON;;;;;N;;;;; +1CDBE;BLOCK OCTANT-12578;So;0;ON;;;;;N;;;;; +1CDBF;BLOCK OCTANT-3578;So;0;ON;;;;;N;;;;; +1CDC0;BLOCK OCTANT-13578;So;0;ON;;;;;N;;;;; +1CDC1;BLOCK OCTANT-23578;So;0;ON;;;;;N;;;;; +1CDC2;BLOCK OCTANT-123578;So;0;ON;;;;;N;;;;; +1CDC3;BLOCK OCTANT-4578;So;0;ON;;;;;N;;;;; +1CDC4;BLOCK OCTANT-14578;So;0;ON;;;;;N;;;;; +1CDC5;BLOCK OCTANT-24578;So;0;ON;;;;;N;;;;; +1CDC6;BLOCK OCTANT-124578;So;0;ON;;;;;N;;;;; +1CDC7;BLOCK OCTANT-34578;So;0;ON;;;;;N;;;;; +1CDC8;BLOCK OCTANT-134578;So;0;ON;;;;;N;;;;; +1CDC9;BLOCK OCTANT-234578;So;0;ON;;;;;N;;;;; +1CDCA;BLOCK OCTANT-1234578;So;0;ON;;;;;N;;;;; +1CDCB;BLOCK OCTANT-678;So;0;ON;;;;;N;;;;; +1CDCC;BLOCK OCTANT-1678;So;0;ON;;;;;N;;;;; +1CDCD;BLOCK OCTANT-2678;So;0;ON;;;;;N;;;;; +1CDCE;BLOCK OCTANT-12678;So;0;ON;;;;;N;;;;; +1CDCF;BLOCK OCTANT-3678;So;0;ON;;;;;N;;;;; +1CDD0;BLOCK OCTANT-13678;So;0;ON;;;;;N;;;;; +1CDD1;BLOCK OCTANT-23678;So;0;ON;;;;;N;;;;; +1CDD2;BLOCK OCTANT-123678;So;0;ON;;;;;N;;;;; +1CDD3;BLOCK OCTANT-4678;So;0;ON;;;;;N;;;;; +1CDD4;BLOCK OCTANT-14678;So;0;ON;;;;;N;;;;; +1CDD5;BLOCK OCTANT-24678;So;0;ON;;;;;N;;;;; +1CDD6;BLOCK OCTANT-124678;So;0;ON;;;;;N;;;;; +1CDD7;BLOCK OCTANT-34678;So;0;ON;;;;;N;;;;; +1CDD8;BLOCK OCTANT-134678;So;0;ON;;;;;N;;;;; +1CDD9;BLOCK OCTANT-234678;So;0;ON;;;;;N;;;;; +1CDDA;BLOCK OCTANT-1234678;So;0;ON;;;;;N;;;;; +1CDDB;BLOCK OCTANT-15678;So;0;ON;;;;;N;;;;; +1CDDC;BLOCK OCTANT-25678;So;0;ON;;;;;N;;;;; +1CDDD;BLOCK OCTANT-125678;So;0;ON;;;;;N;;;;; +1CDDE;BLOCK OCTANT-35678;So;0;ON;;;;;N;;;;; +1CDDF;BLOCK OCTANT-235678;So;0;ON;;;;;N;;;;; +1CDE0;BLOCK OCTANT-1235678;So;0;ON;;;;;N;;;;; +1CDE1;BLOCK OCTANT-45678;So;0;ON;;;;;N;;;;; +1CDE2;BLOCK OCTANT-145678;So;0;ON;;;;;N;;;;; +1CDE3;BLOCK OCTANT-1245678;So;0;ON;;;;;N;;;;; +1CDE4;BLOCK OCTANT-1345678;So;0;ON;;;;;N;;;;; +1CDE5;BLOCK OCTANT-2345678;So;0;ON;;;;;N;;;;; +1CDE6;TOP HALF STANDING PERSON;So;0;ON;;;;;N;;;;; +1CDE7;BOTTOM HALF STANDING PERSON;So;0;ON;;;;;N;;;;; +1CDE8;TOP HALF RIGHT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDE9;BOTTOM HALF RIGHT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDEA;TOP HALF RIGHT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDEB;BOTTOM HALF RIGHT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDEC;TOP HALF LEFT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDED;BOTTOM HALF LEFT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDEE;TOP HALF LEFT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDEF;BOTTOM HALF LEFT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDF0;TOP HALF FORWARD-FACING RUNNER;So;0;ON;;;;;N;;;;; +1CDF1;BOTTOM HALF FORWARD-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDF2;BOTTOM HALF FORWARD-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDF3;BOTTOM HALF FORWARD-FACING RUNNER FRAME-3;So;0;ON;;;;;N;;;;; +1CDF4;BOTTOM HALF FORWARD-FACING RUNNER FRAME-4;So;0;ON;;;;;N;;;;; +1CDF5;MOON LANDER;So;0;ON;;;;;N;;;;; +1CDF6;TOP HALF FLAILING ROBOT FRAME-1;So;0;ON;;;;;N;;;;; +1CDF7;TOP HALF FLAILING ROBOT FRAME-2;So;0;ON;;;;;N;;;;; +1CDF8;DOWN-POINTING AIRPLANE;So;0;ON;;;;;N;;;;; +1CDF9;LEFT-POINTING AIRPLANE;So;0;ON;;;;;N;;;;; +1CDFA;SMALL UP-POINTING AIRPLANE;So;0;ON;;;;;N;;;;; +1CDFB;UP-POINTING FROG;So;0;ON;;;;;N;;;;; +1CDFC;DOWN-POINTING FROG;So;0;ON;;;;;N;;;;; +1CDFD;EXPLOSION FRAME-1;So;0;ON;;;;;N;;;;; +1CDFE;EXPLOSION FRAME-2;So;0;ON;;;;;N;;;;; +1CDFF;EXPLOSION FRAME-3;So;0;ON;;;;;N;;;;; +1CE00;RIGHT HALF AND LEFT HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1CE01;LOWER HALF AND UPPER HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1CE02;EXPLOSION AT HORIZON;So;0;ON;;;;;N;;;;; +1CE03;UPPER HALF HEAVY WHITE SQUARE;So;0;ON;;;;;N;;;;; +1CE04;LOWER HALF HEAVY WHITE SQUARE;So;0;ON;;;;;N;;;;; +1CE05;HEAVY WHITE SQUARE CONTAINING BLACK VERY SMALL SQUARE;So;0;ON;;;;;N;;;;; +1CE06;WHITE VERTICAL RECTANGLE WITH HORIZONTAL BAR;So;0;ON;;;;;N;;;;; +1CE07;TOP LEFT BLACK LEFT-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;;;;; +1CE08;FUNNEL;So;0;ON;;;;;N;;;;; +1CE09;BOX DRAWINGS DOUBLE DIAGONAL LOWER LEFT TO MIDDLE CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CE0A;BOX DRAWINGS DOUBLE DIAGONAL UPPER LEFT TO MIDDLE CENTRE TO UPPER RIGHT;So;0;ON;;;;;N;;;;; +1CE0B;LEFT HALF WHITE ELLIPSE;So;0;ON;;;;;N;;;;; +1CE0C;RIGHT HALF WHITE ELLIPSE;So;0;ON;;;;;N;;;;; +1CE0D;LEFT HALF TRIPLE DASH HORIZONTAL;So;0;ON;;;;;N;;;;; +1CE0E;RIGHT HALF TRIPLE DASH HORIZONTAL;So;0;ON;;;;;N;;;;; +1CE0F;HORIZONTAL LINE WITH TICK MARK;So;0;ON;;;;;N;;;;; +1CE10;LEFT HALF HORIZONTAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE11;RIGHT HALF HORIZONTAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE12;HORIZONTAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE13;LOWER HALF VERTICAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE14;UPPER HALF VERTICAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE15;VERTICAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE16;BOX DRAWINGS LIGHT VERTICAL AND TOP RIGHT;So;0;ON;;;;;N;;;;; +1CE17;BOX DRAWINGS LIGHT VERTICAL AND BOTTOM RIGHT;So;0;ON;;;;;N;;;;; +1CE18;BOX DRAWINGS LIGHT VERTICAL AND TOP LEFT;So;0;ON;;;;;N;;;;; +1CE19;BOX DRAWINGS LIGHT VERTICAL AND BOTTOM LEFT;So;0;ON;;;;;N;;;;; +1CE1A;LARGE TYPE PIECE UPPER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE1B;LARGE TYPE PIECE UPPER LEFT CORNER;So;0;ON;;;;;N;;;;; +1CE1C;LARGE TYPE PIECE UPPER TERMINAL;So;0;ON;;;;;N;;;;; +1CE1D;LARGE TYPE PIECE UPPER LEFT CROTCH;So;0;ON;;;;;N;;;;; +1CE1E;LARGE TYPE PIECE LEFT ARM;So;0;ON;;;;;N;;;;; +1CE1F;LARGE TYPE PIECE CROSSBAR;So;0;ON;;;;;N;;;;; +1CE20;LARGE TYPE PIECE CROSSBAR WITH LOWER STEM;So;0;ON;;;;;N;;;;; +1CE21;LARGE TYPE PIECE UPPER HALF VERTEX OF M;So;0;ON;;;;;N;;;;; +1CE22;LARGE TYPE PIECE DIAGONAL LOWER LEFT;So;0;ON;;;;;N;;;;; +1CE23;LARGE TYPE PIECE SHORT UPPER TERMINAL;So;0;ON;;;;;N;;;;; +1CE24;LARGE TYPE PIECE UPPER RIGHT ARC;So;0;ON;;;;;N;;;;; +1CE25;LARGE TYPE PIECE RIGHT ARM;So;0;ON;;;;;N;;;;; +1CE26;LARGE TYPE PIECE UPPER RIGHT CROTCH;So;0;ON;;;;;N;;;;; +1CE27;LARGE TYPE PIECE UPPER RIGHT CORNER;So;0;ON;;;;;N;;;;; +1CE28;LARGE TYPE PIECE STEM WITH RIGHT CROSSBAR;So;0;ON;;;;;N;;;;; +1CE29;LARGE TYPE PIECE STEM;So;0;ON;;;;;N;;;;; +1CE2A;LARGE TYPE PIECE DIAGONAL UPPER RIGHT AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CE2B;LARGE TYPE PIECE DIAGONAL UPPER RIGHT;So;0;ON;;;;;N;;;;; +1CE2C;LARGE TYPE PIECE DIAGONAL LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CE2D;LARGE TYPE PIECE SHORT LOWER TERMINAL;So;0;ON;;;;;N;;;;; +1CE2E;LARGE TYPE PIECE LOWER LEFT AND UPPER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE2F;LARGE TYPE PIECE CENTRE OF K;So;0;ON;;;;;N;;;;; +1CE30;LARGE TYPE PIECE LOWER HALF VERTEX OF M;So;0;ON;;;;;N;;;;; +1CE31;LARGE TYPE PIECE UPPER HALF VERTEX OF W;So;0;ON;;;;;N;;;;; +1CE32;LARGE TYPE PIECE CENTRE OF X;So;0;ON;;;;;N;;;;; +1CE33;LARGE TYPE PIECE CENTRE OF Y;So;0;ON;;;;;N;;;;; +1CE34;LARGE TYPE PIECE CENTRE OF Z WITH CROSSBAR;So;0;ON;;;;;N;;;;; +1CE35;LARGE TYPE PIECE RAISED UPPER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE36;LARGE TYPE PIECE STEM WITH LEFT CROSSBAR;So;0;ON;;;;;N;;;;; +1CE37;LARGE TYPE PIECE LOWER RIGHT AND UPPER RIGHT ARC;So;0;ON;;;;;N;;;;; +1CE38;LARGE TYPE PIECE DIAGONAL UPPER LEFT AND LOWER LEFT;So;0;ON;;;;;N;;;;; +1CE39;LARGE TYPE PIECE STEM WITH LEFT JOINT;So;0;ON;;;;;N;;;;; +1CE3A;LARGE TYPE PIECE STEM WITH CROSSBAR;So;0;ON;;;;;N;;;;; +1CE3B;LARGE TYPE PIECE DIAGONAL UPPER LEFT;So;0;ON;;;;;N;;;;; +1CE3C;LARGE TYPE PIECE LOWER TERMINAL;So;0;ON;;;;;N;;;;; +1CE3D;LARGE TYPE PIECE LOWER LEFT CORNER;So;0;ON;;;;;N;;;;; +1CE3E;LARGE TYPE PIECE LOWER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE3F;LARGE TYPE PIECE LOWER LEFT CROTCH;So;0;ON;;;;;N;;;;; +1CE40;LARGE TYPE PIECE CROSSBAR WITH UPPER STEM;So;0;ON;;;;;N;;;;; +1CE41;LARGE TYPE PIECE VERTEX OF V;So;0;ON;;;;;N;;;;; +1CE42;LARGE TYPE PIECE LOWER HALF VERTEX OF W;So;0;ON;;;;;N;;;;; +1CE43;LARGE TYPE PIECE LOWER RIGHT ARC;So;0;ON;;;;;N;;;;; +1CE44;LARGE TYPE PIECE LOWER RIGHT CORNER;So;0;ON;;;;;N;;;;; +1CE45;LARGE TYPE PIECE LOWER RIGHT ARC WITH TAIL;So;0;ON;;;;;N;;;;; +1CE46;LARGE TYPE PIECE LOWER RIGHT CROTCH;So;0;ON;;;;;N;;;;; +1CE47;LARGE TYPE PIECE STEM-45;So;0;ON;;;;;N;;;;; +1CE48;LARGE TYPE PIECE STEM-2345;So;0;ON;;;;;N;;;;; +1CE49;LARGE TYPE PIECE STEM-4;So;0;ON;;;;;N;;;;; +1CE4A;LARGE TYPE PIECE STEM-34;So;0;ON;;;;;N;;;;; +1CE4B;LARGE TYPE PIECE STEM-234;So;0;ON;;;;;N;;;;; +1CE4C;LARGE TYPE PIECE STEM-1234;So;0;ON;;;;;N;;;;; +1CE4D;LARGE TYPE PIECE STEM-3;So;0;ON;;;;;N;;;;; +1CE4E;LARGE TYPE PIECE STEM-23;So;0;ON;;;;;N;;;;; +1CE4F;LARGE TYPE PIECE STEM-2;So;0;ON;;;;;N;;;;; +1CE50;LARGE TYPE PIECE STEM-12;So;0;ON;;;;;N;;;;; +1CE51;SEPARATED BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; +1CE52;SEPARATED BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; +1CE53;SEPARATED BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; +1CE54;SEPARATED BLOCK SEXTANT-3;So;0;ON;;;;;N;;;;; +1CE55;SEPARATED BLOCK SEXTANT-13;So;0;ON;;;;;N;;;;; +1CE56;SEPARATED BLOCK SEXTANT-23;So;0;ON;;;;;N;;;;; +1CE57;SEPARATED BLOCK SEXTANT-123;So;0;ON;;;;;N;;;;; +1CE58;SEPARATED BLOCK SEXTANT-4;So;0;ON;;;;;N;;;;; +1CE59;SEPARATED BLOCK SEXTANT-14;So;0;ON;;;;;N;;;;; +1CE5A;SEPARATED BLOCK SEXTANT-24;So;0;ON;;;;;N;;;;; +1CE5B;SEPARATED BLOCK SEXTANT-124;So;0;ON;;;;;N;;;;; +1CE5C;SEPARATED BLOCK SEXTANT-34;So;0;ON;;;;;N;;;;; +1CE5D;SEPARATED BLOCK SEXTANT-134;So;0;ON;;;;;N;;;;; +1CE5E;SEPARATED BLOCK SEXTANT-234;So;0;ON;;;;;N;;;;; +1CE5F;SEPARATED BLOCK SEXTANT-1234;So;0;ON;;;;;N;;;;; +1CE60;SEPARATED BLOCK SEXTANT-5;So;0;ON;;;;;N;;;;; +1CE61;SEPARATED BLOCK SEXTANT-15;So;0;ON;;;;;N;;;;; +1CE62;SEPARATED BLOCK SEXTANT-25;So;0;ON;;;;;N;;;;; +1CE63;SEPARATED BLOCK SEXTANT-125;So;0;ON;;;;;N;;;;; +1CE64;SEPARATED BLOCK SEXTANT-35;So;0;ON;;;;;N;;;;; +1CE65;SEPARATED BLOCK SEXTANT-135;So;0;ON;;;;;N;;;;; +1CE66;SEPARATED BLOCK SEXTANT-235;So;0;ON;;;;;N;;;;; +1CE67;SEPARATED BLOCK SEXTANT-1235;So;0;ON;;;;;N;;;;; +1CE68;SEPARATED BLOCK SEXTANT-45;So;0;ON;;;;;N;;;;; +1CE69;SEPARATED BLOCK SEXTANT-145;So;0;ON;;;;;N;;;;; +1CE6A;SEPARATED BLOCK SEXTANT-245;So;0;ON;;;;;N;;;;; +1CE6B;SEPARATED BLOCK SEXTANT-1245;So;0;ON;;;;;N;;;;; +1CE6C;SEPARATED BLOCK SEXTANT-345;So;0;ON;;;;;N;;;;; +1CE6D;SEPARATED BLOCK SEXTANT-1345;So;0;ON;;;;;N;;;;; +1CE6E;SEPARATED BLOCK SEXTANT-2345;So;0;ON;;;;;N;;;;; +1CE6F;SEPARATED BLOCK SEXTANT-12345;So;0;ON;;;;;N;;;;; +1CE70;SEPARATED BLOCK SEXTANT-6;So;0;ON;;;;;N;;;;; +1CE71;SEPARATED BLOCK SEXTANT-16;So;0;ON;;;;;N;;;;; +1CE72;SEPARATED BLOCK SEXTANT-26;So;0;ON;;;;;N;;;;; +1CE73;SEPARATED BLOCK SEXTANT-126;So;0;ON;;;;;N;;;;; +1CE74;SEPARATED BLOCK SEXTANT-36;So;0;ON;;;;;N;;;;; +1CE75;SEPARATED BLOCK SEXTANT-136;So;0;ON;;;;;N;;;;; +1CE76;SEPARATED BLOCK SEXTANT-236;So;0;ON;;;;;N;;;;; +1CE77;SEPARATED BLOCK SEXTANT-1236;So;0;ON;;;;;N;;;;; +1CE78;SEPARATED BLOCK SEXTANT-46;So;0;ON;;;;;N;;;;; +1CE79;SEPARATED BLOCK SEXTANT-146;So;0;ON;;;;;N;;;;; +1CE7A;SEPARATED BLOCK SEXTANT-246;So;0;ON;;;;;N;;;;; +1CE7B;SEPARATED BLOCK SEXTANT-1246;So;0;ON;;;;;N;;;;; +1CE7C;SEPARATED BLOCK SEXTANT-346;So;0;ON;;;;;N;;;;; +1CE7D;SEPARATED BLOCK SEXTANT-1346;So;0;ON;;;;;N;;;;; +1CE7E;SEPARATED BLOCK SEXTANT-2346;So;0;ON;;;;;N;;;;; +1CE7F;SEPARATED BLOCK SEXTANT-12346;So;0;ON;;;;;N;;;;; +1CE80;SEPARATED BLOCK SEXTANT-56;So;0;ON;;;;;N;;;;; +1CE81;SEPARATED BLOCK SEXTANT-156;So;0;ON;;;;;N;;;;; +1CE82;SEPARATED BLOCK SEXTANT-256;So;0;ON;;;;;N;;;;; +1CE83;SEPARATED BLOCK SEXTANT-1256;So;0;ON;;;;;N;;;;; +1CE84;SEPARATED BLOCK SEXTANT-356;So;0;ON;;;;;N;;;;; +1CE85;SEPARATED BLOCK SEXTANT-1356;So;0;ON;;;;;N;;;;; +1CE86;SEPARATED BLOCK SEXTANT-2356;So;0;ON;;;;;N;;;;; +1CE87;SEPARATED BLOCK SEXTANT-12356;So;0;ON;;;;;N;;;;; +1CE88;SEPARATED BLOCK SEXTANT-456;So;0;ON;;;;;N;;;;; +1CE89;SEPARATED BLOCK SEXTANT-1456;So;0;ON;;;;;N;;;;; +1CE8A;SEPARATED BLOCK SEXTANT-2456;So;0;ON;;;;;N;;;;; +1CE8B;SEPARATED BLOCK SEXTANT-12456;So;0;ON;;;;;N;;;;; +1CE8C;SEPARATED BLOCK SEXTANT-3456;So;0;ON;;;;;N;;;;; +1CE8D;SEPARATED BLOCK SEXTANT-13456;So;0;ON;;;;;N;;;;; +1CE8E;SEPARATED BLOCK SEXTANT-23456;So;0;ON;;;;;N;;;;; +1CE8F;SEPARATED BLOCK SEXTANT-123456;So;0;ON;;;;;N;;;;; +1CE90;UPPER LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE91;UPPER CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE92;UPPER CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE93;UPPER RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE94;UPPER MIDDLE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE95;UPPER MIDDLE CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE96;UPPER MIDDLE CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE97;UPPER MIDDLE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE98;LOWER MIDDLE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE99;LOWER MIDDLE CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9A;LOWER MIDDLE CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9B;LOWER MIDDLE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9C;LOWER LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9D;LOWER CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9E;LOWER CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9F;LOWER RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CEA0;RIGHT HALF LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA1;RIGHT THREE QUARTERS LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA2;LEFT THREE QUARTERS LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA3;LEFT HALF LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA4;LOWER HALF LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA5;LOWER THREE QUARTERS LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA6;UPPER THREE QUARTERS LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA7;UPPER HALF LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA8;LEFT HALF UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA9;LEFT THREE QUARTERS UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAA;RIGHT THREE QUARTERS UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAB;RIGHT HALF UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAC;UPPER HALF RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAD;UPPER THREE QUARTERS RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAE;LOWER THREE QUARTERS RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAF;LOWER HALF RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEB0;HORIZONTAL ZIGZAG LINE;So;0;ON;;;;;N;;;;; +1CEB1;KEYHOLE;So;0;ON;;;;;N;;;;; +1CEB2;OLD PERSONAL COMPUTER WITH MONITOR IN PORTRAIT ORIENTATION;So;0;ON;;;;;N;;;;; +1CEB3;BLACK RIGHT TRIANGLE CARET;So;0;ON;;;;;N;;;;; 1CF00;ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT;Mn;0;NSM;;;;;N;;;;; 1CF01;ZNAMENNY COMBINING MARK NIZKO S KRYZHEM ON LEFT;Mn;0;NSM;;;;;N;;;;; 1CF02;ZNAMENNY COMBINING MARK TSATA ON LEFT;Mn;0;NSM;;;;;N;;;;; @@ -29545,7 +34630,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D6BE;MATHEMATICAL BOLD CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D6BF;MATHEMATICAL BOLD CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D6C0;MATHEMATICAL BOLD CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D6C1;MATHEMATICAL BOLD NABLA;Sm;0;L; 2207;;;;N;;;;; +1D6C1;MATHEMATICAL BOLD NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D6C2;MATHEMATICAL BOLD SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D6C3;MATHEMATICAL BOLD SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D6C4;MATHEMATICAL BOLD SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29603,7 +34688,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D6F8;MATHEMATICAL ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D6F9;MATHEMATICAL ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D6FA;MATHEMATICAL ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D6FB;MATHEMATICAL ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D6FB;MATHEMATICAL ITALIC NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D6FC;MATHEMATICAL ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D6FD;MATHEMATICAL ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D6FE;MATHEMATICAL ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29661,7 +34746,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D732;MATHEMATICAL BOLD ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D733;MATHEMATICAL BOLD ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D734;MATHEMATICAL BOLD ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D735;MATHEMATICAL BOLD ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D735;MATHEMATICAL BOLD ITALIC NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D736;MATHEMATICAL BOLD ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D737;MATHEMATICAL BOLD ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D738;MATHEMATICAL BOLD ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29719,7 +34804,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D76C;MATHEMATICAL SANS-SERIF BOLD CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D76D;MATHEMATICAL SANS-SERIF BOLD CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D76E;MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D76F;MATHEMATICAL SANS-SERIF BOLD NABLA;Sm;0;L; 2207;;;;N;;;;; +1D76F;MATHEMATICAL SANS-SERIF BOLD NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D770;MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D771;MATHEMATICAL SANS-SERIF BOLD SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D772;MATHEMATICAL SANS-SERIF BOLD SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29777,7 +34862,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D7A6;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D7A7;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D7A8;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D7A9;MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D7A9;MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D7AA;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D7AB;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D7AC;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -30875,6 +35960,50 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1E5D0;OL ONAL LETTER O;Lo;0;L;;;;;N;;;;; +1E5D1;OL ONAL LETTER OM;Lo;0;L;;;;;N;;;;; +1E5D2;OL ONAL LETTER ONG;Lo;0;L;;;;;N;;;;; +1E5D3;OL ONAL LETTER ORR;Lo;0;L;;;;;N;;;;; +1E5D4;OL ONAL LETTER OO;Lo;0;L;;;;;N;;;;; +1E5D5;OL ONAL LETTER OY;Lo;0;L;;;;;N;;;;; +1E5D6;OL ONAL LETTER A;Lo;0;L;;;;;N;;;;; +1E5D7;OL ONAL LETTER AD;Lo;0;L;;;;;N;;;;; +1E5D8;OL ONAL LETTER AB;Lo;0;L;;;;;N;;;;; +1E5D9;OL ONAL LETTER AH;Lo;0;L;;;;;N;;;;; +1E5DA;OL ONAL LETTER AL;Lo;0;L;;;;;N;;;;; +1E5DB;OL ONAL LETTER AW;Lo;0;L;;;;;N;;;;; +1E5DC;OL ONAL LETTER I;Lo;0;L;;;;;N;;;;; +1E5DD;OL ONAL LETTER IT;Lo;0;L;;;;;N;;;;; +1E5DE;OL ONAL LETTER IP;Lo;0;L;;;;;N;;;;; +1E5DF;OL ONAL LETTER ITT;Lo;0;L;;;;;N;;;;; +1E5E0;OL ONAL LETTER ID;Lo;0;L;;;;;N;;;;; +1E5E1;OL ONAL LETTER IN;Lo;0;L;;;;;N;;;;; +1E5E2;OL ONAL LETTER U;Lo;0;L;;;;;N;;;;; +1E5E3;OL ONAL LETTER UK;Lo;0;L;;;;;N;;;;; +1E5E4;OL ONAL LETTER UDD;Lo;0;L;;;;;N;;;;; +1E5E5;OL ONAL LETTER UJ;Lo;0;L;;;;;N;;;;; +1E5E6;OL ONAL LETTER UNY;Lo;0;L;;;;;N;;;;; +1E5E7;OL ONAL LETTER UR;Lo;0;L;;;;;N;;;;; +1E5E8;OL ONAL LETTER E;Lo;0;L;;;;;N;;;;; +1E5E9;OL ONAL LETTER ES;Lo;0;L;;;;;N;;;;; +1E5EA;OL ONAL LETTER EH;Lo;0;L;;;;;N;;;;; +1E5EB;OL ONAL LETTER EC;Lo;0;L;;;;;N;;;;; +1E5EC;OL ONAL LETTER ENN;Lo;0;L;;;;;N;;;;; +1E5ED;OL ONAL LETTER EG;Lo;0;L;;;;;N;;;;; +1E5EE;OL ONAL SIGN MU;Mn;230;NSM;;;;;N;;;;; +1E5EF;OL ONAL SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E5F0;OL ONAL SIGN HODDOND;Lo;0;L;;;;;N;;;;; +1E5F1;OL ONAL DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E5F2;OL ONAL DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E5F3;OL ONAL DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E5F4;OL ONAL DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E5F5;OL ONAL DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E5F6;OL ONAL DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E5F7;OL ONAL DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E5F8;OL ONAL DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E5F9;OL ONAL DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E5FA;OL ONAL DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1E5FF;OL ONAL ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -33357,6 +38486,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F8AD;WHITE ARROW SHAFT WIDTH TWO THIRDS;So;0;ON;;;;;N;;;;; 1F8B0;ARROW POINTING UPWARDS THEN NORTH WEST;So;0;ON;;;;;N;;;;; 1F8B1;ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST;So;0;ON;;;;;N;;;;; +1F8B2;RIGHTWARDS ARROW WITH LOWER HOOK;So;0;ON;;;;;N;;;;; +1F8B3;DOWNWARDS BLACK ARROW TO BAR;So;0;ON;;;;;N;;;;; +1F8B4;NEGATIVE SQUARED LEFTWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B5;NEGATIVE SQUARED UPWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B6;NEGATIVE SQUARED RIGHTWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B7;NEGATIVE SQUARED DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B8;NORTH WEST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8B9;NORTH EAST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8BA;SOUTH EAST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8BB;SOUTH WEST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8C0;LEFTWARDS ARROW FROM DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8C1;RIGHTWARDS ARROW FROM DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; 1F900;CIRCLED CROSS FORMEE WITH FOUR DOTS;So;0;ON;;;;;N;;;;; 1F901;CIRCLED CROSS FORMEE WITH TWO DOTS;So;0;ON;;;;;N;;;;; 1F902;CIRCLED CROSS FORMEE;So;0;ON;;;;;N;;;;; @@ -33733,6 +38874,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; 1FA87;MARACAS;So;0;ON;;;;;N;;;;; 1FA88;FLUTE;So;0;ON;;;;;N;;;;; +1FA89;HARP;So;0;ON;;;;;N;;;;; +1FA8F;SHOVEL;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33779,6 +38922,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FABB;HYACINTH;So;0;ON;;;;;N;;;;; 1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; 1FABD;WING;So;0;ON;;;;;N;;;;; +1FABE;LEAFLESS TREE;So;0;ON;;;;;N;;;;; 1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; @@ -33786,6 +38930,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FAC6;FINGERPRINT;So;0;ON;;;;;N;;;;; 1FACE;MOOSE;So;0;ON;;;;;N;;;;; 1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; @@ -33800,6 +38945,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; 1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; 1FADB;PEA POD;So;0;ON;;;;;N;;;;; +1FADC;ROOT VEGETABLE;So;0;ON;;;;;N;;;;; +1FADF;SPLATTER;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33809,6 +38956,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; 1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; +1FAE9;FACE WITH BAGS UNDER EYES;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -34020,6 +39168,43 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FBC8;STICK FIGURE LEANING RIGHT;So;0;ON;;;;;N;;;;; 1FBC9;STICK FIGURE WITH DRESS;So;0;ON;;;;;N;;;;; 1FBCA;WHITE UP-POINTING CHEVRON;So;0;ON;;;;;N;;;;; +1FBCB;WHITE CROSS MARK;So;0;ON;;;;;N;;;;; +1FBCC;RAISED SMALL LEFT SQUARE BRACKET;So;0;ON;;;;;N;;;;; +1FBCD;BLACK SMALL UP-POINTING CHEVRON;So;0;ON;;;;;N;;;;; +1FBCE;LEFT TWO THIRDS BLOCK;So;0;ON;;;;;N;;;;; +1FBCF;LEFT ONE THIRD BLOCK;So;0;ON;;;;;N;;;;; +1FBD0;BOX DRAWINGS LIGHT DIAGONAL MIDDLE RIGHT TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBD1;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO MIDDLE LEFT;So;0;ON;;;;;N;;;;; +1FBD2;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE RIGHT;So;0;ON;;;;;N;;;;; +1FBD3;BOX DRAWINGS LIGHT DIAGONAL MIDDLE LEFT TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBD4;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER CENTRE;So;0;ON;;;;;N;;;;; +1FBD5;BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBD6;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER CENTRE;So;0;ON;;;;;N;;;;; +1FBD7;BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBD8;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE CENTRE TO UPPER RIGHT;So;0;ON;;;;;N;;;;; +1FBD9;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO MIDDLE CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDA;BOX DRAWINGS LIGHT DIAGONAL LOWER LEFT TO MIDDLE CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDB;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE CENTRE TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBDC;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER CENTRE TO UPPER RIGHT;So;0;ON;;;;;N;;;;; +1FBDD;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO MIDDLE LEFT TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDE;BOX DRAWINGS LIGHT DIAGONAL LOWER LEFT TO UPPER CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDF;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE RIGHT TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBE0;TOP JUSTIFIED LOWER HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE1;RIGHT JUSTIFIED LEFT HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE2;BOTTOM JUSTIFIED UPPER HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE3;LEFT JUSTIFIED RIGHT HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE4;UPPER CENTRE ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE5;LOWER CENTRE ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE6;MIDDLE LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE7;MIDDLE RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE8;TOP JUSTIFIED LOWER HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBE9;RIGHT JUSTIFIED LEFT HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEA;BOTTOM JUSTIFIED UPPER HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEB;LEFT JUSTIFIED RIGHT HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEC;TOP RIGHT JUSTIFIED LOWER LEFT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBED;BOTTOM LEFT JUSTIFIED UPPER RIGHT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEE;BOTTOM RIGHT JUSTIFIED UPPER LEFT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEF;TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; 1FBF0;SEGMENTED DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; 1FBF1;SEGMENTED DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; 1FBF2;SEGMENTED DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; diff --git a/src/libre/class.h b/src/libre/class.h index 45fb09bc6..919571982 100644 --- a/src/libre/class.h +++ b/src/libre/class.h @@ -57,6 +57,7 @@ extern const struct class utf8_Egyptian_Hieroglyphs; extern const struct class utf8_Elbasan; extern const struct class utf8_Elymaic; extern const struct class utf8_Ethiopic; +extern const struct class utf8_Garay; extern const struct class utf8_Georgian; extern const struct class utf8_Glagolitic; extern const struct class utf8_Gothic; @@ -65,6 +66,7 @@ extern const struct class utf8_Greek; extern const struct class utf8_Gujarati; extern const struct class utf8_Gunjala_Gondi; extern const struct class utf8_Gurmukhi; +extern const struct class utf8_Gurung_Khema; extern const struct class utf8_Han; extern const struct class utf8_Hangul; extern const struct class utf8_Hanifi_Rohingya; @@ -80,12 +82,14 @@ extern const struct class utf8_Javanese; extern const struct class utf8_Kaithi; extern const struct class utf8_Kannada; extern const struct class utf8_Katakana; +extern const struct class utf8_Kawi; extern const struct class utf8_Kayah_Li; extern const struct class utf8_Kharoshthi; extern const struct class utf8_Khitan_Small_Script; extern const struct class utf8_Khmer; extern const struct class utf8_Khojki; extern const struct class utf8_Khudawadi; +extern const struct class utf8_Kirat_Rai; extern const struct class utf8_Lao; extern const struct class utf8_Latin; extern const struct class utf8_Lepcha; @@ -114,9 +118,10 @@ extern const struct class utf8_Mro; extern const struct class utf8_Multani; extern const struct class utf8_Myanmar; extern const struct class utf8_Nabataean; +extern const struct class utf8_Nag_Mundari; extern const struct class utf8_Nandinagari; -extern const struct class utf8_New_Tai_Lue; extern const struct class utf8_Newa; +extern const struct class utf8_New_Tai_Lue; extern const struct class utf8_Nko; extern const struct class utf8_Nushu; extern const struct class utf8_Nyiakeng_Puachue_Hmong; @@ -131,6 +136,7 @@ extern const struct class utf8_Old_Sogdian; extern const struct class utf8_Old_South_Arabian; extern const struct class utf8_Old_Turkic; extern const struct class utf8_Old_Uyghur; +extern const struct class utf8_Ol_Onal; extern const struct class utf8_Oriya; extern const struct class utf8_Osage; extern const struct class utf8_Osmanya; @@ -153,6 +159,7 @@ extern const struct class utf8_Sogdian; extern const struct class utf8_Sora_Sompeng; extern const struct class utf8_Soyombo; extern const struct class utf8_Sundanese; +extern const struct class utf8_Sunuwar; extern const struct class utf8_Syloti_Nagri; extern const struct class utf8_Syriac; extern const struct class utf8_Tagalog; @@ -170,7 +177,9 @@ extern const struct class utf8_Thai; extern const struct class utf8_Tibetan; extern const struct class utf8_Tifinagh; extern const struct class utf8_Tirhuta; +extern const struct class utf8_Todhri; extern const struct class utf8_Toto; +extern const struct class utf8_Tulu_Tigalari; extern const struct class utf8_Ugaritic; extern const struct class utf8_Vai; extern const struct class utf8_Vithkuqi; diff --git a/src/libre/class/utf8_Arabic.c b/src/libre/class/utf8_Arabic.c index 360564ea3..904ab392e 100644 --- a/src/libre/class/utf8_Arabic.c +++ b/src/libre/class/utf8_Arabic.c @@ -15,7 +15,7 @@ static const struct range ranges[] = { { 0x0750UL, 0x077FUL }, { 0x0870UL, 0x088EUL }, { 0x0890UL, 0x0891UL }, - { 0x0898UL, 0x08E1UL }, + { 0x0897UL, 0x08E1UL }, { 0x08E3UL, 0x08FFUL }, { 0xFB50UL, 0xFBC2UL }, { 0xFBD3UL, 0xFD3DUL }, @@ -26,7 +26,8 @@ static const struct range ranges[] = { { 0xFE70UL, 0xFE74UL }, { 0xFE76UL, 0xFEFCUL }, { 0x10E60UL, 0x10E7EUL }, - { 0x10EFDUL, 0x10EFFUL }, + { 0x10EC2UL, 0x10EC4UL }, + { 0x10EFCUL, 0x10EFFUL }, { 0x1EE00UL, 0x1EE03UL }, { 0x1EE05UL, 0x1EE1FUL }, { 0x1EE21UL, 0x1EE22UL }, diff --git a/src/libre/class/utf8_Balinese.c b/src/libre/class/utf8_Balinese.c index b86ccdd3e..2c742b725 100644 --- a/src/libre/class/utf8_Balinese.c +++ b/src/libre/class/utf8_Balinese.c @@ -4,7 +4,7 @@ static const struct range ranges[] = { { 0x1B00UL, 0x1B4CUL }, - { 0x1B50UL, 0x1B7EUL } + { 0x1B4EUL, 0x1B7FUL } }; const struct class utf8_Balinese = { diff --git a/src/libre/class/utf8_Common.c b/src/libre/class/utf8_Common.c index a6ad8b58a..44e6f3ff2 100644 --- a/src/libre/class/utf8_Common.c +++ b/src/libre/class/utf8_Common.c @@ -50,7 +50,7 @@ static const struct range ranges[] = { { 0x2133UL, 0x214DUL }, { 0x214FUL, 0x215FUL }, { 0x2189UL, 0x218BUL }, - { 0x2190UL, 0x2426UL }, + { 0x2190UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x2460UL, 0x27FFUL }, { 0x2900UL, 0x2B73UL }, @@ -66,7 +66,7 @@ static const struct range ranges[] = { { 0x30A0UL, 0x30A0UL }, { 0x30FBUL, 0x30FCUL }, { 0x3190UL, 0x319FUL }, - { 0x31C0UL, 0x31E3UL }, + { 0x31C0UL, 0x31E5UL }, { 0x31EFUL, 0x31EFUL }, { 0x3220UL, 0x325FUL }, { 0x327FUL, 0x32CFUL }, @@ -101,6 +101,8 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x102E1UL, 0x102FBUL }, { 0x1BCA0UL, 0x1BCA3UL }, + { 0x1CC00UL, 0x1CCF9UL }, + { 0x1CD00UL, 0x1CEB3UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -161,19 +163,18 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, { 0x1F900UL, 0x1FA53UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, + { 0x1FA80UL, 0x1FA89UL }, + { 0x1FA8FUL, 0x1FAC6UL }, + { 0x1FACEUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAE9UL }, { 0x1FAF0UL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL }, - { 0x1FBF0UL, 0x1FBF9UL }, + { 0x1FB94UL, 0x1FBF9UL }, { 0xE0001UL, 0xE0001UL }, { 0xE0020UL, 0xE007FUL } }; diff --git a/src/libre/class/utf8_Cyrillic.c b/src/libre/class/utf8_Cyrillic.c index 2154d4998..822ee6438 100644 --- a/src/libre/class/utf8_Cyrillic.c +++ b/src/libre/class/utf8_Cyrillic.c @@ -5,7 +5,7 @@ static const struct range ranges[] = { { 0x0400UL, 0x0484UL }, { 0x0487UL, 0x052FUL }, - { 0x1C80UL, 0x1C88UL }, + { 0x1C80UL, 0x1C8AUL }, { 0x1D2BUL, 0x1D2BUL }, { 0x1D78UL, 0x1D78UL }, { 0x2DE0UL, 0x2DFFUL }, diff --git a/src/libre/class/utf8_Egyptian_Hieroglyphs.c b/src/libre/class/utf8_Egyptian_Hieroglyphs.c index 2698da1ed..a46b2a361 100644 --- a/src/libre/class/utf8_Egyptian_Hieroglyphs.c +++ b/src/libre/class/utf8_Egyptian_Hieroglyphs.c @@ -3,7 +3,8 @@ #include "class.h" static const struct range ranges[] = { - { 0x13000UL, 0x13455UL } + { 0x13000UL, 0x13455UL }, + { 0x13460UL, 0x143FAUL } }; const struct class utf8_Egyptian_Hieroglyphs = { diff --git a/src/libre/class/utf8_Garay.c b/src/libre/class/utf8_Garay.c new file mode 100644 index 000000000..f1841e248 --- /dev/null +++ b/src/libre/class/utf8_Garay.c @@ -0,0 +1,15 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x10D40UL, 0x10D65UL }, + { 0x10D69UL, 0x10D85UL }, + { 0x10D8EUL, 0x10D8FUL } +}; + +const struct class utf8_Garay = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Gurung_Khema.c b/src/libre/class/utf8_Gurung_Khema.c new file mode 100644 index 000000000..694048f31 --- /dev/null +++ b/src/libre/class/utf8_Gurung_Khema.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x16100UL, 0x16139UL } +}; + +const struct class utf8_Gurung_Khema = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Kawi.c b/src/libre/class/utf8_Kawi.c index 617398734..5fad15253 100644 --- a/src/libre/class/utf8_Kawi.c +++ b/src/libre/class/utf8_Kawi.c @@ -5,7 +5,7 @@ static const struct range ranges[] = { { 0x11F00UL, 0x11F10UL }, { 0x11F12UL, 0x11F3AUL }, - { 0x11F3EUL, 0x11F59UL } + { 0x11F3EUL, 0x11F5AUL } }; const struct class utf8_Kawi = { diff --git a/src/libre/class/utf8_Khitan_Small_Script.c b/src/libre/class/utf8_Khitan_Small_Script.c index 74741da30..f1e054219 100644 --- a/src/libre/class/utf8_Khitan_Small_Script.c +++ b/src/libre/class/utf8_Khitan_Small_Script.c @@ -4,7 +4,8 @@ static const struct range ranges[] = { { 0x16FE4UL, 0x16FE4UL }, - { 0x18B00UL, 0x18CD5UL } + { 0x18B00UL, 0x18CD5UL }, + { 0x18CFFUL, 0x18CFFUL } }; const struct class utf8_Khitan_Small_Script = { diff --git a/src/libre/class/utf8_Kirat_Rai.c b/src/libre/class/utf8_Kirat_Rai.c new file mode 100644 index 000000000..34925abfd --- /dev/null +++ b/src/libre/class/utf8_Kirat_Rai.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x16D40UL, 0x16D79UL } +}; + +const struct class utf8_Kirat_Rai = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_L.c b/src/libre/class/utf8_L.c index 9ecd7ba14..b2c396fcc 100644 --- a/src/libre/class/utf8_L.c +++ b/src/libre/class/utf8_L.c @@ -228,7 +228,7 @@ static const struct range ranges[] = { { 0x1C00UL, 0x1C23UL }, { 0x1C4DUL, 0x1C4FUL }, { 0x1C5AUL, 0x1C7DUL }, - { 0x1C80UL, 0x1C88UL }, + { 0x1C80UL, 0x1C8AUL }, { 0x1C90UL, 0x1CBAUL }, { 0x1CBDUL, 0x1CBFUL }, { 0x1CE9UL, 0x1CECUL }, @@ -314,10 +314,10 @@ static const struct range ranges[] = { { 0xA6A0UL, 0xA6E5UL }, { 0xA717UL, 0xA71FUL }, { 0xA722UL, 0xA788UL }, - { 0xA78BUL, 0xA7CAUL }, + { 0xA78BUL, 0xA7CDUL }, { 0xA7D0UL, 0xA7D1UL }, { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7D9UL }, + { 0xA7D5UL, 0xA7DCUL }, { 0xA7F2UL, 0xA801UL }, { 0xA803UL, 0xA805UL }, { 0xA807UL, 0xA80AUL }, @@ -415,6 +415,7 @@ static const struct range ranges[] = { { 0x105A3UL, 0x105B1UL }, { 0x105B3UL, 0x105B9UL }, { 0x105BBUL, 0x105BCUL }, + { 0x105C0UL, 0x105F3UL }, { 0x10600UL, 0x10736UL }, { 0x10740UL, 0x10755UL }, { 0x10760UL, 0x10767UL }, @@ -451,8 +452,11 @@ static const struct range ranges[] = { { 0x10C80UL, 0x10CB2UL }, { 0x10CC0UL, 0x10CF2UL }, { 0x10D00UL, 0x10D23UL }, + { 0x10D4AUL, 0x10D65UL }, + { 0x10D6FUL, 0x10D85UL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EB0UL, 0x10EB1UL }, + { 0x10EC2UL, 0x10EC4UL }, { 0x10F00UL, 0x10F1CUL }, { 0x10F27UL, 0x10F27UL }, { 0x10F30UL, 0x10F45UL }, @@ -491,6 +495,13 @@ static const struct range ranges[] = { { 0x1133DUL, 0x1133DUL }, { 0x11350UL, 0x11350UL }, { 0x1135DUL, 0x11361UL }, + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113B7UL }, + { 0x113D1UL, 0x113D1UL }, + { 0x113D3UL, 0x113D3UL }, { 0x11400UL, 0x11434UL }, { 0x11447UL, 0x1144AUL }, { 0x1145FUL, 0x11461UL }, @@ -525,6 +536,7 @@ static const struct range ranges[] = { { 0x11A5CUL, 0x11A89UL }, { 0x11A9DUL, 0x11A9DUL }, { 0x11AB0UL, 0x11AF8UL }, + { 0x11BC0UL, 0x11BE0UL }, { 0x11C00UL, 0x11C08UL }, { 0x11C0AUL, 0x11C2EUL }, { 0x11C40UL, 0x11C40UL }, @@ -547,7 +559,9 @@ static const struct range ranges[] = { { 0x12F90UL, 0x12FF0UL }, { 0x13000UL, 0x1342FUL }, { 0x13441UL, 0x13446UL }, + { 0x13460UL, 0x143FAUL }, { 0x14400UL, 0x14646UL }, + { 0x16100UL, 0x1611DUL }, { 0x16800UL, 0x16A38UL }, { 0x16A40UL, 0x16A5EUL }, { 0x16A70UL, 0x16ABEUL }, @@ -556,6 +570,7 @@ static const struct range ranges[] = { { 0x16B40UL, 0x16B43UL }, { 0x16B63UL, 0x16B77UL }, { 0x16B7DUL, 0x16B8FUL }, + { 0x16D40UL, 0x16D6CUL }, { 0x16E40UL, 0x16E7FUL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F50UL, 0x16F50UL }, @@ -565,7 +580,7 @@ static const struct range ranges[] = { { 0x17000UL, 0x17000UL }, { 0x187F7UL, 0x187F7UL }, { 0x18800UL, 0x18CD5UL }, - { 0x18D00UL, 0x18D00UL }, + { 0x18CFFUL, 0x18D00UL }, { 0x18D08UL, 0x18D08UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, @@ -619,6 +634,8 @@ static const struct range ranges[] = { { 0x1E290UL, 0x1E2ADUL }, { 0x1E2C0UL, 0x1E2EBUL }, { 0x1E4D0UL, 0x1E4EBUL }, + { 0x1E5D0UL, 0x1E5EDUL }, + { 0x1E5F0UL, 0x1E5F0UL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, diff --git a/src/libre/class/utf8_Latin.c b/src/libre/class/utf8_Latin.c index ed3e4b180..9f59bd35e 100644 --- a/src/libre/class/utf8_Latin.c +++ b/src/libre/class/utf8_Latin.c @@ -26,10 +26,10 @@ static const struct range ranges[] = { { 0x2160UL, 0x2188UL }, { 0x2C60UL, 0x2C7FUL }, { 0xA722UL, 0xA787UL }, - { 0xA78BUL, 0xA7CAUL }, + { 0xA78BUL, 0xA7CDUL }, { 0xA7D0UL, 0xA7D1UL }, { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7D9UL }, + { 0xA7D5UL, 0xA7DCUL }, { 0xA7F2UL, 0xA7FFUL }, { 0xAB30UL, 0xAB5AUL }, { 0xAB5CUL, 0xAB64UL }, diff --git a/src/libre/class/utf8_Ll.c b/src/libre/class/utf8_Ll.c index 89de4bd0e..889a1da5e 100644 --- a/src/libre/class/utf8_Ll.c +++ b/src/libre/class/utf8_Ll.c @@ -277,6 +277,7 @@ static const struct range ranges[] = { { 0x10FDUL, 0x10FFUL }, { 0x13F8UL, 0x13FDUL }, { 0x1C80UL, 0x1C88UL }, + { 0x1C8AUL, 0x1C8AUL }, { 0x1D00UL, 0x1D2BUL }, { 0x1D6BUL, 0x1D77UL }, { 0x1D79UL, 0x1D9AUL }, @@ -607,11 +608,13 @@ static const struct range ranges[] = { { 0xA7C3UL, 0xA7C3UL }, { 0xA7C8UL, 0xA7C8UL }, { 0xA7CAUL, 0xA7CAUL }, + { 0xA7CDUL, 0xA7CDUL }, { 0xA7D1UL, 0xA7D1UL }, { 0xA7D3UL, 0xA7D3UL }, { 0xA7D5UL, 0xA7D5UL }, { 0xA7D7UL, 0xA7D7UL }, { 0xA7D9UL, 0xA7D9UL }, + { 0xA7DBUL, 0xA7DBUL }, { 0xA7F6UL, 0xA7F6UL }, { 0xA7FAUL, 0xA7FAUL }, { 0xAB30UL, 0xAB5AUL }, @@ -627,6 +630,7 @@ static const struct range ranges[] = { { 0x105B3UL, 0x105B9UL }, { 0x105BBUL, 0x105BCUL }, { 0x10CC0UL, 0x10CF2UL }, + { 0x10D70UL, 0x10D85UL }, { 0x118C0UL, 0x118DFUL }, { 0x16E60UL, 0x16E7FUL }, { 0x1D41AUL, 0x1D433UL }, diff --git a/src/libre/class/utf8_Lm.c b/src/libre/class/utf8_Lm.c index 277354e6a..884125481 100644 --- a/src/libre/class/utf8_Lm.c +++ b/src/libre/class/utf8_Lm.c @@ -63,7 +63,11 @@ static const struct range ranges[] = { { 0x10780UL, 0x10785UL }, { 0x10787UL, 0x107B0UL }, { 0x107B2UL, 0x107BAUL }, + { 0x10D4EUL, 0x10D4EUL }, + { 0x10D6FUL, 0x10D6FUL }, { 0x16B40UL, 0x16B43UL }, + { 0x16D40UL, 0x16D42UL }, + { 0x16D6BUL, 0x16D6CUL }, { 0x16F93UL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE1UL }, { 0x16FE3UL, 0x16FE3UL }, diff --git a/src/libre/class/utf8_Lo.c b/src/libre/class/utf8_Lo.c index c361d6892..0ba2e7a9b 100644 --- a/src/libre/class/utf8_Lo.c +++ b/src/libre/class/utf8_Lo.c @@ -315,6 +315,7 @@ static const struct range ranges[] = { { 0x10450UL, 0x1049DUL }, { 0x10500UL, 0x10527UL }, { 0x10530UL, 0x10563UL }, + { 0x105C0UL, 0x105F3UL }, { 0x10600UL, 0x10736UL }, { 0x10740UL, 0x10755UL }, { 0x10760UL, 0x10767UL }, @@ -346,8 +347,11 @@ static const struct range ranges[] = { { 0x10B80UL, 0x10B91UL }, { 0x10C00UL, 0x10C48UL }, { 0x10D00UL, 0x10D23UL }, + { 0x10D4AUL, 0x10D4DUL }, + { 0x10D4FUL, 0x10D4FUL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EB0UL, 0x10EB1UL }, + { 0x10EC2UL, 0x10EC4UL }, { 0x10F00UL, 0x10F1CUL }, { 0x10F27UL, 0x10F27UL }, { 0x10F30UL, 0x10F45UL }, @@ -386,6 +390,13 @@ static const struct range ranges[] = { { 0x1133DUL, 0x1133DUL }, { 0x11350UL, 0x11350UL }, { 0x1135DUL, 0x11361UL }, + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113B7UL }, + { 0x113D1UL, 0x113D1UL }, + { 0x113D3UL, 0x113D3UL }, { 0x11400UL, 0x11434UL }, { 0x11447UL, 0x1144AUL }, { 0x1145FUL, 0x11461UL }, @@ -419,6 +430,7 @@ static const struct range ranges[] = { { 0x11A5CUL, 0x11A89UL }, { 0x11A9DUL, 0x11A9DUL }, { 0x11AB0UL, 0x11AF8UL }, + { 0x11BC0UL, 0x11BE0UL }, { 0x11C00UL, 0x11C08UL }, { 0x11C0AUL, 0x11C2EUL }, { 0x11C40UL, 0x11C40UL }, @@ -441,7 +453,9 @@ static const struct range ranges[] = { { 0x12F90UL, 0x12FF0UL }, { 0x13000UL, 0x1342FUL }, { 0x13441UL, 0x13446UL }, + { 0x13460UL, 0x143FAUL }, { 0x14400UL, 0x14646UL }, + { 0x16100UL, 0x1611DUL }, { 0x16800UL, 0x16A38UL }, { 0x16A40UL, 0x16A5EUL }, { 0x16A70UL, 0x16ABEUL }, @@ -449,12 +463,13 @@ static const struct range ranges[] = { { 0x16B00UL, 0x16B2FUL }, { 0x16B63UL, 0x16B77UL }, { 0x16B7DUL, 0x16B8FUL }, + { 0x16D43UL, 0x16D6AUL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F50UL, 0x16F50UL }, { 0x17000UL, 0x17000UL }, { 0x187F7UL, 0x187F7UL }, { 0x18800UL, 0x18CD5UL }, - { 0x18D00UL, 0x18D00UL }, + { 0x18CFFUL, 0x18D00UL }, { 0x18D08UL, 0x18D08UL }, { 0x1B000UL, 0x1B122UL }, { 0x1B132UL, 0x1B132UL }, @@ -472,6 +487,8 @@ static const struct range ranges[] = { { 0x1E290UL, 0x1E2ADUL }, { 0x1E2C0UL, 0x1E2EBUL }, { 0x1E4D0UL, 0x1E4EAUL }, + { 0x1E5D0UL, 0x1E5EDUL }, + { 0x1E5F0UL, 0x1E5F0UL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, diff --git a/src/libre/class/utf8_Lu.c b/src/libre/class/utf8_Lu.c index 0e8d00299..150038e7e 100644 --- a/src/libre/class/utf8_Lu.c +++ b/src/libre/class/utf8_Lu.c @@ -279,6 +279,7 @@ static const struct range ranges[] = { { 0x10C7UL, 0x10C7UL }, { 0x10CDUL, 0x10CDUL }, { 0x13A0UL, 0x13F5UL }, + { 0x1C89UL, 0x1C89UL }, { 0x1C90UL, 0x1CBAUL }, { 0x1CBDUL, 0x1CBFUL }, { 0x1E00UL, 0x1E00UL }, @@ -603,9 +604,12 @@ static const struct range ranges[] = { { 0xA7C2UL, 0xA7C2UL }, { 0xA7C4UL, 0xA7C7UL }, { 0xA7C9UL, 0xA7C9UL }, + { 0xA7CBUL, 0xA7CCUL }, { 0xA7D0UL, 0xA7D0UL }, { 0xA7D6UL, 0xA7D6UL }, { 0xA7D8UL, 0xA7D8UL }, + { 0xA7DAUL, 0xA7DAUL }, + { 0xA7DCUL, 0xA7DCUL }, { 0xA7F5UL, 0xA7F5UL }, { 0xFF21UL, 0xFF3AUL }, { 0x10400UL, 0x10427UL }, @@ -615,6 +619,7 @@ static const struct range ranges[] = { { 0x1058CUL, 0x10592UL }, { 0x10594UL, 0x10595UL }, { 0x10C80UL, 0x10CB2UL }, + { 0x10D50UL, 0x10D65UL }, { 0x118A0UL, 0x118BFUL }, { 0x16E40UL, 0x16E5FUL }, { 0x1D400UL, 0x1D419UL }, diff --git a/src/libre/class/utf8_M.c b/src/libre/class/utf8_M.c index 38c00e3f9..f0b3e3af6 100644 --- a/src/libre/class/utf8_M.c +++ b/src/libre/class/utf8_M.c @@ -27,7 +27,7 @@ static const struct range ranges[] = { { 0x0825UL, 0x0827UL }, { 0x0829UL, 0x082DUL }, { 0x0859UL, 0x085BUL }, - { 0x0898UL, 0x089FUL }, + { 0x0897UL, 0x089FUL }, { 0x08CAUL, 0x08E1UL }, { 0x08E3UL, 0x0903UL }, { 0x093AUL, 0x093CUL }, @@ -203,8 +203,9 @@ static const struct range ranges[] = { { 0x10A3FUL, 0x10A3FUL }, { 0x10AE5UL, 0x10AE6UL }, { 0x10D24UL, 0x10D27UL }, + { 0x10D69UL, 0x10D6DUL }, { 0x10EABUL, 0x10EACUL }, - { 0x10EFDUL, 0x10EFFUL }, + { 0x10EFCUL, 0x10EFFUL }, { 0x10F46UL, 0x10F50UL }, { 0x10F82UL, 0x10F85UL }, { 0x11000UL, 0x11002UL }, @@ -235,6 +236,13 @@ static const struct range ranges[] = { { 0x11362UL, 0x11363UL }, { 0x11366UL, 0x1136CUL }, { 0x11370UL, 0x11374UL }, + { 0x113B8UL, 0x113C0UL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113D0UL }, + { 0x113D2UL, 0x113D2UL }, + { 0x113E1UL, 0x113E2UL }, { 0x11435UL, 0x11446UL }, { 0x1145EUL, 0x1145EUL }, { 0x114B0UL, 0x114C3UL }, @@ -276,8 +284,10 @@ static const struct range ranges[] = { { 0x11F03UL, 0x11F03UL }, { 0x11F34UL, 0x11F3AUL }, { 0x11F3EUL, 0x11F42UL }, + { 0x11F5AUL, 0x11F5AUL }, { 0x13440UL, 0x13440UL }, { 0x13447UL, 0x13455UL }, + { 0x1611EUL, 0x1612FUL }, { 0x16AF0UL, 0x16AF4UL }, { 0x16B30UL, 0x16B36UL }, { 0x16F4FUL, 0x16F4FUL }, @@ -310,6 +320,7 @@ static const struct range ranges[] = { { 0x1E2AEUL, 0x1E2AEUL }, { 0x1E2ECUL, 0x1E2EFUL }, { 0x1E4ECUL, 0x1E4EFUL }, + { 0x1E5EEUL, 0x1E5EFUL }, { 0x1E8D0UL, 0x1E8D6UL }, { 0x1E944UL, 0x1E94AUL }, { 0xE0100UL, 0xE01EFUL } diff --git a/src/libre/class/utf8_Mc.c b/src/libre/class/utf8_Mc.c index 752d1bed5..77917668c 100644 --- a/src/libre/class/utf8_Mc.c +++ b/src/libre/class/utf8_Mc.c @@ -137,6 +137,12 @@ static const struct range ranges[] = { { 0x1134BUL, 0x1134DUL }, { 0x11357UL, 0x11357UL }, { 0x11362UL, 0x11363UL }, + { 0x113B8UL, 0x113BAUL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113CDUL }, + { 0x113CFUL, 0x113CFUL }, { 0x11435UL, 0x11437UL }, { 0x11440UL, 0x11441UL }, { 0x11445UL, 0x11445UL }, @@ -153,6 +159,7 @@ static const struct range ranges[] = { { 0x116ACUL, 0x116ACUL }, { 0x116AEUL, 0x116AFUL }, { 0x116B6UL, 0x116B6UL }, + { 0x1171EUL, 0x1171EUL }, { 0x11720UL, 0x11721UL }, { 0x11726UL, 0x11726UL }, { 0x1182CUL, 0x1182EUL }, @@ -181,6 +188,7 @@ static const struct range ranges[] = { { 0x11F34UL, 0x11F35UL }, { 0x11F3EUL, 0x11F3FUL }, { 0x11F41UL, 0x11F41UL }, + { 0x1612AUL, 0x1612CUL }, { 0x16F51UL, 0x16F87UL }, { 0x16FF0UL, 0x16FF1UL }, { 0x1D165UL, 0x1D166UL }, diff --git a/src/libre/class/utf8_Mn.c b/src/libre/class/utf8_Mn.c index 2167dc311..c204fa592 100644 --- a/src/libre/class/utf8_Mn.c +++ b/src/libre/class/utf8_Mn.c @@ -27,7 +27,7 @@ static const struct range ranges[] = { { 0x0825UL, 0x0827UL }, { 0x0829UL, 0x082DUL }, { 0x0859UL, 0x085BUL }, - { 0x0898UL, 0x089FUL }, + { 0x0897UL, 0x089FUL }, { 0x08CAUL, 0x08E1UL }, { 0x08E3UL, 0x0902UL }, { 0x093AUL, 0x093AUL }, @@ -225,8 +225,9 @@ static const struct range ranges[] = { { 0x10A3FUL, 0x10A3FUL }, { 0x10AE5UL, 0x10AE6UL }, { 0x10D24UL, 0x10D27UL }, + { 0x10D69UL, 0x10D6DUL }, { 0x10EABUL, 0x10EACUL }, - { 0x10EFDUL, 0x10EFFUL }, + { 0x10EFCUL, 0x10EFFUL }, { 0x10F46UL, 0x10F50UL }, { 0x10F82UL, 0x10F85UL }, { 0x11001UL, 0x11001UL }, @@ -257,6 +258,11 @@ static const struct range ranges[] = { { 0x11340UL, 0x11340UL }, { 0x11366UL, 0x1136CUL }, { 0x11370UL, 0x11374UL }, + { 0x113BBUL, 0x113C0UL }, + { 0x113CEUL, 0x113CEUL }, + { 0x113D0UL, 0x113D0UL }, + { 0x113D2UL, 0x113D2UL }, + { 0x113E1UL, 0x113E2UL }, { 0x11438UL, 0x1143FUL }, { 0x11442UL, 0x11444UL }, { 0x11446UL, 0x11446UL }, @@ -276,7 +282,8 @@ static const struct range ranges[] = { { 0x116ADUL, 0x116ADUL }, { 0x116B0UL, 0x116B5UL }, { 0x116B7UL, 0x116B7UL }, - { 0x1171DUL, 0x1171FUL }, + { 0x1171DUL, 0x1171DUL }, + { 0x1171FUL, 0x1171FUL }, { 0x11722UL, 0x11725UL }, { 0x11727UL, 0x1172BUL }, { 0x1182FUL, 0x11837UL }, @@ -315,8 +322,11 @@ static const struct range ranges[] = { { 0x11F36UL, 0x11F3AUL }, { 0x11F40UL, 0x11F40UL }, { 0x11F42UL, 0x11F42UL }, + { 0x11F5AUL, 0x11F5AUL }, { 0x13440UL, 0x13440UL }, { 0x13447UL, 0x13455UL }, + { 0x1611EUL, 0x16129UL }, + { 0x1612DUL, 0x1612FUL }, { 0x16AF0UL, 0x16AF4UL }, { 0x16B30UL, 0x16B36UL }, { 0x16F4FUL, 0x16F4FUL }, @@ -346,6 +356,7 @@ static const struct range ranges[] = { { 0x1E2AEUL, 0x1E2AEUL }, { 0x1E2ECUL, 0x1E2EFUL }, { 0x1E4ECUL, 0x1E4EFUL }, + { 0x1E5EEUL, 0x1E5EFUL }, { 0x1E8D0UL, 0x1E8D6UL }, { 0x1E944UL, 0x1E94AUL }, { 0xE0100UL, 0xE01EFUL } diff --git a/src/libre/class/utf8_Myanmar.c b/src/libre/class/utf8_Myanmar.c index deb2f419c..43a91b0d5 100644 --- a/src/libre/class/utf8_Myanmar.c +++ b/src/libre/class/utf8_Myanmar.c @@ -5,7 +5,8 @@ static const struct range ranges[] = { { 0x1000UL, 0x109FUL }, { 0xA9E0UL, 0xA9FEUL }, - { 0xAA60UL, 0xAA7FUL } + { 0xAA60UL, 0xAA7FUL }, + { 0x116D0UL, 0x116E3UL } }; const struct class utf8_Myanmar = { diff --git a/src/libre/class/utf8_N.c b/src/libre/class/utf8_N.c index 3e180474f..a4b7fdebc 100644 --- a/src/libre/class/utf8_N.c +++ b/src/libre/class/utf8_N.c @@ -96,6 +96,7 @@ static const struct range ranges[] = { { 0x10BA9UL, 0x10BAFUL }, { 0x10CFAUL, 0x10CFFUL }, { 0x10D30UL, 0x10D39UL }, + { 0x10D40UL, 0x10D49UL }, { 0x10E60UL, 0x10E7EUL }, { 0x10F1DUL, 0x10F26UL }, { 0x10F51UL, 0x10F54UL }, @@ -110,20 +111,25 @@ static const struct range ranges[] = { { 0x114D0UL, 0x114D9UL }, { 0x11650UL, 0x11659UL }, { 0x116C0UL, 0x116C9UL }, + { 0x116D0UL, 0x116E3UL }, { 0x11730UL, 0x1173BUL }, { 0x118E0UL, 0x118F2UL }, { 0x11950UL, 0x11959UL }, + { 0x11BF0UL, 0x11BF9UL }, { 0x11C50UL, 0x11C6CUL }, { 0x11D50UL, 0x11D59UL }, { 0x11DA0UL, 0x11DA9UL }, { 0x11F50UL, 0x11F59UL }, { 0x11FC0UL, 0x11FD4UL }, { 0x12400UL, 0x1246EUL }, + { 0x16130UL, 0x16139UL }, { 0x16A60UL, 0x16A69UL }, { 0x16AC0UL, 0x16AC9UL }, { 0x16B50UL, 0x16B59UL }, { 0x16B5BUL, 0x16B61UL }, + { 0x16D70UL, 0x16D79UL }, { 0x16E80UL, 0x16E96UL }, + { 0x1CCF0UL, 0x1CCF9UL }, { 0x1D2C0UL, 0x1D2D3UL }, { 0x1D2E0UL, 0x1D2F3UL }, { 0x1D360UL, 0x1D378UL }, @@ -131,6 +137,7 @@ static const struct range ranges[] = { { 0x1E140UL, 0x1E149UL }, { 0x1E2F0UL, 0x1E2F9UL }, { 0x1E4F0UL, 0x1E4F9UL }, + { 0x1E5F1UL, 0x1E5FAUL }, { 0x1E8C7UL, 0x1E8CFUL }, { 0x1E950UL, 0x1E959UL }, { 0x1EC71UL, 0x1ECABUL }, diff --git a/src/libre/class/utf8_Nd.c b/src/libre/class/utf8_Nd.c index 684b0c73f..7506f2d2e 100644 --- a/src/libre/class/utf8_Nd.c +++ b/src/libre/class/utf8_Nd.c @@ -42,6 +42,7 @@ static const struct range ranges[] = { { 0xFF10UL, 0xFF19UL }, { 0x104A0UL, 0x104A9UL }, { 0x10D30UL, 0x10D39UL }, + { 0x10D40UL, 0x10D49UL }, { 0x11066UL, 0x1106FUL }, { 0x110F0UL, 0x110F9UL }, { 0x11136UL, 0x1113FUL }, @@ -51,20 +52,26 @@ static const struct range ranges[] = { { 0x114D0UL, 0x114D9UL }, { 0x11650UL, 0x11659UL }, { 0x116C0UL, 0x116C9UL }, + { 0x116D0UL, 0x116E3UL }, { 0x11730UL, 0x11739UL }, { 0x118E0UL, 0x118E9UL }, { 0x11950UL, 0x11959UL }, + { 0x11BF0UL, 0x11BF9UL }, { 0x11C50UL, 0x11C59UL }, { 0x11D50UL, 0x11D59UL }, { 0x11DA0UL, 0x11DA9UL }, { 0x11F50UL, 0x11F59UL }, + { 0x16130UL, 0x16139UL }, { 0x16A60UL, 0x16A69UL }, { 0x16AC0UL, 0x16AC9UL }, { 0x16B50UL, 0x16B59UL }, + { 0x16D70UL, 0x16D79UL }, + { 0x1CCF0UL, 0x1CCF9UL }, { 0x1D7CEUL, 0x1D7FFUL }, { 0x1E140UL, 0x1E149UL }, { 0x1E2F0UL, 0x1E2F9UL }, { 0x1E4F0UL, 0x1E4F9UL }, + { 0x1E5F1UL, 0x1E5FAUL }, { 0x1E950UL, 0x1E959UL }, { 0x1FBF0UL, 0x1FBF9UL } }; diff --git a/src/libre/class/utf8_Ol_Onal.c b/src/libre/class/utf8_Ol_Onal.c new file mode 100644 index 000000000..7223f22e5 --- /dev/null +++ b/src/libre/class/utf8_Ol_Onal.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x1E5D0UL, 0x1E5FAUL }, + { 0x1E5FFUL, 0x1E5FFUL } +}; + +const struct class utf8_Ol_Onal = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_P.c b/src/libre/class/utf8_P.c index 38e217087..9c506bd13 100644 --- a/src/libre/class/utf8_P.c +++ b/src/libre/class/utf8_P.c @@ -68,8 +68,9 @@ static const struct range ranges[] = { { 0x1A1EUL, 0x1A1FUL }, { 0x1AA0UL, 0x1AA6UL }, { 0x1AA8UL, 0x1AADUL }, + { 0x1B4EUL, 0x1B4FUL }, { 0x1B5AUL, 0x1B60UL }, - { 0x1B7DUL, 0x1B7EUL }, + { 0x1B7DUL, 0x1B7FUL }, { 0x1BFCUL, 0x1BFFUL }, { 0x1C3BUL, 0x1C3FUL }, { 0x1C7EUL, 0x1C7FUL }, @@ -148,6 +149,7 @@ static const struct range ranges[] = { { 0x10AF0UL, 0x10AF6UL }, { 0x10B39UL, 0x10B3FUL }, { 0x10B99UL, 0x10B9CUL }, + { 0x10D6EUL, 0x10D6EUL }, { 0x10EADUL, 0x10EADUL }, { 0x10F55UL, 0x10F59UL }, { 0x10F86UL, 0x10F89UL }, @@ -162,6 +164,8 @@ static const struct range ranges[] = { { 0x111DDUL, 0x111DFUL }, { 0x11238UL, 0x1123DUL }, { 0x112A9UL, 0x112A9UL }, + { 0x113D4UL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, { 0x1144BUL, 0x1144FUL }, { 0x1145AUL, 0x1145BUL }, { 0x1145DUL, 0x1145DUL }, @@ -178,6 +182,7 @@ static const struct range ranges[] = { { 0x11A9AUL, 0x11A9CUL }, { 0x11A9EUL, 0x11AA2UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11BE1UL, 0x11BE1UL }, { 0x11C41UL, 0x11C45UL }, { 0x11C70UL, 0x11C71UL }, { 0x11EF7UL, 0x11EF8UL }, @@ -189,10 +194,12 @@ static const struct range ranges[] = { { 0x16AF5UL, 0x16AF5UL }, { 0x16B37UL, 0x16B3BUL }, { 0x16B44UL, 0x16B44UL }, + { 0x16D6DUL, 0x16D6FUL }, { 0x16E97UL, 0x16E9AUL }, { 0x16FE2UL, 0x16FE2UL }, { 0x1BC9FUL, 0x1BC9FUL }, { 0x1DA87UL, 0x1DA8BUL }, + { 0x1E5FFUL, 0x1E5FFUL }, { 0x1E95EUL, 0x1E95FUL } }; diff --git a/src/libre/class/utf8_Pd.c b/src/libre/class/utf8_Pd.c index 0681ece71..c15bf5021 100644 --- a/src/libre/class/utf8_Pd.c +++ b/src/libre/class/utf8_Pd.c @@ -21,6 +21,7 @@ static const struct range ranges[] = { { 0xFE58UL, 0xFE58UL }, { 0xFE63UL, 0xFE63UL }, { 0xFF0DUL, 0xFF0DUL }, + { 0x10D6EUL, 0x10D6EUL }, { 0x10EADUL, 0x10EADUL } }; diff --git a/src/libre/class/utf8_Po.c b/src/libre/class/utf8_Po.c index 95ccb96b9..ef95b8586 100644 --- a/src/libre/class/utf8_Po.c +++ b/src/libre/class/utf8_Po.c @@ -62,8 +62,9 @@ static const struct range ranges[] = { { 0x1A1EUL, 0x1A1FUL }, { 0x1AA0UL, 0x1AA6UL }, { 0x1AA8UL, 0x1AADUL }, + { 0x1B4EUL, 0x1B4FUL }, { 0x1B5AUL, 0x1B60UL }, - { 0x1B7DUL, 0x1B7EUL }, + { 0x1B7DUL, 0x1B7FUL }, { 0x1BFCUL, 0x1BFFUL }, { 0x1C3BUL, 0x1C3FUL }, { 0x1C7EUL, 0x1C7FUL }, @@ -158,6 +159,8 @@ static const struct range ranges[] = { { 0x111DDUL, 0x111DFUL }, { 0x11238UL, 0x1123DUL }, { 0x112A9UL, 0x112A9UL }, + { 0x113D4UL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, { 0x1144BUL, 0x1144FUL }, { 0x1145AUL, 0x1145BUL }, { 0x1145DUL, 0x1145DUL }, @@ -174,6 +177,7 @@ static const struct range ranges[] = { { 0x11A9AUL, 0x11A9CUL }, { 0x11A9EUL, 0x11AA2UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11BE1UL, 0x11BE1UL }, { 0x11C41UL, 0x11C45UL }, { 0x11C70UL, 0x11C71UL }, { 0x11EF7UL, 0x11EF8UL }, @@ -185,10 +189,12 @@ static const struct range ranges[] = { { 0x16AF5UL, 0x16AF5UL }, { 0x16B37UL, 0x16B3BUL }, { 0x16B44UL, 0x16B44UL }, + { 0x16D6DUL, 0x16D6FUL }, { 0x16E97UL, 0x16E9AUL }, { 0x16FE2UL, 0x16FE2UL }, { 0x1BC9FUL, 0x1BC9FUL }, { 0x1DA87UL, 0x1DA8BUL }, + { 0x1E5FFUL, 0x1E5FFUL }, { 0x1E95EUL, 0x1E95FUL } }; diff --git a/src/libre/class/utf8_S.c b/src/libre/class/utf8_S.c index 1ad790595..0b69e43fe 100644 --- a/src/libre/class/utf8_S.c +++ b/src/libre/class/utf8_S.c @@ -93,7 +93,7 @@ static const struct range ranges[] = { { 0x218AUL, 0x218BUL }, { 0x2190UL, 0x2307UL }, { 0x230CUL, 0x2328UL }, - { 0x232BUL, 0x2426UL }, + { 0x232BUL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x249CUL, 0x24E9UL }, { 0x2500UL, 0x2767UL }, @@ -119,7 +119,7 @@ static const struct range ranges[] = { { 0x309BUL, 0x309CUL }, { 0x3190UL, 0x3191UL }, { 0x3196UL, 0x319FUL }, - { 0x31C0UL, 0x31E3UL }, + { 0x31C0UL, 0x31E5UL }, { 0x31EFUL, 0x31EFUL }, { 0x3200UL, 0x321EUL }, { 0x322AUL, 0x3247UL }, @@ -163,11 +163,14 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x10877UL, 0x10878UL }, { 0x10AC8UL, 0x10AC8UL }, + { 0x10D8EUL, 0x10D8FUL }, { 0x1173FUL, 0x1173FUL }, { 0x11FD5UL, 0x11FF1UL }, { 0x16B3CUL, 0x16B3FUL }, { 0x16B45UL, 0x16B45UL }, { 0x1BC9CUL, 0x1BC9CUL }, + { 0x1CC00UL, 0x1CCEFUL }, + { 0x1CD00UL, 0x1CEB3UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -224,18 +227,18 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, { 0x1F900UL, 0x1FA53UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, + { 0x1FA80UL, 0x1FA89UL }, + { 0x1FA8FUL, 0x1FAC6UL }, + { 0x1FACEUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAE9UL }, { 0x1FAF0UL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL } + { 0x1FB94UL, 0x1FBEFUL } }; const struct class utf8_S = { diff --git a/src/libre/class/utf8_Sm.c b/src/libre/class/utf8_Sm.c index 38e1624d6..9c39b8003 100644 --- a/src/libre/class/utf8_Sm.c +++ b/src/libre/class/utf8_Sm.c @@ -56,6 +56,7 @@ static const struct range ranges[] = { { 0xFF5EUL, 0xFF5EUL }, { 0xFFE2UL, 0xFFE2UL }, { 0xFFE9UL, 0xFFECUL }, + { 0x10D8EUL, 0x10D8FUL }, { 0x1D6C1UL, 0x1D6C1UL }, { 0x1D6DBUL, 0x1D6DBUL }, { 0x1D6FBUL, 0x1D6FBUL }, diff --git a/src/libre/class/utf8_So.c b/src/libre/class/utf8_So.c index 3bb2b2f82..d7e608ca4 100644 --- a/src/libre/class/utf8_So.c +++ b/src/libre/class/utf8_So.c @@ -69,7 +69,7 @@ static const struct range ranges[] = { { 0x232BUL, 0x237BUL }, { 0x237DUL, 0x239AUL }, { 0x23B4UL, 0x23DBUL }, - { 0x23E2UL, 0x2426UL }, + { 0x23E2UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x249CUL, 0x24E9UL }, { 0x2500UL, 0x25B6UL }, @@ -97,7 +97,7 @@ static const struct range ranges[] = { { 0x303EUL, 0x303FUL }, { 0x3190UL, 0x3191UL }, { 0x3196UL, 0x319FUL }, - { 0x31C0UL, 0x31E3UL }, + { 0x31C0UL, 0x31E5UL }, { 0x31EFUL, 0x31EFUL }, { 0x3200UL, 0x321EUL }, { 0x322AUL, 0x3247UL }, @@ -132,6 +132,8 @@ static const struct range ranges[] = { { 0x16B3CUL, 0x16B3FUL }, { 0x16B45UL, 0x16B45UL }, { 0x1BC9CUL, 0x1BC9CUL }, + { 0x1CC00UL, 0x1CCEFUL }, + { 0x1CD00UL, 0x1CEB3UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -176,18 +178,18 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, { 0x1F900UL, 0x1FA53UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, + { 0x1FA80UL, 0x1FA89UL }, + { 0x1FA8FUL, 0x1FAC6UL }, + { 0x1FACEUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAE9UL }, { 0x1FAF0UL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL } + { 0x1FB94UL, 0x1FBEFUL } }; const struct class utf8_So = { diff --git a/src/libre/class/utf8_Sunuwar.c b/src/libre/class/utf8_Sunuwar.c new file mode 100644 index 000000000..ce53148d0 --- /dev/null +++ b/src/libre/class/utf8_Sunuwar.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x11BC0UL, 0x11BE1UL }, + { 0x11BF0UL, 0x11BF9UL } +}; + +const struct class utf8_Sunuwar = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Todhri.c b/src/libre/class/utf8_Todhri.c new file mode 100644 index 000000000..732870a64 --- /dev/null +++ b/src/libre/class/utf8_Todhri.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x105C0UL, 0x105F3UL } +}; + +const struct class utf8_Todhri = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Tulu_Tigalari.c b/src/libre/class/utf8_Tulu_Tigalari.c new file mode 100644 index 000000000..b611066f2 --- /dev/null +++ b/src/libre/class/utf8_Tulu_Tigalari.c @@ -0,0 +1,23 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113C0UL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, + { 0x113E1UL, 0x113E2UL } +}; + +const struct class utf8_Tulu_Tigalari = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_assigned.c b/src/libre/class/utf8_assigned.c index d29b53bf2..d38345570 100644 --- a/src/libre/class/utf8_assigned.c +++ b/src/libre/class/utf8_assigned.c @@ -26,7 +26,7 @@ static const struct range ranges[] = { { 0x0860UL, 0x086AUL }, { 0x0870UL, 0x088EUL }, { 0x0890UL, 0x0891UL }, - { 0x0898UL, 0x0983UL }, + { 0x0897UL, 0x0983UL }, { 0x0985UL, 0x098CUL }, { 0x098FUL, 0x0990UL }, { 0x0993UL, 0x09A8UL }, @@ -218,11 +218,10 @@ static const struct range ranges[] = { { 0x1AA0UL, 0x1AADUL }, { 0x1AB0UL, 0x1ACEUL }, { 0x1B00UL, 0x1B4CUL }, - { 0x1B50UL, 0x1B7EUL }, - { 0x1B80UL, 0x1BF3UL }, + { 0x1B4EUL, 0x1BF3UL }, { 0x1BFCUL, 0x1C37UL }, { 0x1C3BUL, 0x1C49UL }, - { 0x1C4DUL, 0x1C88UL }, + { 0x1C4DUL, 0x1C8AUL }, { 0x1C90UL, 0x1CBAUL }, { 0x1CBDUL, 0x1CC7UL }, { 0x1CD0UL, 0x1CFAUL }, @@ -249,7 +248,7 @@ static const struct range ranges[] = { { 0x20A0UL, 0x20C0UL }, { 0x20D0UL, 0x20F0UL }, { 0x2100UL, 0x218BUL }, - { 0x2190UL, 0x2426UL }, + { 0x2190UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x2460UL, 0x2B73UL }, { 0x2B76UL, 0x2B95UL }, @@ -277,7 +276,7 @@ static const struct range ranges[] = { { 0x3099UL, 0x30FFUL }, { 0x3105UL, 0x312FUL }, { 0x3131UL, 0x318EUL }, - { 0x3190UL, 0x31E3UL }, + { 0x3190UL, 0x31E5UL }, { 0x31EFUL, 0x321EUL }, { 0x3220UL, 0x3400UL }, { 0x4DBFUL, 0x4E00UL }, @@ -285,10 +284,10 @@ static const struct range ranges[] = { { 0xA490UL, 0xA4C6UL }, { 0xA4D0UL, 0xA62BUL }, { 0xA640UL, 0xA6F7UL }, - { 0xA700UL, 0xA7CAUL }, + { 0xA700UL, 0xA7CDUL }, { 0xA7D0UL, 0xA7D1UL }, { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7D9UL }, + { 0xA7D5UL, 0xA7DCUL }, { 0xA7F2UL, 0xA82CUL }, { 0xA830UL, 0xA839UL }, { 0xA840UL, 0xA877UL }, @@ -384,6 +383,7 @@ static const struct range ranges[] = { { 0x105A3UL, 0x105B1UL }, { 0x105B3UL, 0x105B9UL }, { 0x105BBUL, 0x105BCUL }, + { 0x105C0UL, 0x105F3UL }, { 0x10600UL, 0x10736UL }, { 0x10740UL, 0x10755UL }, { 0x10760UL, 0x10767UL }, @@ -427,11 +427,15 @@ static const struct range ranges[] = { { 0x10CC0UL, 0x10CF2UL }, { 0x10CFAUL, 0x10D27UL }, { 0x10D30UL, 0x10D39UL }, + { 0x10D40UL, 0x10D65UL }, + { 0x10D69UL, 0x10D85UL }, + { 0x10D8EUL, 0x10D8FUL }, { 0x10E60UL, 0x10E7EUL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EABUL, 0x10EADUL }, { 0x10EB0UL, 0x10EB1UL }, - { 0x10EFDUL, 0x10F27UL }, + { 0x10EC2UL, 0x10EC4UL }, + { 0x10EFCUL, 0x10F27UL }, { 0x10F30UL, 0x10F59UL }, { 0x10F70UL, 0x10F89UL }, { 0x10FB0UL, 0x10FCBUL }, @@ -471,6 +475,17 @@ static const struct range ranges[] = { { 0x1135DUL, 0x11363UL }, { 0x11366UL, 0x1136CUL }, { 0x11370UL, 0x11374UL }, + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113C0UL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, + { 0x113E1UL, 0x113E2UL }, { 0x11400UL, 0x1145BUL }, { 0x1145DUL, 0x11461UL }, { 0x11480UL, 0x114C7UL }, @@ -482,6 +497,7 @@ static const struct range ranges[] = { { 0x11660UL, 0x1166CUL }, { 0x11680UL, 0x116B9UL }, { 0x116C0UL, 0x116C9UL }, + { 0x116D0UL, 0x116E3UL }, { 0x11700UL, 0x1171AUL }, { 0x1171DUL, 0x1172BUL }, { 0x11730UL, 0x11746UL }, @@ -502,6 +518,8 @@ static const struct range ranges[] = { { 0x11A50UL, 0x11AA2UL }, { 0x11AB0UL, 0x11AF8UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11BC0UL, 0x11BE1UL }, + { 0x11BF0UL, 0x11BF9UL }, { 0x11C00UL, 0x11C08UL }, { 0x11C0AUL, 0x11C36UL }, { 0x11C38UL, 0x11C45UL }, @@ -525,7 +543,7 @@ static const struct range ranges[] = { { 0x11EE0UL, 0x11EF8UL }, { 0x11F00UL, 0x11F10UL }, { 0x11F12UL, 0x11F3AUL }, - { 0x11F3EUL, 0x11F59UL }, + { 0x11F3EUL, 0x11F5AUL }, { 0x11FB0UL, 0x11FB0UL }, { 0x11FC0UL, 0x11FF1UL }, { 0x11FFFUL, 0x12399UL }, @@ -534,7 +552,9 @@ static const struct range ranges[] = { { 0x12480UL, 0x12543UL }, { 0x12F90UL, 0x12FF2UL }, { 0x13000UL, 0x13455UL }, + { 0x13460UL, 0x143FAUL }, { 0x14400UL, 0x14646UL }, + { 0x16100UL, 0x16139UL }, { 0x16800UL, 0x16A38UL }, { 0x16A40UL, 0x16A5EUL }, { 0x16A60UL, 0x16A69UL }, @@ -547,6 +567,7 @@ static const struct range ranges[] = { { 0x16B5BUL, 0x16B61UL }, { 0x16B63UL, 0x16B77UL }, { 0x16B7DUL, 0x16B8FUL }, + { 0x16D40UL, 0x16D79UL }, { 0x16E40UL, 0x16E9AUL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F4FUL, 0x16F87UL }, @@ -556,7 +577,7 @@ static const struct range ranges[] = { { 0x17000UL, 0x17000UL }, { 0x187F7UL, 0x187F7UL }, { 0x18800UL, 0x18CD5UL }, - { 0x18D00UL, 0x18D00UL }, + { 0x18CFFUL, 0x18D00UL }, { 0x18D08UL, 0x18D08UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, @@ -572,6 +593,8 @@ static const struct range ranges[] = { { 0x1BC80UL, 0x1BC88UL }, { 0x1BC90UL, 0x1BC99UL }, { 0x1BC9CUL, 0x1BCA3UL }, + { 0x1CC00UL, 0x1CCF9UL }, + { 0x1CD00UL, 0x1CEB3UL }, { 0x1CF00UL, 0x1CF2DUL }, { 0x1CF30UL, 0x1CF46UL }, { 0x1CF50UL, 0x1CFC3UL }, @@ -623,6 +646,8 @@ static const struct range ranges[] = { { 0x1E2C0UL, 0x1E2F9UL }, { 0x1E2FFUL, 0x1E2FFUL }, { 0x1E4D0UL, 0x1E4F9UL }, + { 0x1E5D0UL, 0x1E5FAUL }, + { 0x1E5FFUL, 0x1E5FFUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -692,19 +717,18 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, { 0x1F900UL, 0x1FA53UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, + { 0x1FA80UL, 0x1FA89UL }, + { 0x1FA8FUL, 0x1FAC6UL }, + { 0x1FACEUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAE9UL }, { 0x1FAF0UL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL }, - { 0x1FBF0UL, 0x1FBF9UL }, + { 0x1FB94UL, 0x1FBF9UL }, { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, diff --git a/src/libre/class_name.c b/src/libre/class_name.c index 30c5aaa56..d96a1d610 100644 --- a/src/libre/class_name.c +++ b/src/libre/class_name.c @@ -67,7 +67,7 @@ static struct { { &utf8_Coptic, "Coptic" }, { &utf8_Cuneiform, "Cuneiform" }, { &utf8_Cypriot, "Cypriot" }, - { &utf8_Cypro_Minoan, "Cypro-Minoan" }, + { &utf8_Cypro_Minoan, "Cypro Minoan" }, { &utf8_Cyrillic, "Cyrillic" }, { &utf8_Deseret, "Deseret" }, { &utf8_Devanagari, "Devanagari" }, @@ -78,6 +78,7 @@ static struct { { &utf8_Elbasan, "Elbasan" }, { &utf8_Elymaic, "Elymaic" }, { &utf8_Ethiopic, "Ethiopic" }, + { &utf8_Garay, "Garay" }, { &utf8_Georgian, "Georgian" }, { &utf8_Glagolitic, "Glagolitic" }, { &utf8_Gothic, "Gothic" }, @@ -86,6 +87,7 @@ static struct { { &utf8_Gujarati, "Gujarati" }, { &utf8_Gunjala_Gondi, "Gunjala Gondi" }, { &utf8_Gurmukhi, "Gurmukhi" }, + { &utf8_Gurung_Khema, "Gurung Khema" }, { &utf8_Han, "Han" }, { &utf8_Hangul, "Hangul" }, { &utf8_Hanifi_Rohingya, "Hanifi Rohingya" }, @@ -101,12 +103,14 @@ static struct { { &utf8_Kaithi, "Kaithi" }, { &utf8_Kannada, "Kannada" }, { &utf8_Katakana, "Katakana" }, + { &utf8_Kawi, "Kawi" }, { &utf8_Kayah_Li, "Kayah Li" }, { &utf8_Kharoshthi, "Kharoshthi" }, { &utf8_Khitan_Small_Script, "Khitan Small Script" }, { &utf8_Khmer, "Khmer" }, { &utf8_Khojki, "Khojki" }, { &utf8_Khudawadi, "Khudawadi" }, + { &utf8_Kirat_Rai, "Kirat Rai" }, { &utf8_Lao, "Lao" }, { &utf8_Latin, "Latin" }, { &utf8_Lepcha, "Lepcha" }, @@ -135,9 +139,10 @@ static struct { { &utf8_Multani, "Multani" }, { &utf8_Myanmar, "Myanmar" }, { &utf8_Nabataean, "Nabataean" }, + { &utf8_Nag_Mundari, "Nag Mundari" }, { &utf8_Nandinagari, "Nandinagari" }, - { &utf8_New_Tai_Lue, "New Tai Lue" }, { &utf8_Newa, "Newa" }, + { &utf8_New_Tai_Lue, "New Tai Lue" }, { &utf8_Nko, "Nko" }, { &utf8_Nushu, "Nushu" }, { &utf8_Nyiakeng_Puachue_Hmong, "Nyiakeng Puachue Hmong" }, @@ -152,6 +157,7 @@ static struct { { &utf8_Old_South_Arabian, "Old South Arabian" }, { &utf8_Old_Turkic, "Old Turkic" }, { &utf8_Old_Uyghur, "Old Uyghur" }, + { &utf8_Ol_Onal, "Ol Onal" }, { &utf8_Oriya, "Oriya" }, { &utf8_Osage, "Osage" }, { &utf8_Osmanya, "Osmanya" }, @@ -174,6 +180,7 @@ static struct { { &utf8_Sora_Sompeng, "Sora Sompeng" }, { &utf8_Soyombo, "Soyombo" }, { &utf8_Sundanese, "Sundanese" }, + { &utf8_Sunuwar, "Sunuwar" }, { &utf8_Syloti_Nagri, "Syloti Nagri" }, { &utf8_Syriac, "Syriac" }, { &utf8_Tagalog, "Tagalog" }, @@ -183,15 +190,17 @@ static struct { { &utf8_Tai_Viet, "Tai Viet" }, { &utf8_Takri, "Takri" }, { &utf8_Tamil, "Tamil" }, - { &utf8_Tangut, "Tangut" }, { &utf8_Tangsa, "Tangsa" }, + { &utf8_Tangut, "Tangut" }, { &utf8_Telugu, "Telugu" }, { &utf8_Thaana, "Thaana" }, { &utf8_Thai, "Thai" }, { &utf8_Tibetan, "Tibetan" }, { &utf8_Tifinagh, "Tifinagh" }, { &utf8_Tirhuta, "Tirhuta" }, + { &utf8_Todhri, "Todhri" }, { &utf8_Toto, "Toto" }, + { &utf8_Tulu_Tigalari, "Tulu Tigalari" }, { &utf8_Ugaritic, "Ugaritic" }, { &utf8_Vai, "Vai" }, { &utf8_Vithkuqi, "Vithkuqi" }, From 9aa1b9ac870fa74074f2c5b9b5615cf4de874d07 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 17 Sep 2024 16:21:36 -0400 Subject: [PATCH 265/373] fsm_compact_states must remap endids, to avoid dangling references. Add a regression test showing possible endid false negatives when FSMs were trimmed (called from fsm_minimise) without updating endids. --- src/libfsm/endids.c | 62 ++++++++++++++++++ src/libfsm/endids.h | 3 + src/libfsm/state.c | 7 +- .../endids/endids_trim_must_compact_endids.c | 64 +++++++++++++++++++ 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 tests/endids/endids_trim_must_compact_endids.c diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 05f49b3d2..d0dfc9e38 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -880,6 +880,8 @@ fsm_endid_iter_bulk(const struct fsm *fsm, return 1; } + const fsm_state_t state_count = fsm_countstates(fsm); + bucket_count = ei->bucket_count; for (b_i = 0; b_i < bucket_count; b_i++) { @@ -891,6 +893,7 @@ fsm_endid_iter_bulk(const struct fsm *fsm, count = b->ids->count; + assert(b->state < state_count); if (!cb(b->state, &b->ids->ids[0], count, opaque)) { return 0; } @@ -1026,3 +1029,62 @@ fsm_increndids(struct fsm * fsm, int delta) fsm_mapendids(fsm, incr_remap, &delta); } +int +fsm_endid_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count) +{ + /* Don't reallocate unless something has actually changed. */ + bool changes = false; + for (size_t i = 0; i < mapping_count; i++) { + if (mapping[i] != i) { + changes = true; + break; + } + } + + /* nothing to do */ + if (!changes) { return 1; } + + struct endid_info *ei = fsm->endid_info; + + struct endid_info_bucket *nbuckets = f_malloc(fsm->alloc, + ei->bucket_count * sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + + const uint64_t mask = ei->bucket_count - 1; + assert((ei->bucket_count & mask) == 0); + + /* initialize to empty */ + for (size_t nb_i = 0; nb_i < ei->bucket_count; nb_i++) { + nbuckets[nb_i].state = BUCKET_NO_STATE; + } + + for (size_t ob_i = 0; ob_i < ei->bucket_count; ob_i++) { + const struct endid_info_bucket *ob = &ei->buckets[ob_i]; + if (ob->state == BUCKET_NO_STATE) { continue; } + + assert(ob->state < mapping_count); + const fsm_state_t nstate = mapping[ob->state]; + if (nstate == FSM_STATE_REMAP_NO_STATE) { continue; } + + const uint64_t hash = hash_id(nstate); + + bool placed = false; + for (size_t probes = 0; probes < ei->bucket_count; probes++) { + const size_t nb_i = (hash + probes) & mask; + struct endid_info_bucket *nb = &nbuckets[nb_i]; + if (nb->state == BUCKET_NO_STATE) { + nb->state = nstate; + nb->ids = ob->ids; + placed = true; + break; + } + } + assert(placed); + } + + f_free(fsm->alloc, ei->buckets); + ei->buckets = nbuckets; + return 1; +} diff --git a/src/libfsm/endids.h b/src/libfsm/endids.h index a921f7a93..944ec1df2 100644 --- a/src/libfsm/endids.h +++ b/src/libfsm/endids.h @@ -63,4 +63,7 @@ fsm_endid_iter_state(const struct fsm *fsm, fsm_state_t state, void fsm_endid_dump(FILE *f, const struct fsm *fsm); +int +fsm_endid_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); + #endif diff --git a/src/libfsm/state.c b/src/libfsm/state.c index 47b7b5395..c845cbe46 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -18,6 +18,7 @@ #include #include "internal.h" +#include "endids.h" int fsm_addstate(struct fsm *fsm, fsm_state_t *state) @@ -179,7 +180,7 @@ fsm_compact_states(struct fsm *fsm, const fsm_state_t orig_statecount = fsm->statecount; fsm_state_t *mapping = f_malloc(fsm->alloc, - fsm->statecount * sizeof(mapping[0])); + orig_statecount * sizeof(mapping[0])); if (mapping == NULL) { return 0; } @@ -254,6 +255,10 @@ fsm_compact_states(struct fsm *fsm, } } + /* Remap end metadata */ + if (!fsm_endid_compact(fsm, mapping, orig_statecount)) { + return 0; + } assert(dst == kept); assert(kept == fsm->statecount); diff --git a/tests/endids/endids_trim_must_compact_endids.c b/tests/endids/endids_trim_must_compact_endids.c new file mode 100644 index 000000000..13e524993 --- /dev/null +++ b/tests/endids/endids_trim_must_compact_endids.c @@ -0,0 +1,64 @@ +#include +#include + +#include + +#include +#include + +/* Regression: When fsm_trim called fsm_compact_states it previously + * didn't remap endids, so the endid collection could refer to stale + * state IDs or past the end of fsm->states[]. */ +int main(void) +{ + size_t limit = 100; + const fsm_end_id_t end_id = 12345; + + /* Create FSMs with a bunch of garbage states, to increase + * the likelihood that the endid's state not being updated + * after calling fsm_trim triggers the bug. */ + for (size_t end_state = 1; end_state < limit; end_state++) { + struct fsm *fsm = fsm_new(NULL); + assert(fsm != NULL); + + for (size_t i = 0; i < limit; i++) { + if (!fsm_addstate(fsm, NULL)) { + return EXIT_FAILURE; + } + } + + const fsm_state_t start_state = end_state - 1; + fsm_setstart(fsm, start_state); + fsm_setend(fsm, end_state, 1); + + /* add an any edge, so the end state is reachable */ + if (!fsm_addedge_any(fsm, start_state, end_state)) { + return EXIT_FAILURE; + } + + if (!fsm_setendid(fsm, end_id)) { + return EXIT_FAILURE; + } + + /* Call fsm_trim, as fsm_minimise would. */ + (void)fsm_trim(fsm, FSM_TRIM_START_AND_END_REACHABLE, NULL); + + /* Execute the fsm and check that the endid's state was updated. */ + fsm_state_t end; + const char *input = "x"; + int ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + assert(ret == 1); + + size_t count = fsm_endid_count(fsm, end); + assert(count == 1); + + fsm_end_id_t id_buf[1]; + if (!fsm_endid_get(fsm, end, 1, id_buf)) { + return EXIT_FAILURE; + } + assert(id_buf[0] == end_id); + + fsm_free(fsm); + } + return EXIT_SUCCESS; +} From 3295240bf6703982838f17eb999ee45fc33d0864 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 18 Sep 2024 16:49:41 -0400 Subject: [PATCH 266/373] Change fsm_detect_required_characters interface. `struct bm` isn't part of the public API, so use a uint64_t[4] instead, and add an optional parameter for the count. Update the tests. --- include/fsm/walk.h | 6 ++--- src/libfsm/detect_required.c | 24 ++++++++++++------- .../detect_required_step_limit.c | 8 +++---- tests/detect_required/testutil.c | 23 ++++++++---------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/include/fsm/walk.h b/include/fsm/walk.h index bc030a017..e1ab5f29e 100644 --- a/include/fsm/walk.h +++ b/include/fsm/walk.h @@ -141,8 +141,8 @@ fsm_generate_matches_cb fsm_generate_cb_printf_escaped; * check and return ERROR_MISUSE if it is not, otherwise this is an * unchecked error. * - * The bitmap will be cleared before populating. Afterward, - * bm_count(bitmap) will return how many required characters were + * The character map will be cleared before populating. If *count is + * non-NULL it will be updated with how many required characters were * found. * * There is an optional step_limit -- if this is reached, then it will @@ -157,7 +157,7 @@ enum fsm_detect_required_characters_res { }; enum fsm_detect_required_characters_res fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, - struct bm *bitmap); + uint64_t charmap[4], size_t *count); #endif diff --git a/src/libfsm/detect_required.c b/src/libfsm/detect_required.c index 3701c7150..8c152e6cc 100644 --- a/src/libfsm/detect_required.c +++ b/src/libfsm/detect_required.c @@ -95,10 +95,11 @@ static void check_symbols(const struct edge_group_iter_info *info, uint16_t *lab * edges with only one label that must be followed by all matches), so it can take * prohibitively long for large/complex DFAs. */ enum fsm_detect_required_characters_res -fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, struct bm *bitmap) +fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, + uint64_t charmap[4], size_t *count) { assert(dfa != NULL); - assert(bitmap != NULL); + assert(charmap != NULL); #if EXPENSIVE_CHECKS if (!fsm_all(dfa, fsm_isdfa)) { @@ -128,7 +129,7 @@ fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, struct } #endif - bm_clear(bitmap); + for (size_t i = 0; i < 4; i++) { charmap[i] = 0; } /* If the start state is also an end state, then * it matches the empty string, so we're done. */ @@ -193,8 +194,8 @@ fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, struct if (nsf->label != LABEL_GROUP) { size_t offset = (nsf->label & 0xff); - size_t count = ++env.counts[offset]; - if (count == 1) { + const size_t label_count = ++env.counts[offset]; + if (label_count == 1) { bm_set(&env.current, offset); } } @@ -233,8 +234,8 @@ fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, struct * it from the constraint set. */ if (sf->label != LABEL_GROUP) { size_t offset = (sf->label & 0xff); - size_t count = --env.counts[offset]; - if (count == 0) { + const size_t label_count = --env.counts[offset]; + if (label_count == 0) { bm_unset(&env.current, offset); } } @@ -250,8 +251,15 @@ fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, struct fprintf(stderr, "%s: finished in %zu/%zu steps\n", __func__, env.steps, step_limit); } + for (size_t i = 0; i < 4; i++) { + charmap[i] = *bm_nth_word(&env.overall, i); + } + + if (count != NULL) { + *count = bm_count(&env.overall); + } + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; - bm_copy(bitmap, &env.overall); cleanup: f_free(dfa->alloc, env.stack.frames); diff --git a/tests/detect_required/detect_required_step_limit.c b/tests/detect_required/detect_required_step_limit.c index 42c1c0f5e..743173f90 100644 --- a/tests/detect_required/detect_required_step_limit.c +++ b/tests/detect_required/detect_required_step_limit.c @@ -4,7 +4,6 @@ #include #include #include -#include #include int main() @@ -25,7 +24,7 @@ int main() return EXIT_FAILURE; } - struct bm bitmap; + uint64_t charmap[4]; /* keep decreasing the step limit until it's hit, and check that * the bitmap is cleared. */ @@ -34,14 +33,13 @@ int main() while (!hit_step_limit) { assert(step_limit > 0); - const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, &bitmap); + const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, charmap, NULL); if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { hit_step_limit = true; /* this should not contain any partially complete information */ for (size_t i = 0; i < 4; i++) { - const uint64_t *w = bm_nth_word(&bitmap, i); - if (*w != 0) { + if (charmap[i] != 0) { fprintf(stderr, "-- Test failure: partial information set when step limit reached\n"); return EXIT_FAILURE; } diff --git a/tests/detect_required/testutil.c b/tests/detect_required/testutil.c index a9682ee7d..a6ec8c296 100644 --- a/tests/detect_required/testutil.c +++ b/tests/detect_required/testutil.c @@ -6,11 +6,11 @@ #include #include #include -#include #include - #include +#include + bool run_test(const struct testcase *tc) { @@ -38,8 +38,7 @@ run_test(const struct testcase *tc) return false; } - struct bm bitmap; - bm_clear(&bitmap); + uint64_t charmap[4]; { const size_t statecount = fsm_countstates(fsm); @@ -53,7 +52,7 @@ run_test(const struct testcase *tc) } - const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, &bitmap); + const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, charmap, NULL); if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { fprintf(stderr, "-- step limit reached, halting\n"); goto cleanup; @@ -62,14 +61,12 @@ run_test(const struct testcase *tc) char buf[257] = {0}; size_t used = 0; - assert(!bm_get(&bitmap, 0)); /* does not contain 0x00 */ - - int i = 0; - for (;;) { - const size_t next = bm_next(&bitmap, i, 1); - if (next > UCHAR_MAX) { break; } - buf[used++] = (char)next; - i = next; + assert(!u64bitset_get(charmap, 0)); /* does not contain 0x00 */ + + for (size_t i = 0; i < 256; i++) { + if (u64bitset_get(charmap, i)) { + buf[used++] = (char)i; + } } if (0 != strcmp(required, buf)) { From 5dcdeb663bbbe4c443b021998aca77865151ff0d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 19 Sep 2024 10:00:18 -0400 Subject: [PATCH 267/373] fsm_detect_required_characters: Set count to 0 when matching "". --- src/libfsm/detect_required.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libfsm/detect_required.c b/src/libfsm/detect_required.c index 8c152e6cc..8488376fc 100644 --- a/src/libfsm/detect_required.c +++ b/src/libfsm/detect_required.c @@ -135,6 +135,9 @@ fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, * it matches the empty string, so we're done. */ if (fsm_isend(dfa, start_state)) { res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + if (count != NULL) { + *count = 0; + } goto cleanup; } From 208de5fdf51c1b06956b4fa015a87aef46561e0d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 19 Sep 2024 11:34:50 -0400 Subject: [PATCH 268/373] Move detect_required1.c tests to .re/.txt files. Add -q chars to fsm. --- src/fsm/main.c | 50 +++++++++++++++++++++++- tests/detect_required/Makefile | 43 ++++++++++++++------ tests/detect_required/detect_required1.c | 32 --------------- tests/detect_required/in0.re | 1 + tests/detect_required/in1.re | 1 + tests/detect_required/in2.re | 1 + tests/detect_required/in3.re | 1 + tests/detect_required/in4.re | 1 + tests/detect_required/in5.re | 1 + tests/detect_required/in6.re | 1 + tests/detect_required/in7.re | 1 + tests/detect_required/in8.re | 1 + tests/detect_required/in9.re | 1 + tests/detect_required/out0.txt | 1 + tests/detect_required/out1.txt | 1 + tests/detect_required/out2.txt | 1 + tests/detect_required/out3.txt | 1 + tests/detect_required/out4.txt | 1 + tests/detect_required/out5.txt | 1 + tests/detect_required/out6.txt | 1 + tests/detect_required/out7.txt | 1 + tests/detect_required/out8.txt | 1 + tests/detect_required/out9.txt | 1 + 23 files changed, 98 insertions(+), 47 deletions(-) delete mode 100644 tests/detect_required/detect_required1.c create mode 100644 tests/detect_required/in0.re create mode 100644 tests/detect_required/in1.re create mode 100644 tests/detect_required/in2.re create mode 100644 tests/detect_required/in3.re create mode 100644 tests/detect_required/in4.re create mode 100644 tests/detect_required/in5.re create mode 100644 tests/detect_required/in6.re create mode 100644 tests/detect_required/in7.re create mode 100644 tests/detect_required/in8.re create mode 100644 tests/detect_required/in9.re create mode 100644 tests/detect_required/out0.txt create mode 100644 tests/detect_required/out1.txt create mode 100644 tests/detect_required/out2.txt create mode 100644 tests/detect_required/out3.txt create mode 100644 tests/detect_required/out4.txt create mode 100644 tests/detect_required/out5.txt create mode 100644 tests/detect_required/out6.txt create mode 100644 tests/detect_required/out7.txt create mode 100644 tests/detect_required/out8.txt create mode 100644 tests/detect_required/out9.txt diff --git a/src/fsm/main.c b/src/fsm/main.c index 194a18bcd..da65791dd 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -25,6 +25,7 @@ #include #include /* XXX */ +#include #include "libfsm/internal.h" /* XXX */ @@ -101,6 +102,16 @@ query_epsilonclosure(const struct fsm *fsm, fsm_state_t state) abort(); } +static int +query_required_chars(const struct fsm *fsm, fsm_state_t state) +{ + (void) fsm; + (void) state; + + /* never called */ + abort(); +} + static void usage(void) { @@ -227,7 +238,9 @@ static int { "hasambiguity", fsm_has, fsm_hasnondeterminism }, { "hasnondeterminism", fsm_has, fsm_hasnondeterminism }, { "hasepsilons", fsm_has, fsm_hasepsilons }, - { "epsilons", fsm_has, fsm_hasepsilons } + { "epsilons", fsm_has, fsm_hasepsilons }, + { "requiredchars", NULL, query_required_chars }, + { "chars", NULL, query_required_chars }, }; assert(name != NULL); @@ -378,6 +391,7 @@ main(int argc, char *argv[]) int xfiles; int r; size_t generate_bounds = 0; + size_t step_limit = 0; int (*query)(const struct fsm *, fsm_state_t); int (*walk )(const struct fsm *, @@ -404,7 +418,7 @@ main(int argc, char *argv[]) { int c; - while (c = getopt(argc, argv, "h" "aCcgwXe:k:i:" "xpq:l:dG:mrt:EU:W:"), c != -1) { + while (c = getopt(argc, argv, "h" "aCcgwXe:k:i:" "xpq:l:dG:mrt:ES:U:W:"), c != -1) { switch (c) { case 'a': opt.anonymous_states = 1; break; case 'c': opt.consolidate_edges = 1; break; @@ -451,6 +465,10 @@ main(int argc, char *argv[]) } break; + case 'S': + step_limit = strtoul(optarg, NULL, 10); + break; /* can be 0 */ + case 'h': usage(); exit(EXIT_SUCCESS); @@ -669,6 +687,34 @@ main(int argc, char *argv[]) closure_free(fsm, closures, fsm->statecount); return 0; + } else if (query == query_required_chars) { + assert(walk == NULL); + uint64_t charmap[4]; + size_t count; + enum fsm_detect_required_characters_res res; + res = fsm_detect_required_characters(fsm, step_limit, charmap, &count); + if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { + fprintf(stderr, "fsm_detect_required_characters: step limit reached (%zd)\n", step_limit); + exit(EXIT_FAILURE); + } else { + assert(res == FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN); + char buf[257] = {0}; + size_t used = 0; + for (size_t i = 0; i < 256; i++) { + if (u64bitset_get(charmap, i)) { + buf[used++] = (char)i; + } + } + printf("%zd ", count); + for (size_t i = 0; i < used; i++) { + c_escputc_str(stdout, &opt, buf[i]); + } + printf("\n"); + + fsm_free(fsm); + fsm_to_cleanup = NULL; + return EXIT_SUCCESS; + } } else { assert(walk != NULL); r |= !walk(fsm, query); diff --git a/tests/detect_required/Makefile b/tests/detect_required/Makefile index 34214f07e..06d76277e 100644 --- a/tests/detect_required/Makefile +++ b/tests/detect_required/Makefile @@ -1,26 +1,43 @@ .include "../../share/mk/top.mk" TEST.tests/detect_required != ls -1 tests/detect_required/detect_required*.c +TEST.tests/detect_required != ls -1 tests/detect_required/out*.txt TEST_SRCDIR.tests/detect_required = tests/detect_required TEST_OUTDIR.tests/detect_required = ${BUILD}/tests/detect_required -.for n in ${TEST.tests/detect_required:T:R:C/^detect_required//} -test:: ${TEST_OUTDIR.tests/detect_required}/res${n} -SRC += ${TEST_SRCDIR.tests/detect_required}/detect_required${n}.c -CFLAGS.${TEST_SRCDIR.tests/detect_required}/detect_required${n}.c = -UNDEBUG +FSM=${BUILD}/bin/fsm +RE=${BUILD}/bin/re -${TEST_OUTDIR.tests/detect_required}/run${n}: ${TEST_OUTDIR.tests/detect_required}/detect_required${n}.o ${TEST_OUTDIR.tests/detect_required}/testutil.o - ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/detect_required}/run${n} ${TEST_OUTDIR.tests/detect_required}/detect_required${n}.o ${TEST_OUTDIR.tests/detect_required}/testutil.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a +${TEST_OUTDIR.tests/detect_required}/testutil.o: tests/detect_required/testutil.c + ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/detect_required}/testutil.o tests/detect_required/testutil.c -${TEST_OUTDIR.tests/detect_required}/detect_required${n}.o: tests/detect_required/testutil.h +.for n in ${TEST.tests/detect_required:T:Mout*.txt:R:C/^out//} -${TEST_OUTDIR.tests/detect_required}/res${n}: ${TEST_OUTDIR.tests/detect_required}/run${n} - ( ${TEST_OUTDIR.tests/detect_required}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/detect_required}/res${n} +${TEST_OUTDIR.tests/detect_required}/got${n}.txt: ${TEST_SRCDIR.tests/detect_required}/in${n}.re + ( ${RE} -r pcre -l fsm -p -y ${.ALLSRC:M*.re} | ${FSM} -q chars ) \ + > $@ -.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} -${TEST_OUTDIR.tests/detect_required}/run${n}: ${BUILD}/lib/${lib:R}.a +${TEST_OUTDIR.tests/detect_required}/res${n}: \ + ${TEST_SRCDIR.tests/detect_required}/out${n}.txt \ + ${TEST_OUTDIR.tests/detect_required}/got${n}.txt + +TXTTEST_RESULT += ${TEST_OUTDIR.tests/detect_required}/res${n} .endfor + + +test:: ${TEST_OUTDIR.tests/detect_required}/res_step_limit +SRC += ${TEST_SRCDIR.tests/detect_required}/detect_required_step_limit.c +CFLAGS.${TEST_SRCDIR.tests/detect_required}/detect_required_step_limit.c = -UNDEBUG + +${TEST_OUTDIR.tests/detect_required}/run_step_limit: ${TEST_OUTDIR.tests/detect_required}/detect_required_step_limit.o ${TEST_OUTDIR.tests/detect_required}/testutil.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/detect_required}/run_step_limit ${TEST_OUTDIR.tests/detect_required}/detect_required_step_limit.o ${TEST_OUTDIR.tests/detect_required}/testutil.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + +${TEST_OUTDIR.tests/detect_required}/detect_required_step_limit.o: tests/detect_required/testutil.h + +${TEST_OUTDIR.tests/detect_required}/res_step_limit: ${TEST_OUTDIR.tests/detect_required}/run_step_limit + ( ${TEST_OUTDIR.tests/detect_required}/run_step_limit 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/detect_required}/res_step_limit + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${TEST_OUTDIR.tests/detect_required}/run_step_limit: ${BUILD}/lib/${lib:R}.a .endfor -${TEST_OUTDIR.tests/detect_required}/testutil.o: tests/detect_required/testutil.c - ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/detect_required}/testutil.o tests/detect_required/testutil.c diff --git a/tests/detect_required/detect_required1.c b/tests/detect_required/detect_required1.c deleted file mode 100644 index 52bb83477..000000000 --- a/tests/detect_required/detect_required1.c +++ /dev/null @@ -1,32 +0,0 @@ -#include "testutil.h" - -const struct testcase tests[] = { - { .regex = "^$", .required = "" }, - { .regex = "^a$", .required = "a" }, - { .regex = "^abcde$", .required = "abcde" }, - { .regex = "^(ab|cd)$", .required = "" }, - { .regex = "^(ab|cd|ef)$", .required = "" }, - { .regex = "^(abc|def)$", .required = "" }, - { .regex = "^(abc|dbf)$", .required = "b" }, - { .regex = "^abc(def)*ghi$", .required = "abcghi" }, - { .regex = "^abc(def)+ghi$", .required = "abcdefghi" }, - { .regex = "^ghi(def)abc$", .required = "abcdefghi" }, -}; - -int main() -{ - const bool first_fail = getenv("FIRST_FAIL") != NULL; - const size_t testcount = sizeof(tests)/sizeof(tests[0]); - - size_t failures = 0; - for (size_t i = 0; i < testcount; i++) { - if (!run_test(&tests[i])) { - failures++; - if (first_fail) { break; } - } - } - - return failures == 0 - ? EXIT_SUCCESS - : EXIT_FAILURE; -} diff --git a/tests/detect_required/in0.re b/tests/detect_required/in0.re new file mode 100644 index 000000000..5d762054a --- /dev/null +++ b/tests/detect_required/in0.re @@ -0,0 +1 @@ +^$ \ No newline at end of file diff --git a/tests/detect_required/in1.re b/tests/detect_required/in1.re new file mode 100644 index 000000000..25081a78c --- /dev/null +++ b/tests/detect_required/in1.re @@ -0,0 +1 @@ +^a$ \ No newline at end of file diff --git a/tests/detect_required/in2.re b/tests/detect_required/in2.re new file mode 100644 index 000000000..fe1ec125a --- /dev/null +++ b/tests/detect_required/in2.re @@ -0,0 +1 @@ +^abcde$ \ No newline at end of file diff --git a/tests/detect_required/in3.re b/tests/detect_required/in3.re new file mode 100644 index 000000000..bed2c040e --- /dev/null +++ b/tests/detect_required/in3.re @@ -0,0 +1 @@ +^(ab|cd)$ \ No newline at end of file diff --git a/tests/detect_required/in4.re b/tests/detect_required/in4.re new file mode 100644 index 000000000..72c6c30d0 --- /dev/null +++ b/tests/detect_required/in4.re @@ -0,0 +1 @@ +^(ab|cd|ef)$ \ No newline at end of file diff --git a/tests/detect_required/in5.re b/tests/detect_required/in5.re new file mode 100644 index 000000000..454fb8261 --- /dev/null +++ b/tests/detect_required/in5.re @@ -0,0 +1 @@ +^(abc|def)$ \ No newline at end of file diff --git a/tests/detect_required/in6.re b/tests/detect_required/in6.re new file mode 100644 index 000000000..bdbaa3c85 --- /dev/null +++ b/tests/detect_required/in6.re @@ -0,0 +1 @@ +^(abc|dbf)$ \ No newline at end of file diff --git a/tests/detect_required/in7.re b/tests/detect_required/in7.re new file mode 100644 index 000000000..1957f292a --- /dev/null +++ b/tests/detect_required/in7.re @@ -0,0 +1 @@ +^abc(def)*ghi$ \ No newline at end of file diff --git a/tests/detect_required/in8.re b/tests/detect_required/in8.re new file mode 100644 index 000000000..9ec28ce00 --- /dev/null +++ b/tests/detect_required/in8.re @@ -0,0 +1 @@ +^abc(def)+ghi$ \ No newline at end of file diff --git a/tests/detect_required/in9.re b/tests/detect_required/in9.re new file mode 100644 index 000000000..e3f42fa55 --- /dev/null +++ b/tests/detect_required/in9.re @@ -0,0 +1 @@ +^ghi(def)abc$ \ No newline at end of file diff --git a/tests/detect_required/out0.txt b/tests/detect_required/out0.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out0.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out1.txt b/tests/detect_required/out1.txt new file mode 100644 index 000000000..8ba4a7a8d --- /dev/null +++ b/tests/detect_required/out1.txt @@ -0,0 +1 @@ +1 a diff --git a/tests/detect_required/out2.txt b/tests/detect_required/out2.txt new file mode 100644 index 000000000..7a88aba43 --- /dev/null +++ b/tests/detect_required/out2.txt @@ -0,0 +1 @@ +5 abcde diff --git a/tests/detect_required/out3.txt b/tests/detect_required/out3.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out3.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out4.txt b/tests/detect_required/out4.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out4.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out5.txt b/tests/detect_required/out5.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out5.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out6.txt b/tests/detect_required/out6.txt new file mode 100644 index 000000000..bacfc4097 --- /dev/null +++ b/tests/detect_required/out6.txt @@ -0,0 +1 @@ +1 b diff --git a/tests/detect_required/out7.txt b/tests/detect_required/out7.txt new file mode 100644 index 000000000..099f9b056 --- /dev/null +++ b/tests/detect_required/out7.txt @@ -0,0 +1 @@ +6 abcghi diff --git a/tests/detect_required/out8.txt b/tests/detect_required/out8.txt new file mode 100644 index 000000000..19fa3c37b --- /dev/null +++ b/tests/detect_required/out8.txt @@ -0,0 +1 @@ +9 abcdefghi diff --git a/tests/detect_required/out9.txt b/tests/detect_required/out9.txt new file mode 100644 index 000000000..19fa3c37b --- /dev/null +++ b/tests/detect_required/out9.txt @@ -0,0 +1 @@ +9 abcdefghi From aab442218e1d198c0969350504837da6e9cd9e1b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 19 Sep 2024 11:40:49 -0400 Subject: [PATCH 269/373] fsm.1.xml: Add basic info about -S and -q requiredchars. --- man/fsm.1/fsm.1.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/man/fsm.1/fsm.1.xml b/man/fsm.1/fsm.1.xml index 6df32e186..776aaaab9 100644 --- a/man/fsm.1/fsm.1.xml +++ b/man/fsm.1/fsm.1.xml @@ -33,6 +33,7 @@ -G &length.arg;"> -k &io.arg;"> -i &iterations.arg;"> + -S &limit.arg;"> -U &charset.arg;"> -X"> @@ -325,6 +326,14 @@ + + &S.opt; + + + Set a step limit for long-running operations. + + + &t.opt; @@ -487,6 +496,13 @@ of each state in the &fsm;. Printed to &stdout.lit;; exit status is always true.
requiredchars–Determine characters that must appear in any + inputs that could match the &fsm;. Exit status is true + unless the step limit was reached.
From 4758734065d334e2a863750c935850f9a6b84432 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 23 Sep 2024 16:22:40 -0400 Subject: [PATCH 270/373] print API: Box end_ids and end_id_count in a struct for callbacks. Move the end_id array and its count into a struct for state metadata, and rename access throughout to end_ids and end_id_count. Upcoming changes for eager output IDs will soon be passing more info to all of these callbacks, but only callers making use of those fields need to care. Instead of making callers add more `(void) param;` declarations all over the place to avoid warnings, just pass in a metadata struct pointer. Also, "count" is a pretty generic name and what it refers to will soon be ambiguous. This should not be a functional change on its own. --- include/fsm/print.h | 11 ++++++--- src/libfsm/print.c | 26 +++++++++++--------- src/libfsm/print.h | 10 +++++--- src/libfsm/print/awk.c | 32 +++++++++++++----------- src/libfsm/print/c.c | 29 +++++++++++++--------- src/libfsm/print/dot.c | 21 ++++++++++------ src/libfsm/print/fsm.c | 19 ++++++++------ src/libfsm/print/go.c | 25 +++++++++++-------- src/libfsm/print/irdot.c | 7 +++++- src/libfsm/print/irjson.c | 7 +++++- src/libfsm/print/json.c | 7 +++++- src/libfsm/print/llvm.c | 32 +++++++++++++++--------- src/libfsm/print/rust.c | 18 ++++++++++---- src/libfsm/print/sh.c | 14 +++++++---- src/libfsm/print/vmasm.c | 10 +++++--- src/libfsm/print/vmc.c | 47 ++++++++++++++++++----------------- src/libfsm/print/vmdot.c | 18 ++++++++------ src/libfsm/print/vmops.c | 19 +++++++++----- src/lx/main.c | 10 ++++---- src/lx/print/c.c | 8 +++--- src/lx/print/dot.c | 16 ++++++------ src/re/main.c | 52 +++++++++++++++++++-------------------- src/rx/main.c | 16 ++++++------ 23 files changed, 271 insertions(+), 183 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index ed9699e07..9f7264e81 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -42,6 +42,11 @@ enum fsm_print_lang { FSM_PRINT_VMOPS_MAIN }; +struct fsm_state_metadata { + const fsm_end_id_t *end_ids; + size_t end_id_count; +}; + /* * Hooks to override generated code. These give an oportunity to * emit application-specific constructs, especially based on ids @@ -66,19 +71,19 @@ struct fsm_hooks { void *lang_opaque, void *hook_opaque); int (*accept)(FILE *, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque); int (*reject)(FILE *, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque); int (*comment)(FILE *, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *hook_opaque); /* only called for AMBIG_ERROR; see opt.ambig */ int (*conflict)(FILE *, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, const char *example, void *hook_opaque); diff --git a/src/libfsm/print.c b/src/libfsm/print.c index 501b12b96..c555c5e48 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -53,9 +53,9 @@ int print_hook_accept(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, int (*default_accept)(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque), void *lang_opaque) { @@ -64,7 +64,7 @@ print_hook_accept(FILE *f, assert(hooks != NULL); if (opt->ambig == AMBIG_ERROR) { - assert(count <= 1); + assert(state_metadata->end_id_count <= 1); } if (default_accept == NULL) { @@ -72,10 +72,10 @@ print_hook_accept(FILE *f, } if (hooks->accept != NULL) { - return hooks->accept(f, opt, ids, count, + return hooks->accept(f, opt, state_metadata, lang_opaque, hooks->hook_opaque); } else if (default_accept != NULL) { - return default_accept(f, opt, ids, count, + return default_accept(f, opt, state_metadata, lang_opaque, hooks->hook_opaque); } @@ -86,21 +86,21 @@ int print_hook_comment(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const fsm_end_id_t *ids, size_t count) + const struct fsm_state_metadata *state_metadata) { assert(f != NULL); assert(opt != NULL); assert(hooks != NULL); if (opt->ambig == AMBIG_ERROR) { - assert(count <= 1); + assert(state_metadata->end_id_count <= 1); } if (opt->comments && hooks->comment != NULL) { /* this space is a polyglot */ fprintf(f, " "); - return hooks->comment(f, opt, ids, count, + return hooks->comment(f, opt, state_metadata, hooks->hook_opaque); } @@ -138,7 +138,7 @@ int print_hook_conflict(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, const char *example) { assert(opt != NULL); @@ -147,7 +147,7 @@ print_hook_conflict(FILE *f, /* f may be NULL for FSM_PRINT_NONE */ if (hooks->conflict != NULL) { - return hooks->conflict(f, opt, ids, count, + return hooks->conflict(f, opt, state_metadata, example, hooks->hook_opaque); } @@ -211,8 +211,12 @@ print_conflicts(FILE *f, const struct fsm *fsm, * The conflict hook is called here (rather in the caller), * because it can be called multiple times, for different states. */ + struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; if (-1 == print_hook_conflict(f, opt, hooks, - ids, count, + &state_metadata, example)) { goto error; diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 6fea83274..0a8e2e0c1 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -14,6 +14,8 @@ struct ir; struct dfavm_op_ir; struct ret_list; +#include + int print_hook_args(FILE *f, const struct fsm_options *opt, @@ -26,9 +28,9 @@ int print_hook_accept(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, int (*default_accept)(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque), void *lang_opaque); @@ -36,7 +38,7 @@ int print_hook_comment(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const fsm_end_id_t *ids, size_t count); + const struct fsm_state_metadata *state_metadata); int print_hook_reject(FILE *f, @@ -50,7 +52,7 @@ int print_hook_conflict(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, const char *example); typedef int fsm_print_f(FILE *f, diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index 7bff10eb6..bd78e8ef7 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -51,7 +51,7 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -69,37 +69,37 @@ default_accept(FILE *f, const struct fsm_options *opt, case AMBIG_ERROR: // TODO: decide if we deal with this ahead of the call to print or not - if (count > 1) { + if (state_metadata->end_id_count > 1) { errno = EINVAL; return -1; } - - fprintf(f, "return %u;", ids[0]); + + fprintf(f, "return %u;", state_metadata->end_ids[0]); break; - + case AMBIG_EARLIEST: /* * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, "return %u;", ids[0]); + fprintf(f, "return %u;", state_metadata->end_ids[0]); break; - + case AMBIG_MULTIPLE: /* awk can't return an array */ fprintf(f, "return \""); - for (size_t i = 0; i < count; i++) { - fprintf(f, "%u", ids[i]); + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); - if (i < count - 1) { + if (i < state_metadata->end_id_count - 1) { fprintf(f, ","); } } fprintf(f, "\""); break; - + default: assert(!"unreached"); abort(); @@ -156,9 +156,13 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_FAIL: return print_hook_reject(f, opt, hooks, default_reject, NULL); - case VM_END_SUCC: + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, default_accept, NULL)) { @@ -166,7 +170,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, } if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 2199b66c3..1eefaf231 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -56,17 +56,17 @@ ir_hasend(const struct ir *ir) // TODO: centralise vmc/c static int print_ids(FILE *f, - enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) + enum fsm_ambig ambig, const struct fsm_state_metadata *state_metadata) { switch (ambig) { case AMBIG_NONE: // TODO: for C99 we'd return bool - fprintf(f, "return 1;"); + fprintf(f, "return 1;"); break; case AMBIG_ERROR: // TODO: decide if we deal with this ahead of the call to print or not - if (count > 1) { + if (state_metadata->end_id_count > 1) { errno = EINVAL; return -1; } @@ -79,7 +79,7 @@ print_ids(FILE *f, * and only appear once each, and are sorted. */ fprintf(f, "{\n"); - fprintf(f, "\t\t*id = %u;\n", ids[0]); + fprintf(f, "\t\t*id = %u;\n", state_metadata->end_ids[0]); fprintf(f, "\t\treturn 1;\n"); fprintf(f, "\t}"); break; @@ -94,15 +94,15 @@ print_ids(FILE *f, fprintf(f, "{\n"); fprintf(f, "\t\tstatic const unsigned a[] = { "); - for (fsm_end_id_t i = 0; i < count; i++) { - fprintf(f, "%u", ids[i]); - if (i + 1 < count) { + for (fsm_end_id_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } fprintf(f, " };\n"); fprintf(f, "\t\t*ids = a;\n"); - fprintf(f, "\t\t*count = %zu;\n", count); + fprintf(f, "\t\t*count = %zu;\n", state_metadata->end_id_count); fprintf(f, "\t\treturn 1;\n"); fprintf(f, "\t}"); break; @@ -117,7 +117,7 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -127,7 +127,7 @@ default_accept(FILE *f, const struct fsm_options *opt, (void) lang_opaque; (void) hook_opaque; - if (-1 == print_ids(f, opt->ambig, ids, count)) { + if (-1 == print_ids(f, opt->ambig, state_metadata)) { return -1; } @@ -374,8 +374,13 @@ print_endstates(FILE *f, fprintf(f, "\tcase S%u: ", i); + const struct fsm_state_metadata state_metadata = { + .end_ids = ir->states[i].ids, + .end_id_count = ir->states[i].count, + }; + if (-1 == print_hook_accept(f, opt, hooks, - ir->states[i].ids, ir->states[i].count, + &state_metadata, default_accept, NULL)) { @@ -383,7 +388,7 @@ print_endstates(FILE *f, } if (-1 == print_hook_comment(f, opt, hooks, - ir->states[i].ids, ir->states[i].count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index f4168e1d3..c1ac9b875 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -27,7 +27,7 @@ static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { fsm_state_t s; @@ -45,15 +45,15 @@ default_accept(FILE *f, const struct fsm_options *opt, if (!opt->anonymous_states) { fprintf(f, "%u", s); - if (count > 0) { + if (state_metadata->end_id_count > 0) { fprintf(f, "
"); } } - for (size_t i = 0; i < count; i++) { - fprintf(f, "#%u", ids[i]); + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "#%u", state_metadata->end_ids[i]); - if (i < count - 1) { + if (i < state_metadata->end_id_count - 1) { fprintf(f, ","); } } @@ -66,7 +66,7 @@ default_accept(FILE *f, const struct fsm_options *opt, static int default_reject(FILE *f, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque) -{ +{ fsm_state_t s; assert(f != NULL); @@ -258,8 +258,13 @@ print_dotfrag(FILE *f, fprintf(f, ", "); + const struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; + if (-1 == print_hook_accept(f, opt, hooks, - ids, count, + &state_metadata, default_accept, &s)) { return -1; @@ -269,7 +274,7 @@ print_dotfrag(FILE *f, fprintf(f, ","); if (-1 == print_hook_comment(f, opt, hooks, - ids, count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 0c93e4338..06c7e3403 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -27,7 +27,7 @@ static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -36,16 +36,16 @@ default_accept(FILE *f, const struct fsm_options *opt, (void) hook_opaque; - if (count == 0) { + if (state_metadata->end_id_count == 0) { return 0; } fprintf(f, " = ["); - for (size_t i = 0; i < count; i++) { - fprintf(f, "%u", ids[i]); + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } @@ -361,8 +361,13 @@ fsm_print_fsm(FILE *f, fprintf(f, "%u", s); + const struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; + if (-1 == print_hook_accept(f, opt, hooks, - ids, count, + &state_metadata, default_accept, NULL)) { @@ -370,7 +375,7 @@ fsm_print_fsm(FILE *f, } if (-1 == print_hook_comment(f, opt, hooks, - ids, count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 3222746da..3575c5b75 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -48,7 +48,7 @@ cmp_operator(int cmp) static int print_ids(FILE *f, - enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count, + enum fsm_ambig ambig, const struct fsm_state_metadata *state_metadata, size_t i) { switch (ambig) { @@ -57,12 +57,12 @@ print_ids(FILE *f, case AMBIG_ERROR: // TODO: decide if we deal with this ahead of the call to print or not - if (count > 1) { + if (state_metadata->end_id_count > 1) { errno = EINVAL; return -1; } - fprintf(f, ", %u", ids[0]); + fprintf(f, ", %u", state_metadata->end_ids[0]); break; case AMBIG_EARLIEST: @@ -70,7 +70,7 @@ print_ids(FILE *f, * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, ", %u", ids[0]); + fprintf(f, ", %u", state_metadata->end_ids[0]); break; case AMBIG_MULTIPLE: @@ -87,7 +87,7 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { size_t i; @@ -102,7 +102,7 @@ default_accept(FILE *f, const struct fsm_options *opt, fprintf(f, "return true"); - if (-1 == print_ids(f, opt->ambig, ids, count, i)) { + if (-1 == print_ids(f, opt->ambig, state_metadata, i)) { return -1; } @@ -195,15 +195,20 @@ print_end(FILE *f, const struct dfavm_op_ir *op, fprintf(f, "{\n"); fprintf(f, "\t\t"); + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, default_accept, &i)) { return -1; } if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } @@ -420,7 +425,7 @@ fsm_print_go(FILE *f, if (hooks->args != NULL) { fprintf(f, ", "); - + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { return -1; } @@ -445,7 +450,7 @@ fsm_print_go(FILE *f, default: assert(!"unreached"); abort(); - } + } fprintf(f, " {\n"); diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 4541eabc9..f8e62bc3b 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -219,8 +219,13 @@ print_state(FILE *f, if (cs->isend && hooks->accept != NULL) { fprintf(f, "\t\t "); + const struct fsm_state_metadata state_metadata = { + .end_ids = cs->ids, + .end_id_count = cs->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, - cs->ids, cs->count, + &state_metadata, NULL, NULL)) { diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 147ba263f..89c4e52b3 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -139,12 +139,17 @@ print_state(FILE *f, fprintf(f, "],\n"); } + const struct fsm_state_metadata state_metadata = { + .end_ids = cs->ids, + .end_id_count = cs->count, + }; + /* showing hook in addition to existing content */ if (cs->isend && hooks->accept != NULL) { fprintf(f, "\t\t\t\"endleaf\": "); if (-1 == print_hook_accept(f, opt, hooks, - cs->ids, cs->count, + &state_metadata, NULL, NULL)) { return -1; diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 025d1fe6f..b1e9f0ca2 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -280,8 +280,13 @@ fsm_print_json(FILE *f, fprintf(f, "{ %u, ", i); + const struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; + if (-1 == print_hook_accept(f, opt, hooks, - ids, count, + &state_metadata, NULL, NULL)) { return -1; diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index ec931769c..14b116555 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -90,7 +90,7 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -107,12 +107,12 @@ default_accept(FILE *f, const struct fsm_options *opt, case AMBIG_ERROR: // TODO: decide if we deal with this ahead of the call to print or not - if (count > 1) { + if (state_metadata->end_id_count > 1) { errno = EINVAL; return -1; } - fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); + fprintf(f, "%%rt { i1 true, i32 %u }", state_metadata->end_ids[0]); break; case AMBIG_EARLIEST: @@ -120,14 +120,14 @@ default_accept(FILE *f, const struct fsm_options *opt, * The libfsm api guarentees these ids are unique, * and only appear once each, and are sorted. */ - fprintf(f, "%%rt { i1 true, i32 %u }", ids[0]); + fprintf(f, "%%rt { i1 true, i32 %u }", state_metadata->end_ids[0]); break; case AMBIG_MULTIPLE: - fprintf(f, "internal unnamed_addr constant [%zu x i32] [", count); - for (size_t j = 0; j < count; j++) { - fprintf(f, "i32 %u", ids[j]); - if (j + 1 < count) { + fprintf(f, "internal unnamed_addr constant [%zu x i32] [", state_metadata->end_id_count); + for (size_t j = 0; j < state_metadata->end_id_count; j++) { + fprintf(f, "i32 %u", state_metadata->end_ids[j]); + if (j + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } @@ -662,15 +662,19 @@ fsm_print_llvm(FILE *f, if (opt->ambig == AMBIG_MULTIPLE) { for (size_t i = 0; i < retlist->count; i++) { fprintf(f, "@%sr%zu = ", prefix, i); + const struct fsm_state_metadata state_metadata = { + .end_ids = retlist->a[i].ids, + .end_id_count = retlist->a[i].count, + }; if (-1 == print_hook_accept(f, opt, hooks, - retlist->a[i].ids, retlist->a[i].count, + &state_metadata, default_accept, NULL)) { return -1; } if (-1 == print_hook_comment(f, opt, hooks, - retlist->a[i].ids, retlist->a[i].count)) + &state_metadata)) { return -1; } @@ -687,8 +691,12 @@ fsm_print_llvm(FILE *f, ptr_i32, retlist->a[i].count, prefix, i, ptr_i32, retlist->a[i].count); fprintf(f, ","); } else { + const struct fsm_state_metadata state_metadata = { + .end_ids = retlist->a[i].ids, + .end_id_count = retlist->a[i].count, + }; if (-1 == print_hook_accept(f, opt, hooks, - retlist->a[i].ids, retlist->a[i].count, + &state_metadata, default_accept, NULL)) { return -1; @@ -697,7 +705,7 @@ fsm_print_llvm(FILE *f, fprintf(f, ","); if (-1 == print_hook_comment(f, opt, hooks, - retlist->a[i].ids, retlist->a[i].count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index e6c559287..682bc051f 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -91,7 +91,7 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { size_t i; @@ -104,7 +104,7 @@ default_accept(FILE *f, const struct fsm_options *opt, i = * (const size_t *) lang_opaque; - if (-1 == print_ids(f, opt->ambig, ids, count, i)) { + if (-1 == print_ids(f, opt->ambig, state_metadata->end_ids, state_metadata->end_id_count, i)) { return -1; } @@ -181,9 +181,13 @@ print_end(FILE *f, const struct dfavm_op_ir *op, i = op->ret - retlist->a; + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + return print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, - default_accept, + &state_metadata, default_accept, &i); default: @@ -365,8 +369,12 @@ fsm_print_rustfrag(FILE *f, } if (op->u.stop.end_bits == VM_END_SUCC) { + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 7a870a991..3c11c2f1f 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -107,7 +107,7 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -119,7 +119,7 @@ default_accept(FILE *f, const struct fsm_options *opt, fprintf(f, "matched"); - if (-1 == print_ids(f, opt->ambig, ids, count)) { + if (-1 == print_ids(f, opt->ambig, state_metadata->end_ids, state_metadata->end_id_count)) { return -1; } @@ -204,9 +204,13 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_FAIL: return print_hook_reject(f, opt, hooks, default_reject, NULL); - case VM_END_SUCC: + case VM_END_SUCC:; + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, default_accept, NULL)) { @@ -214,7 +218,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, } if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index a861fc17e..e29eda24d 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -49,15 +49,19 @@ print_end(FILE *f, const struct dfavm_op_ir *op, } break; - case VM_END_SUCC: + case VM_END_SUCC:; + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, NULL, NULL)) { return -1; } if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index c43073dca..2fef2da65 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -49,7 +49,7 @@ cmp_operator(int cmp) // TODO: centralise vmc/c static int print_ids(FILE *f, - enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) + enum fsm_ambig ambig, const struct fsm_state_metadata *state_metadata) { switch (ambig) { case AMBIG_NONE: @@ -59,7 +59,7 @@ print_ids(FILE *f, case AMBIG_ERROR: // TODO: decide if we deal with this ahead of the call to print or not - if (count > 1) { + if (state_metadata->end_id_count > 1) { errno = EINVAL; return -1; } @@ -72,11 +72,11 @@ print_ids(FILE *f, * and only appear once each, and are sorted. */ fprintf(f, "{\n"); - fprintf(f, "\t\t*id = %u;\n", ids[0]); + fprintf(f, "\t\t*id = %u;\n", state_metadata->end_ids[0]); fprintf(f, "\t\treturn 1;\n"); fprintf(f, "\t}"); break; - + case AMBIG_MULTIPLE: /* * Here I would like to emit (static unsigned []) { 1, 2, 3 } @@ -84,22 +84,22 @@ print_ids(FILE *f, * is a compiler extension. * So I'm emitting a static const variable declaration instead. */ - + fprintf(f, "{\n"); fprintf(f, "\t\tstatic const unsigned a[] = { "); - for (fsm_end_id_t i = 0; i < count; i++) { - fprintf(f, "%u", ids[i]); - if (i + 1 < count) { + for (fsm_end_id_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } fprintf(f, " };\n"); fprintf(f, "\t\t*ids = a;\n"); - fprintf(f, "\t\t*count = %zu;\n", count); + fprintf(f, "\t\t*count = %zu;\n", state_metadata->end_id_count); fprintf(f, "\t\treturn 1;\n"); fprintf(f, "\t}"); break; - + default: assert(!"unreached"); abort(); @@ -110,7 +110,7 @@ print_ids(FILE *f, static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -119,18 +119,18 @@ default_accept(FILE *f, const struct fsm_options *opt, (void) lang_opaque; (void) hook_opaque; - - if (-1 == print_ids(f, opt->ambig, ids, count)) { + + if (-1 == print_ids(f, opt->ambig, state_metadata)) { return -1; } return 0; -} - +} + static int default_reject(FILE *f, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque) -{ +{ assert(f != NULL); assert(opt != NULL); assert(lang_opaque == NULL); @@ -177,9 +177,13 @@ print_end(FILE *f, const struct dfavm_op_ir *op, case VM_END_FAIL: return print_hook_reject(f, opt, hooks, default_reject, NULL); - case VM_END_SUCC: + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, default_accept, NULL)) { @@ -187,7 +191,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, } if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } @@ -246,10 +250,10 @@ walk_sequence(struct dfavm_op_ir *op, /* * Here we're looking for a sequence of: - * + * * FETCH: (or fail) * STOP: c != 'x' (or fail) - * + * * This catches situations like the "abc" and "xyz" in /^abc[01]xyz/, * but not for runs in unanchored regexes. For those, we'd be better * off adding VM instructions for sets of strings, and producing @@ -619,4 +623,3 @@ fsm_print_vmc(FILE *f, return 0; } - diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 85fbf63a1..512cc893b 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -48,7 +48,7 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -60,13 +60,13 @@ default_accept(FILE *f, const struct fsm_options *opt, fprintf(f, "match"); - if (count > 0) { + if (state_metadata->end_id_count > 0) { fprintf(f, " / "); - for (size_t i = 0; i < count; i++) { - fprintf(f, "#%u", ids[i]); + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "#%u", state_metadata->end_ids[i]); - if (i < count - 1) { + if (i < state_metadata->end_id_count - 1) { fprintf(f, " "); } } @@ -128,9 +128,13 @@ print_end(FILE *f, case VM_END_FAIL: return print_hook_reject(f, opt, hooks, default_reject, NULL); - case VM_END_SUCC: + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, default_accept, NULL)) { diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index d95263062..59de464f1 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -47,7 +47,7 @@ cmp_operator(int cmp) static int default_accept(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { const char *prefix; @@ -65,8 +65,7 @@ default_accept(FILE *f, const struct fsm_options *opt, prefix = "fsm_"; } - (void) ids; - (void) count; + (void) state_metadata; fprintf(f, "%sactionRET, 1", prefix); if (opt->ambig != AMBIG_NONE) { @@ -152,13 +151,17 @@ print_end(FILE *f, const struct dfavm_op_ir *op, } break; - case VM_END_SUCC: + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; assert(op->ret >= retlist->a); i = op->ret - retlist->a; if (-1 == print_hook_accept(f, opt, hooks, - op->ret->ids, op->ret->count, + &state_metadata, default_accept, &i)) { return -1; @@ -306,8 +309,12 @@ fsm_print_vmops_c(FILE *f, fprintf(f, "},"); if (op->instr == VM_OP_STOP && op->u.stop.end_bits == VM_END_SUCC) { + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; if (-1 == print_hook_comment(f, opt, hooks, - op->ret->ids, op->ret->count)) + &state_metadata)) { return -1; } diff --git a/src/lx/main.c b/src/lx/main.c index f7a3fb5a5..ea16ee70d 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -523,7 +523,7 @@ run_threads(int concurrency, void *(*fn)(void *)) static int conflict(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, const char *example, + const struct fsm_state_metadata *state_metadata, const char *example, void *hook_opaque) { const struct ast *ast; @@ -531,7 +531,7 @@ conflict(FILE *f, const struct fsm_options *opt, assert(opt != NULL); assert(opt->ambig == AMBIG_ERROR); - assert(count > 1); + assert(state_metadata->end_id_count > 1); (void) f; (void) opt; @@ -544,7 +544,7 @@ conflict(FILE *f, const struct fsm_options *opt, fprintf(stderr, "ambiguous mappings to "); - for (i = 0; i < count; i++) { + for (i = 0; i < state_metadata->end_id_count; i++) { const struct ast_mapping *m; /* @@ -557,7 +557,7 @@ conflict(FILE *f, const struct fsm_options *opt, continue; } - m = ast_getendmappingbyendid(ids[i]); + m = ast_getendmappingbyendid(state_metadata->end_ids[i]); if (m->token != NULL) { fprintf(stderr, "$%s", m->token->s); @@ -573,7 +573,7 @@ conflict(FILE *f, const struct fsm_options *opt, fprintf(stderr, "z%p", (void *) m->to); /* TODO: zindexof(n->to) */ } - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(stderr, ", "); } } diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 58f3279ce..dae4bd937 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -139,7 +139,7 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, static int accept_c(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { const struct ast *ast; @@ -147,13 +147,13 @@ accept_c(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ids != NULL); - assert(count > 0); + assert(state_metadata->end_ids != NULL); + assert(state_metadata->end_id_count > 0); assert(lang_opaque == NULL); assert(hook_opaque != NULL); ast = hook_opaque; - m = ast_getendmappingbyendid(ids[0]); + m = ast_getendmappingbyendid(state_metadata->end_ids[0]); /* XXX: don't need this if complete */ fprintf(f, "%sungetc(lx, c); ", prefix.api); diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index 0369cfd33..09c14ce5b 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -66,7 +66,7 @@ mapping(FILE *f, const struct ast_mapping *m, const struct ast *ast) static int accept_dot(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { const struct ast_mapping *m; @@ -75,8 +75,8 @@ accept_dot(FILE *f, const struct fsm_options *opt, assert(f != NULL); assert(opt != NULL); - assert(ids != NULL); - assert(count > 0); + assert(state_metadata->end_ids != NULL); + assert(state_metadata->end_id_count > 0); assert(lang_opaque != NULL); assert(hook_opaque != NULL); @@ -89,19 +89,19 @@ accept_dot(FILE *f, const struct fsm_options *opt, fprintf(f, "%u
", s); } - if (count == 1) { - m = ast_getendmappingbyendid(ids[0]); + if (state_metadata->end_id_count == 1) { + m = ast_getendmappingbyendid(state_metadata->end_ids[0]); mapping(f, m, ast); } else { size_t i; fprintf(f, ""); - for (i = 0; i < count; i++) { - m = ast_getendmappingbyendid(ids[i]); + for (i = 0; i < state_metadata->end_id_count; i++) { + m = ast_getendmappingbyendid(state_metadata->end_ids[i]); mapping(f, m, ast); - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, "
"); } } diff --git a/src/re/main.c b/src/re/main.c index ff575ca03..62e51f78d 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -294,14 +294,14 @@ printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) static int conflict(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, const char *example, + const struct fsm_state_metadata *state_metadata, const char *example, void *hook_opaque) { size_t i; assert(opt != NULL); assert(opt->ambig == AMBIG_ERROR); - assert(count > 1); + assert(state_metadata->end_id_count > 1); (void) f; (void) opt; @@ -312,13 +312,13 @@ conflict(FILE *f, const struct fsm_options *opt, fprintf(stderr, "ambiguous matches for "); - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); /* TODO: print nicely */ - fprintf(stderr, "/%s/", matchv[ids[i]]); + fprintf(stderr, "/%s/", matchv[state_metadata->end_ids[i]]); - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(stderr, ", "); } } @@ -335,7 +335,7 @@ conflict(FILE *f, const struct fsm_options *opt, static int comment_c(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *hook_opaque) { size_t i; @@ -348,12 +348,12 @@ comment_c(FILE *f, const struct fsm_options *opt, fprintf(f, "/* "); - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } @@ -365,7 +365,7 @@ comment_c(FILE *f, const struct fsm_options *opt, static int comment_rust(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *hook_opaque) { size_t i; @@ -378,12 +378,12 @@ comment_rust(FILE *f, const struct fsm_options *opt, fprintf(f, "// "); - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } @@ -393,7 +393,7 @@ comment_rust(FILE *f, const struct fsm_options *opt, static int comment_llvm(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *hook_opaque) { size_t i; @@ -406,12 +406,12 @@ comment_llvm(FILE *f, const struct fsm_options *opt, fprintf(f, "; "); - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } @@ -421,7 +421,7 @@ comment_llvm(FILE *f, const struct fsm_options *opt, static int comment_dot(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, void *hook_opaque) { size_t i; @@ -434,12 +434,12 @@ comment_dot(FILE *f, const struct fsm_options *opt, fprintf(f, "/* "); - for (i = 0; i < count; i++) { - assert(ids[i] < matchc); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - fprintf(f, "\"%s\"", matchv[ids[i]]); /* XXX: escape string (and comment) */ + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (i + 1 < count) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, ", "); } } diff --git a/src/rx/main.c b/src/rx/main.c index 0328f900a..8fe40eb58 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -567,7 +567,7 @@ build_pattern_fsm(bool show_stats, static int conflict(FILE *f, const struct fsm_options *opt, - const fsm_end_id_t *ids, size_t count, + const struct fsm_state_metadata *state_metadata, const char *example, void *hook_opaque) { @@ -578,10 +578,10 @@ conflict(FILE *f, const struct fsm_options *opt, assert(env->general != NULL); /* in rx all end states have an end id */ - assert(count > 0); + assert(state_metadata->end_id_count > 0); /* ...and we only conflict when there's more than one */ - assert(count > 1); + assert(state_metadata->end_id_count > 1); (void) f; (void) opt; @@ -592,14 +592,14 @@ conflict(FILE *f, const struct fsm_options *opt, } fprintf(stderr, "\n"); - for (fsm_end_id_t i = 0; i < count; i++) { + for (fsm_end_id_t i = 0; i < state_metadata->end_id_count; i++) { for (size_t k = 0; k < sizeof *env->literals / sizeof **env->literals; k++) { const struct literal_set *literals = &(*env->literals)[k]; for (size_t j = 0; j < literals->count; j++) { - if (literals->a[j].id == ids[i]) { + if (literals->a[j].id == state_metadata->end_ids[i]) { // TODO: escape, centralise with libre's re_perror() fprintf(stderr, "#%u: /%s%.*s%s/\n", - ids[i], + state_metadata->end_ids[i], (k & RE_STRINGS_ANCHOR_LEFT) ? "^" : "", (int) literals->a[j].n, (const char *) literals->a[j].p, (k & RE_STRINGS_ANCHOR_RIGHT) ? "$" : ""); @@ -610,9 +610,9 @@ conflict(FILE *f, const struct fsm_options *opt, for (size_t j = 0; j < env->general->count; j++) { // TODO: delimiters per dialect, centralise with libre's re_perror() - if (env->general->a[i].id == ids[i]) { + if (env->general->a[i].id == state_metadata->end_ids[i]) { fprintf(stderr, "#%u: /%s/\n", - ids[i], + state_metadata->end_ids[i], env->general->a[i].s); goto next; } From 8487fcb1bdd339d5dd3c8b321f8210891635b93d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 23 Sep 2024 17:02:32 -0400 Subject: [PATCH 271/373] Move call to fsm_countstates inside #ifndef NDEBUG. It's only used for the assertion. --- src/libfsm/endids.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index d0dfc9e38..6d8e885bd 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -880,7 +880,9 @@ fsm_endid_iter_bulk(const struct fsm *fsm, return 1; } +#ifndef NDEBUG const fsm_state_t state_count = fsm_countstates(fsm); +#endif bucket_count = ei->bucket_count; From 2d61cf94d85977cdecbcdd49d24d8b848b1196ba Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 23 Sep 2024 17:03:40 -0400 Subject: [PATCH 272/373] libfsm.syms: Expose fsm_new_statealloc. It's already in fsm/fsm.h. --- src/libfsm/libfsm.syms | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 56734e7f2..98f79fffb 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -62,6 +62,7 @@ fsm_print_vmops_main fsm_clone fsm_free fsm_new +fsm_new_statealloc fsm_move fsm_merge fsm_addstate From 1295005ed710f5a631fc40c4c214bc7417f7effe Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 24 Sep 2024 10:49:10 -0400 Subject: [PATCH 273/373] ir: Wrap ids and count in a struct, to namespace them as endids. The IR struct is about to get another id & count pair. This loses storing the count as a 31-bit bitfield, but if the goal for that is saving memory then the ids array allocation could be replaced with a struct that contains the count, and then each IR without endids will save more space than the current approach. --- src/libfsm/print.c | 2 +- src/libfsm/print/c.c | 4 ++-- src/libfsm/print/ir.c | 10 +++++----- src/libfsm/print/ir.h | 6 ++++-- src/libfsm/print/irdot.c | 12 ++++++------ src/libfsm/print/irjson.c | 12 ++++++------ src/libfsm/vm/ir.c | 2 +- src/libfsm/vm/retlist.c | 2 +- 8 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/libfsm/print.c b/src/libfsm/print.c index c555c5e48..e1a425cc0 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -358,7 +358,7 @@ fsm_print(FILE *f, const struct fsm *fsm, continue; } - assert(ir->states[i].count <= 1); + assert(ir->states[i].endids.count <= 1); } } diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 1eefaf231..22b03963e 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -375,8 +375,8 @@ print_endstates(FILE *f, fprintf(f, "\tcase S%u: ", i); const struct fsm_state_metadata state_metadata = { - .end_ids = ir->states[i].ids, - .end_id_count = ir->states[i].count, + .end_ids = ir->states[i].endids.ids, + .end_id_count = ir->states[i].endids.count, }; if (-1 == print_hook_accept(f, opt, hooks, diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index b3831b164..457716dcc 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -541,8 +541,8 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) assert(i < ir->n); ir->states[i].isend = fsm_isend(fsm, i); - ir->states[i].ids = NULL; - ir->states[i].count = 0; + ir->states[i].endids.ids = NULL; + ir->states[i].endids.count = 0; if (fsm_isend(fsm, i)) { fsm_end_id_t *ids; @@ -563,8 +563,8 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) assert(res == 1); } - ir->states[i].ids = ids; - ir->states[i].count = count; + ir->states[i].endids.ids = ids; + ir->states[i].endids.count = count; } if (make_state(fsm, i, &ir->states[i]) == -1) { @@ -629,7 +629,7 @@ free_ir(const struct fsm *fsm, struct ir *ir) for (i = 0; i < ir->n; i++) { f_free(fsm->alloc, (void *) ir->states[i].example); - f_free(fsm->alloc, (void *) ir->states[i].ids); + f_free(fsm->alloc, (void *) ir->states[i].endids.ids); switch (ir->states[i].strategy) { case IR_TABLE: diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index bbfd5006e..b375ba850 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -54,8 +54,10 @@ struct ir_error { struct ir_state { const char *example; - fsm_end_id_t *ids; /* NULL -> 0 */ - size_t count:31; // :31 for packing + struct ir_state_endids { + fsm_end_id_t *ids; /* NULL -> 0 */ + size_t count; + } endids; unsigned int isend:1; diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index f8e62bc3b..37dc34296 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -195,13 +195,13 @@ print_state(FILE *f, fprintf(f, "\t\t S%u%s\n", ir_indexof(ir, cs), strategy_name(cs->strategy)); - if (cs->isend && cs->count > 0) { + if (cs->isend && cs->endids.count > 0) { fprintf(f, "\t\t end id"); - for (size_t i = 0; i < cs->count; i++) { - fprintf(f, "#%u", cs->ids[i]); + for (size_t i = 0; i < cs->endids.count; i++) { + fprintf(f, "#%u", cs->endids.ids[i]); - if (i < (size_t) cs->count - 1) { + if (i < (size_t) cs->endids.count - 1) { fprintf(f, " "); } } @@ -220,8 +220,8 @@ print_state(FILE *f, fprintf(f, "\t\t "); const struct fsm_state_metadata state_metadata = { - .end_ids = cs->ids, - .end_id_count = cs->count, + .end_ids = cs->endids.ids, + .end_id_count = cs->endids.count, }; if (-1 == print_hook_accept(f, opt, hooks, diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 89c4e52b3..a96250e0f 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -127,12 +127,12 @@ print_state(FILE *f, fprintf(f, "\t\t{\n"); fprintf(f, "\t\t\t\"end\": %s,\n", cs->isend ? "true" : "false"); - if (cs->isend && cs->count > 0) { + if (cs->isend && cs->endids.count > 0) { fprintf(f, "\t\t\t\"end_id\": ["); - for (size_t i = 0; i < cs->count; i++) { - fprintf(f, "%u", cs->ids[i]); + for (size_t i = 0; i < cs->endids.count; i++) { + fprintf(f, "%u", cs->endids.ids[i]); - if (i < (size_t) cs->count - 1) { + if (i < (size_t) cs->endids.count - 1) { fprintf(f, ", "); } } @@ -140,8 +140,8 @@ print_state(FILE *f, } const struct fsm_state_metadata state_metadata = { - .end_ids = cs->ids, - .end_id_count = cs->count, + .end_ids = cs->endids.ids, + .end_id_count = cs->endids.count, }; /* showing hook in addition to existing content */ diff --git a/src/libfsm/vm/ir.c b/src/libfsm/vm/ir.c index a32bca463..4930f4bc1 100644 --- a/src/libfsm/vm/ir.c +++ b/src/libfsm/vm/ir.c @@ -309,7 +309,7 @@ opasm_new(struct dfavm_assembler_ir *a, const struct ret_list *retlist, if (ir_state != NULL) { op->example = ir_state->example; op->ret = ir_state->isend - ? find_ret(retlist, ir_state->ids, ir_state->count) + ? find_ret(retlist, ir_state->endids.ids, ir_state->endids.count) : NULL; } diff --git a/src/libfsm/vm/retlist.c b/src/libfsm/vm/retlist.c index 031d1d101..97d4eb3f0 100644 --- a/src/libfsm/vm/retlist.c +++ b/src/libfsm/vm/retlist.c @@ -105,7 +105,7 @@ build_retlist(struct ret_list *list, const struct ir *ir) continue; } - if (!append_ret(list, ir->states[i].ids, ir->states[i].count)) { + if (!append_ret(list, ir->states[i].endids.ids, ir->states[i].endids.count)) { return false; } } From e3e868f9002246ac398be38e558f19810193e1dd Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 9 Oct 2024 16:08:15 -0400 Subject: [PATCH 274/373] Temporary interface: Add fsm_determinise_with_config. This works, but we may want to make some naming changes. I'd prefer to use a struct rather than a single parameter here -- as long as other fields have a sensible default when zeroed, the callers will only need to make the interface change once. --- include/fsm/fsm.h | 15 ++++++++++++++ src/libfsm/determinise.c | 43 +++++++++++++++++++++++++++++++++------- src/libfsm/libfsm.syms | 1 + 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 877d5c1bf..1dd710d0e 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -385,6 +385,21 @@ fsm_remove_epsilons(struct fsm *fsm); int fsm_determinise(struct fsm *fsm); +/* Determinise, with a passed in configuration + * and a distinct return value for reaching + * the state limit. */ +struct fsm_determinise_config { + size_t state_limit; /* 0: no limit */ +}; +enum fsm_determinise_with_config_res { + FSM_DETERMINISE_WITH_CONFIG_OK, + FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED, + FSM_DETERMINISE_WITH_CONFIG_ERRNO, +}; +enum fsm_determinise_with_config_res +fsm_determinise_with_config(struct fsm *fsm, + const struct fsm_determinise_config *config); + /* * Make a DFA complete, as per fsm_iscomplete. */ diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 42992b6bc..0190e65ad 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -17,16 +17,20 @@ dump_labels(FILE *f, const uint64_t labels[4]) } } -int -fsm_determinise(struct fsm *nfa) +enum fsm_determinise_with_config_res +fsm_determinise_with_config(struct fsm *nfa, + const struct fsm_determinise_config *config) { - int res = 0; + enum fsm_determinise_with_config_res res = FSM_DETERMINISE_WITH_CONFIG_ERRNO; struct mappingstack *stack = NULL; struct interned_state_set_pool *issp = NULL; struct map map = { NULL, 0, 0, NULL }; struct mapping *curr = NULL; size_t dfacount = 0; + const size_t state_limit = config == NULL + ? 0 + : config->state_limit; struct analyze_closures_env ac_env = { 0 }; @@ -40,7 +44,7 @@ fsm_determinise(struct fsm *nfa) */ if (fsm_has(nfa, fsm_hasepsilons)) { if (!fsm_remove_epsilons(nfa)) { - return 0; + return FSM_DETERMINISE_WITH_CONFIG_ERRNO; } } @@ -52,7 +56,12 @@ fsm_determinise(struct fsm *nfa) issp = interned_state_set_pool_alloc(nfa->alloc); if (issp == NULL) { - return 0; + return FSM_DETERMINISE_WITH_CONFIG_ERRNO; + } + + if (state_limit != 0 && fsm_countstates(nfa) > state_limit) { + res = FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED; + goto cleanup; } { @@ -74,7 +83,7 @@ fsm_determinise(struct fsm *nfa) */ if (!fsm_getstart(nfa, &start)) { - res = 1; + res = FSM_DETERMINISE_WITH_CONFIG_OK; goto cleanup; } @@ -150,6 +159,11 @@ fsm_determinise(struct fsm *nfa) assert(m->dfastate < dfacount); } else { /* not found -- add a new one and push it to the stack for processing */ + + if (state_limit != 0 && dfacount > state_limit) { + res = FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED; + goto cleanup; + } if (!map_add(&map, dfacount, iss, &m)) { goto cleanup; } @@ -260,7 +274,7 @@ fsm_determinise(struct fsm *nfa) assert(fsm_all(nfa, fsm_isdfa)); #endif - res = 1; + res = FSM_DETERMINISE_WITH_CONFIG_OK; cleanup: map_free(&map); @@ -311,6 +325,21 @@ fsm_determinise(struct fsm *nfa) return res; } +int +fsm_determinise(struct fsm *nfa) +{ + enum fsm_determinise_with_config_res res = fsm_determinise_with_config(nfa, NULL); + switch (res) { + case FSM_DETERMINISE_WITH_CONFIG_OK: + return 1; + case FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED: + /* unreachable */ + return 0; + case FSM_DETERMINISE_WITH_CONFIG_ERRNO: + return 0; + } +} + /* Add DFA_state to the list for NFA_state. */ static int add_reverse_mapping(const struct fsm_alloc *alloc, diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 34be09e77..67497d00a 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -101,6 +101,7 @@ fsm_countstates fsm_trim fsm_reverse fsm_determinise +fsm_determinise_with_config fsm_remove_epsilons fsm_complete fsm_minimise From 07e1a0ce6a0c6d4d11b4790435217cac8c371bbf Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 11:44:01 -0400 Subject: [PATCH 275/373] Add 'default:' (should be unreachable) for switch case. This is getting rejected by CI. --- src/libfsm/determinise.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 0190e65ad..559230175 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -336,6 +336,7 @@ fsm_determinise(struct fsm *nfa) /* unreachable */ return 0; case FSM_DETERMINISE_WITH_CONFIG_ERRNO: + default: return 0; } } From a61a9daa0a9614a58def3defcc5035950c8cd581 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 9 Oct 2024 09:48:21 -0400 Subject: [PATCH 276/373] fsm_generate_matches: Add seed argument, random choice if non-zero. There isn't a flag to set the seed in fsm or re yet. Update callers, add a 0 seed argument to default to the existing behavior. Also, no end states means nothing to do, so exit right away. --- include/fsm/walk.h | 6 +++- src/fsm/main.c | 2 +- src/libfsm/gen.c | 69 ++++++++++++++++++++++++++++++++++++++++++---- src/re/main.c | 2 +- tests/gen/gen1.c | 1 + tests/gen/gen2.c | 2 +- tests/gen/gen3.c | 2 +- 7 files changed, 74 insertions(+), 10 deletions(-) diff --git a/include/fsm/walk.h b/include/fsm/walk.h index e1ab5f29e..ea5a223e0 100644 --- a/include/fsm/walk.h +++ b/include/fsm/walk.h @@ -90,6 +90,10 @@ fsm_walk_edges(const struct fsm *fsm, void *opaque, * functionally equivalent cases makes testing dramatically faster, * but exploring every edge could be added later. * + * If seed is zero then it will generate the first label in the label + * set, otherwise a label from the set will be chosen using rand() + * (favoring printable characters). + * * Note: fsm is non-const because it calls fsm_trim on the FSM * internally. This records the shortest distance from each state to an * end state, which is used to prune branches that would not produce @@ -114,7 +118,7 @@ fsm_generate_matches_cb(const struct fsm *fsm, const char *input, size_t input_length, fsm_state_t end_state, void *opaque); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, +fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, fsm_generate_matches_cb *cb, void *opaque); /* Callback provided for the most basic use case for diff --git a/src/fsm/main.c b/src/fsm/main.c index da65791dd..f9d1bb3b0 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -770,7 +770,7 @@ main(int argc, char *argv[]) } if (generate_bounds > 0) { - r = fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt); + r = fsm_generate_matches(fsm, generate_bounds, 0, fsm_generate_cb_printf_escaped, &opt); } fsm_free(fsm); diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 26a77be1c..9f78e67db 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -77,6 +77,7 @@ struct gen_ctx { fsm_generate_matches_cb *cb; bool done; + bool randomized; size_t buf_ceil; size_t buf_used; @@ -139,7 +140,7 @@ static bool grow_stack(struct gen_ctx *ctx); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, +fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, fsm_generate_matches_cb *cb, void *opaque) { if (max_length == 0) { @@ -147,9 +148,13 @@ fsm_generate_matches(struct fsm *fsm, size_t max_length, return 0; } + if (!fsm_has(fsm, fsm_isend)) { + return 1; /* no end state -> nothing to do */ + } + INIT_TIMERS(); TIME(&pre); - int res = gen_init_outer(fsm, max_length, cb, opaque, false, 0); + int res = gen_init_outer(fsm, max_length, cb, opaque, seed != 0, seed); TIME(&post); DIFF_MSEC("fsm_generate_matches", pre, post, NULL); @@ -208,8 +213,9 @@ gen_init_outer(struct fsm *fsm, size_t max_length, assert(fsm_all(fsm, fsm_isdfa)); /* DFA-only */ - assert(!randomized); /* not yet supported */ - (void)seed; + if (randomized) { + srand(seed); + } #if LOG_GEN > 1 fprintf(stderr, "%s: %u states\n", __func__, fsm_countstates(fsm)); @@ -224,6 +230,7 @@ gen_init_outer(struct fsm *fsm, size_t max_length, .max_length = max_length, .cb = cb, .opaque = opaque, + .randomized = randomized, }; if (!gen_init(&ctx, fsm)) { @@ -524,6 +531,55 @@ first_symbol(const uint64_t *symbols) return 0; } +static unsigned char +random_symbol(const uint64_t *symbols) +{ + bool has_zero = false; + unsigned i = 0; + + /* printable and non-printable character choices */ + size_t choice_count = 0; + unsigned char choices[256]; + size_t np_choice_count = 0; + unsigned char np_choices[256]; + + while (i < 256) { + const uint64_t w = symbols[i/64]; + if ((i & 63) == 0 && w == 0) { + i += 64; + continue; + } + if (w & (1ULL << (i & 63))) { + if (i == 0) { + has_zero = true; + } else if (isprint(i)) { + choices[choice_count++] = (unsigned char)i; + } else { + np_choices[np_choice_count++] = (unsigned char)i; + } + } + i++; + } + + if (choice_count > 0) { + const size_t c = rand() % choice_count; + return choices[c]; + } + + if (np_choice_count > 0) { + const size_t c = rand() % np_choice_count; + return np_choices[c]; + } + + /* Prefer anything besides 0x00 if present, since that will truncate the string. */ + if (has_zero) { + return 0; + } + + assert(!"empty set"); + return 0; +} + #if DUMP_EDGES static void dump_edges(fsm_state_t state, struct edge_set *edges) @@ -538,6 +594,7 @@ dump_edges(fsm_state_t state, struct edge_set *edges) size_t i = 0; while (edge_set_group_iter_next(&ei, &eg)) { const unsigned char symbol = first_symbol(eg.symbols); + const unsigned char symbol = random_symbol(eg.symbols); fprintf(stderr, "%s: %d -- %zu/%zu -- 0x%02x (%c) -> %d\n", __func__, state, i, count, symbol, isprint(symbol) ? symbol : '.', eg.to); @@ -585,7 +642,9 @@ sfs_step_edges(struct gen_ctx *ctx, struct gen_stack_frame *sf) struct edge_group_iter_info eg; if (iter_next_transition(ctx, sf, &eg)) { - const unsigned char symbol = first_symbol(eg.symbols); + const unsigned char symbol = ctx->randomized + ? random_symbol(eg.symbols) + : first_symbol(eg.symbols); const fsm_state_t state = eg.to; LOG(2, "sfs_step_edges: got edge 0x%x ('%c')\n", diff --git a/src/re/main.c b/src/re/main.c index 62e51f78d..9059af96e 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -1047,7 +1047,7 @@ main(int argc, char *argv[]) } if (generate_bounds > 0) { - if (!fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt)) { + if (!fsm_generate_matches(fsm, generate_bounds, 0, fsm_generate_cb_printf_escaped, &opt)) { exit(EXIT_FAILURE); } diff --git a/tests/gen/gen1.c b/tests/gen/gen1.c index 25b30b82b..b25ab2bbc 100644 --- a/tests/gen/gen1.c +++ b/tests/gen/gen1.c @@ -34,6 +34,7 @@ int main(void) { assert(fsm != NULL); if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1 /* for \0 */, + 0, gtest_matches_cb, &matches)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); diff --git a/tests/gen/gen2.c b/tests/gen/gen2.c index 02faa5e50..a475e5395 100644 --- a/tests/gen/gen2.c +++ b/tests/gen/gen2.c @@ -28,7 +28,7 @@ int main(void) { struct fsm *fsm = gtest_fsm_of_matches(&matches); assert(fsm != NULL); - if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1, gtest_matches_cb, &matches)) { + if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1, 0, gtest_matches_cb, &matches)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); } diff --git a/tests/gen/gen3.c b/tests/gen/gen3.c index f24217622..7aa0aebce 100644 --- a/tests/gen/gen3.c +++ b/tests/gen/gen3.c @@ -146,7 +146,7 @@ int main(void) { struct fsm *fsm = build(); assert(fsm != NULL); - if (!fsm_generate_matches(fsm, 11, matches_cb, NULL)) { + if (!fsm_generate_matches(fsm, 11, 0, matches_cb, NULL)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); } From ffe6f69fa68b5aaa5f8dd0d1515ece3f4ec341ca Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 11 Sep 2024 10:20:47 -0400 Subject: [PATCH 277/373] common.h: TRACK_TIMES should be 0 when BUILD_FOR_FUZZER is set. It fills up the fuzzer logs with lots of uninteresting output. --- include/adt/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/adt/common.h b/include/adt/common.h index 8604edacd..a83604c97 100644 --- a/include/adt/common.h +++ b/include/adt/common.h @@ -36,7 +36,7 @@ /* If non-zero, expand the timer macros defined below, otherwise * they compile away. */ #ifndef TRACK_TIMES -#define TRACK_TIMES 0 +#define TRACK_TIMES (0 && !BUILD_FOR_FUZZER) #endif #if EXPENSIVE_CHECKS && TRACK_TIMES From b2f9d3d93c858231e8b4de4abf5d49bd01c9226b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 10 Sep 2024 13:51:20 -0400 Subject: [PATCH 278/373] determinise: Remove stale comment. --- src/libfsm/determinise.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 559230175..fc7c68ba4 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -185,8 +185,6 @@ fsm_determinise_with_config(struct fsm *nfa, } ac_env.output_count = 0; - - /* All elements in sclosures[] are interned, so they will be freed later. */ } while ((curr = stack_pop(stack))); { From 35892eb03cf1e866991fd8d7de81d0c26b296bba Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 9 Sep 2024 08:29:49 -0400 Subject: [PATCH 279/373] trim's integrity_check should only run with EXPENSIVE_CHECKS. --- src/libfsm/trim.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libfsm/trim.c b/src/libfsm/trim.c index 4f45607bd..c37965fd8 100644 --- a/src/libfsm/trim.c +++ b/src/libfsm/trim.c @@ -462,6 +462,10 @@ integrity_check(const char *descr, const struct fsm *fsm) return; #endif +#if !EXPENSIVE_CHECKS + return; +#endif + if (LOG_TRIM > 1) { fprintf(stderr, "integrity check: %s...\n", descr); } From cf8fc651ea87b2f3b44071161f42db8b7d9aecaa Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 11 Oct 2024 13:50:28 -0400 Subject: [PATCH 280/373] Remove direct call to srand(), just recommend the caller use it. Instead of taking a seed argument, just treat `int randomized` like a flag. (I'd use `bool`, but the header consistently uses `int` for boolean arguments.) --- include/fsm/walk.h | 9 +++++---- src/libfsm/gen.c | 12 ++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/include/fsm/walk.h b/include/fsm/walk.h index ea5a223e0..cb97e989a 100644 --- a/include/fsm/walk.h +++ b/include/fsm/walk.h @@ -90,9 +90,10 @@ fsm_walk_edges(const struct fsm *fsm, void *opaque, * functionally equivalent cases makes testing dramatically faster, * but exploring every edge could be added later. * - * If seed is zero then it will generate the first label in the label - * set, otherwise a label from the set will be chosen using rand() - * (favoring printable characters). + * If randomized is zero then it will generate the first label in the + * label set, otherwise a label from the set will be chosen using rand() + * (favoring printable characters). The caller can use srand() + * beforehand to set a PRNG seed. * * Note: fsm is non-const because it calls fsm_trim on the FSM * internally. This records the shortest distance from each state to an @@ -118,7 +119,7 @@ fsm_generate_matches_cb(const struct fsm *fsm, const char *input, size_t input_length, fsm_state_t end_state, void *opaque); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, +fsm_generate_matches(struct fsm *fsm, size_t max_length, int randomized, fsm_generate_matches_cb *cb, void *opaque); /* Callback provided for the most basic use case for diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 9f78e67db..8b8551489 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -107,7 +107,7 @@ struct gen_ctx { static bool gen_init_outer(struct fsm *fsm, size_t max_length, fsm_generate_matches_cb *cb, void *opaque, - bool randomized, unsigned seed); + bool randomized); static bool gen_init(struct gen_ctx *ctx, struct fsm *fsm); @@ -140,7 +140,7 @@ static bool grow_stack(struct gen_ctx *ctx); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, +fsm_generate_matches(struct fsm *fsm, size_t max_length, int randomized, fsm_generate_matches_cb *cb, void *opaque) { if (max_length == 0) { @@ -154,7 +154,7 @@ fsm_generate_matches(struct fsm *fsm, size_t max_length, unsigned seed, INIT_TIMERS(); TIME(&pre); - int res = gen_init_outer(fsm, max_length, cb, opaque, seed != 0, seed); + int res = gen_init_outer(fsm, max_length, cb, opaque, randomized != 0); TIME(&post); DIFF_MSEC("fsm_generate_matches", pre, post, NULL); @@ -204,7 +204,7 @@ fsm_generate_cb_printf(const struct fsm *fsm, static bool gen_init_outer(struct fsm *fsm, size_t max_length, fsm_generate_matches_cb *cb, void *opaque, - bool randomized, unsigned seed) + bool randomized) { int res = false; if (fsm == NULL || cb == NULL || max_length == 0) { @@ -213,10 +213,6 @@ gen_init_outer(struct fsm *fsm, size_t max_length, assert(fsm_all(fsm, fsm_isdfa)); /* DFA-only */ - if (randomized) { - srand(seed); - } - #if LOG_GEN > 1 fprintf(stderr, "%s: %u states\n", __func__, fsm_countstates(fsm)); #endif From 92e6e827e2d69eb28110a26628b15d353ec498a6 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 16 Sep 2024 16:00:21 -0400 Subject: [PATCH 281/373] Add re_is_anchored interface. --- include/re/re.h | 16 ++++++++++++++++ src/libre/libre.syms | 1 + src/libre/re.c | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/include/re/re.h b/include/re/re.h index 20408e98a..841e4e946 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -136,6 +136,22 @@ re_comp(enum re_dialect dialect, const struct fsm_alloc *alloc, enum re_flags flags, struct re_err *err); +/* Parse and analyze the regex enough to determine whether it is + * anchored at the start and/or end. + * + * As long as the result is checked for RE_IS_ANCHORED_ERROR first, + * the result can be used like a bitset. */ +enum re_is_anchored_res { + RE_IS_ANCHORED_NONE = 0x00, + RE_IS_ANCHORED_START = 0x01, + RE_IS_ANCHORED_END = 0x02, + RE_IS_ANCHORED_BOTH = 0x03, + RE_IS_ANCHORED_ERROR = 0xFFFF, +}; +enum re_is_anchored_res +re_is_anchored(enum re_dialect dialect, re_getchar_fun *f, void *opaque, + enum re_flags flags, struct re_err *err); + /* * Return a human-readable string describing a given error code. The string * returned has static storage, and must not be freed. diff --git a/src/libre/libre.syms b/src/libre/libre.syms index a4f1a223b..9d381cb0f 100644 --- a/src/libre/libre.syms +++ b/src/libre/libre.syms @@ -3,6 +3,7 @@ re_is_literal re_flags re_strerror re_perror +re_is_anchored ast_print ast_print_dot diff --git a/src/libre/re.c b/src/libre/re.c index 15af848b5..013e2b58c 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -335,3 +335,40 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, return -1; } +enum re_is_anchored_res +re_is_anchored(enum re_dialect dialect, re_getchar_fun *getc, void *opaque, + enum re_flags flags, struct re_err *err) +{ + /* FIXME: copy/pasted from above, factor out common code later. */ + + struct ast *ast; + const struct dialect *m; + int unsatisfiable; + + assert(getc != NULL); + + m = re_dialect(dialect); + if (m == NULL) { + if (err != NULL) { err->e = RE_EBADDIALECT; } + return RE_IS_ANCHORED_ERROR; + } + + flags |= m->flags; + + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); + if (ast == NULL) { + return RE_IS_ANCHORED_ERROR; + } + + /* Copy anchoring flags, ending up with NONE, START, END, or BOTH. */ + enum re_is_anchored_res res = RE_IS_ANCHORED_NONE; + if (ast->expr->flags & AST_FLAG_ANCHORED_START) { + res |= RE_IS_ANCHORED_START; + } + if (ast->expr->flags & AST_FLAG_ANCHORED_END) { + res |= RE_IS_ANCHORED_END; + } + + ast_free(ast); + return res; +} From 98dfb2dc8ea8c42483906b8ed5555f16c55ca369 Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 23 Oct 2024 11:15:01 +0100 Subject: [PATCH 282/373] Missing documentation for -x --- man/rx.1/rx.1.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/man/rx.1/rx.1.xml b/man/rx.1/rx.1.xml index a9f1ffd18..27d358177 100644 --- a/man/rx.1/rx.1.xml +++ b/man/rx.1/rx.1.xml @@ -44,6 +44,7 @@ -v"> -w"> -X"> + -x"> -h"> ]> @@ -462,6 +463,17 @@ + + &x.opt; + + + Literals are unanchored. + This applies to all literals; for finer control use a regex dialect. + The default is that literals are anchored, + as if written ^abc$ in regex syntax. + + + &h.opt; From 280ca7c30da78b12925a67531dea3be48ed4fc3c Mon Sep 17 00:00:00 2001 From: Kate F Date: Wed, 23 Oct 2024 11:19:45 +0100 Subject: [PATCH 283/373] Too much documentation for -u. --- man/rx.1/rx.1.xml | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/man/rx.1/rx.1.xml b/man/rx.1/rx.1.xml index 27d358177..01956b47a 100644 --- a/man/rx.1/rx.1.xml +++ b/man/rx.1/rx.1.xml @@ -404,29 +404,20 @@ &u.opt; - Allow ambiguities. - This means patterns with different ids may match the same text. - The default is to error for conflicts. + Allow ambiguities between patterns. + This means patterns with different ids may match the same text. - It's possible to have multiple patterns with the same id - (i.e. by being in the same file when using multi-file mode), - and these are not considered a conflict because they key - to the same id. - - - - - &u.opt; - - - Allow ambiguities between regexps, - such that multiple regexps may match the same text. - The default is to error for ambiguities, + The default is to error for ambiguities, requiring all regexps unioned to be non-overlapping. Formally, the requirement is that they are disjoint languages. Erroring for ambiguities applies after multiple regexps are joined, either by union or by concatenation (&s.opt;). + It's possible to have multiple patterns with the same id + (i.e. by being in the same file when using multi-file mode), + and these are not considered a conflict because they key + to the same id. + &u.opt; is implied by &n.opt;. From 7ebc1ebf0745b7fdbaa05197650ed3a66b26adf5 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 7 Nov 2024 17:16:37 -0500 Subject: [PATCH 284/373] Fix memory leak. The edge sets leak when halting with FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED. --- src/libfsm/determinise.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index fc7c68ba4..d5ba396a4 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -237,6 +237,7 @@ fsm_determinise_with_config(struct fsm *nfa, assert(dfa->states[m->dfastate].edges == NULL); dfa->states[m->dfastate].edges = m->edges; + m->edges = NULL; /* transfer ownership */ /* * The current DFA state is an end state if any of its associated NFA @@ -593,6 +594,8 @@ map_free(struct map *map) if (b == NULL) { continue; } + /* free any edge sets that didn't get transferred */ + edge_set_free(map->alloc, b->edges); f_free(map->alloc, b); } From c9a240a62ce54b0b0cbf883612ffc6e199ca7f23 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 8 Nov 2024 16:22:57 -0500 Subject: [PATCH 285/373] Add a regression test for the memory leak. I verified that LeakSanitizer catches the leak during this test when the fix is reverted. This commit adds a test directory, `tests/regressions`, for misc. regression tests that need a .c file and aren't easily testable via fsm, re, etc. inputs. --- Makefile | 1 + tests/regressions/Makefile | 20 ++++++ ...regressions_determinise_state_limit_leak.c | 69 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 tests/regressions/Makefile create mode 100644 tests/regressions/regressions_determinise_state_limit_leak.c diff --git a/Makefile b/Makefile index 514f80bba..7075e432d 100644 --- a/Makefile +++ b/Makefile @@ -136,6 +136,7 @@ SUBDIR += tests/pcre-repeat SUBDIR += tests/pred SUBDIR += tests/re_literal SUBDIR += tests/re_strings +SUBDIR += tests/regressions SUBDIR += tests/reverse SUBDIR += tests/trim SUBDIR += tests/union diff --git a/tests/regressions/Makefile b/tests/regressions/Makefile new file mode 100644 index 000000000..624353ef6 --- /dev/null +++ b/tests/regressions/Makefile @@ -0,0 +1,20 @@ +.include "../../share/mk/top.mk" + +TEST.tests/regressions != ls -1 tests/regressions/regressions*.c +TEST_SRCDIR.tests/regressions = tests/regressions +TEST_OUTDIR.tests/regressions = ${BUILD}/tests/regressions + +.for n in ${TEST.tests/regressions:T:R:C/^regressions//} +INCDIR.${TEST_SRCDIR.tests/regressions}/regressions${n}.c += src/adt +.endfor + +.for n in ${TEST.tests/regressions:T:R:C/^regressions//} +test:: ${TEST_OUTDIR.tests/regressions}/res${n} +SRC += ${TEST_SRCDIR.tests/regressions}/regressions${n}.c +CFLAGS.${TEST_SRCDIR.tests/regressions}/regressions${n}.c += -UNDEBUG + +${TEST_OUTDIR.tests/regressions}/run${n}: ${TEST_OUTDIR.tests/regressions}/regressions${n}.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/regressions}/regressions${n}.c} -o ${TEST_OUTDIR.tests/regressions}/run${n} ${TEST_OUTDIR.tests/regressions}/regressions${n}.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a +${TEST_OUTDIR.tests/regressions}/res${n}: ${TEST_OUTDIR.tests/regressions}/run${n} + ( ${TEST_OUTDIR.tests/regressions}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/regressions}/res${n} +.endfor diff --git a/tests/regressions/regressions_determinise_state_limit_leak.c b/tests/regressions/regressions_determinise_state_limit_leak.c new file mode 100644 index 000000000..afdfbd9c5 --- /dev/null +++ b/tests/regressions/regressions_determinise_state_limit_leak.c @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include + +static const char *strings[] = { + [0] = "apple", + [1] = "banana", + [2] = "carrot", + [3] = "durian", + [4] = "eggplant", +}; +#define STRING_COUNT sizeof(strings)/sizeof(strings[0]) + +int main(void) +{ + struct fsm *fsms[STRING_COUNT] = {0}; + + for (size_t i = 0; i < STRING_COUNT; i++) { + fsms[i] = re_comp(RE_PCRE, fsm_sgetc, &strings[i], NULL, 0, NULL); + assert(fsms[i] != NULL); + } + + struct fsm *combined_fsm = fsm_union_array(STRING_COUNT, fsms, NULL); + assert(combined_fsm != NULL); + + size_t state_limit_base = fsm_countstates(combined_fsm); + size_t max_state_limit = state_limit_base + 100; + + bool hit_state_limit = false; + + for (size_t state_limit = state_limit_base; state_limit < max_state_limit; state_limit += 10) { + struct fsm *cp = fsm_clone(combined_fsm); + + const struct fsm_determinise_config det_config = { + .state_limit = state_limit, + }; + + /* Previously this would leak memory when hitting the STATE_LIMIT_REACHED + * early exit, because the edge sets for the DFA being constructed were + * not freed properly. + * + * The first time this should fail immediately because the state limit IS the starting size, + * but later on it should halt in the middle of construction. */ + switch (fsm_determinise_with_config(cp, &det_config)) { + case FSM_DETERMINISE_WITH_CONFIG_OK: + fsm_free(cp); + break; + case FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED: + hit_state_limit = true; + fsm_free(cp); + break; + case FSM_DETERMINISE_WITH_CONFIG_ERRNO: + assert(!"internal error"); + return EXIT_FAILURE; + } + } + + assert(hit_state_limit); + + fsm_free(combined_fsm); + return EXIT_SUCCESS; +} From 3572258a2688ed843ffc4648021d54d252a58779 Mon Sep 17 00:00:00 2001 From: Ricky Hosfelt Date: Tue, 11 Feb 2025 11:57:57 -0500 Subject: [PATCH 286/373] Update CI to use Ubuntu 22.04 for now so we do not get bleeding edge clang and fail builds --- .github/workflows/ci.yml | 59 +++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfb6182f2..157506a14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: checkout: name: "Checkout" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Cache checkout @@ -44,7 +44,10 @@ jobs: pcre_suite: name: "Import PCRE suite" - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-22.04 ] needs: [ build ] # for cvtpcre steps: @@ -71,7 +74,7 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -79,7 +82,7 @@ jobs: id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitrary build, just for cvtpcre - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -130,14 +133,14 @@ jobs: build: name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ checkout ] strategy: fail-fast: true matrix: san: [ NO_SANITIZER, AUSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -169,7 +172,7 @@ jobs: key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && steps.cache-build.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-22.04' && steps.cache-build.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake electric-fence @@ -216,14 +219,14 @@ jobs: # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ checkout, build ] strategy: fail-fast: false matrix: san: [ NO_SANITIZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake, pmake ] debug: [ EXPENSIVE_CHECKS, DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -258,7 +261,7 @@ jobs: run: find ${{ env.wc }} -type f -name '*.c' | sort -r | head -5 | xargs touch - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install pmake bmake pcregrep @@ -301,14 +304,14 @@ jobs: test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ build ] strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -329,7 +332,7 @@ jobs: key: checkout-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake pcregrep electric-fence @@ -362,7 +365,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] @@ -370,7 +373,7 @@ jobs: fail-fast: false matrix: san: [ FUZZER ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -388,7 +391,7 @@ jobs: key: checkout-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake @@ -470,14 +473,14 @@ jobs: test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ pcre_suite ] # and also build, but pcre_suite gives us that strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -492,7 +495,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && matrix.san == 'EFENCE' + if: matrix.os == 'ubuntu-22.04' && matrix.san == 'EFENCE' run: | uname -a sudo apt-get install electric-fence @@ -506,7 +509,7 @@ jobs: ${{ matrix.cc }} --version - name: Dependencies (Ubuntu/Go) - if: matrix.os == 'ubuntu' && (matrix.lang == 'go' || matrix.lang == 'goasm') + if: matrix.os == 'ubuntu-22.04' && (matrix.lang == 'go' || matrix.lang == 'goasm') run: | uname -a sudo apt-get install golang @@ -524,14 +527,14 @@ jobs: id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst docs: name: Documentation - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ checkout ] env: @@ -582,12 +585,12 @@ jobs: install: name: Install - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ build, docs ] env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -601,7 +604,7 @@ jobs: key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' && steps.cache-prefix.outputs.cache-hit != 'true' + if: env.os == 'ubuntu-22.04' && steps.cache-prefix.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake @@ -643,12 +646,12 @@ jobs: fpm: name: Package ${{ matrix.pkg }} - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ install ] env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -661,7 +664,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' + if: env.os == 'ubuntu-22.04' run: | uname -a sudo gem install --no-document fpm From c89c15ddbb37d119671fd72219d994a89a413c4f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 17 Jun 2025 12:24:28 -0400 Subject: [PATCH 287/373] retest: Remove `isatty` check and extra logging output. While discussing other issues, kate said she'd rather not have this have different behavior when stdout is / isn't a tty. --- src/retest/main.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/retest/main.c b/src/retest/main.c index 53eba216c..08850325b 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -56,7 +56,6 @@ struct match { struct match *next; }; -static int tty_output = 0; static int do_timing = 0; static int do_watchdog = 0; @@ -969,14 +968,6 @@ process_test_file(const char *filename, flagstring(flags, &flagdesc[0]); - if (tty_output) { - char *re = dup_str_esc(regexp, NULL); - printf("[ ] line %d: working on %s regexp /%s/%s ...\r", - linenum, dialect_name, re, flagdesc); - fflush(stdout); - free(re); - } - re_str = regexp; fsm = re_comp(dialect, fsm_sgetc, &re_str, alloc, flags, &err); if (fsm == NULL) { @@ -1230,9 +1221,6 @@ main(int argc, char *argv[]) int optlevel = 1; - /* is output to a tty or not? */ - tty_output = isatty(fileno(stdout)); - /* note these defaults are the opposite than for fsm(1) */ opt.anonymous_states = 1; opt.consolidate_edges = 1; From 825303f382619bba8d431fd45cb53192a43c4286 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 3 Jul 2025 12:54:25 -0400 Subject: [PATCH 288/373] Remove dependence on internal.h in print/ir.h and lx/print/c.c. print/ir.h only depended on it for FSM_SIGMA_COUNT in `struct ir_state_table`, and IR_TABLE isn't actually implemented, so this can be removed for now. lx/print/c.c only needed internal.h because of print/ir.h, and because of several direct accesses to fsm->statecount, which are easily replaced by calls to fsm_countstates. --- src/libfsm/print/ir.h | 6 +----- src/lx/print/c.c | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index b375ba850..074097da3 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -91,11 +91,7 @@ struct ir_state { } error; struct { - /* Note: This is allocated separately, to avoid - * making the union significantly larger. */ - struct ir_state_table { - unsigned to[FSM_SIGMA_COUNT]; - } *table; + int not_yet_implemented; } table; } u; }; diff --git a/src/lx/print/c.c b/src/lx/print/c.c index dae4bd937..4f06475ac 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -18,7 +18,6 @@ #include -#include "libfsm/internal.h" /* XXX */ #include "libfsm/print/ir.h" /* XXX */ #include "lx/lx.h" @@ -38,7 +37,7 @@ skip(const struct fsm *fsm, fsm_state_t state) struct ast_mapping *m; assert(fsm != NULL); - assert(state < fsm->statecount); + assert(state < fsm_countstates(fsm)); if (!fsm_isend(fsm, state)) { return 1; @@ -100,7 +99,8 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, (void) fsm_getstart(fsm, &goal); min = INT_MAX; - for (i = 0; i < fsm->statecount; i++) { + const size_t statecount = fsm_countstates(fsm); + for (i = 0; i < statecount; i++) { const struct ast_mapping *m; int n; @@ -607,10 +607,11 @@ print_stateenum(FILE *f, const struct fsm *fsm) fprintf(f, "\tenum {\n"); fprintf(f, "\t\t"); - for (i = 0; i < fsm->statecount; i++) { + const size_t statecount = fsm_countstates(fsm); + for (i = 0; i < statecount; i++) { fprintf(f, "S%u, ", i); - if (i + 1 < fsm->statecount && (i + 1) % 10 == 0) { + if (i + 1 < statecount && (i + 1) % 10 == 0) { fprintf(f, "\n"); fprintf(f, "\t\t"); } @@ -718,7 +719,8 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, has_skips = 0; - for (i = 0; i < z->fsm->statecount; i++) { + const size_t statecount = fsm_countstates(z->fsm); + for (i = 0; i < statecount; i++) { int r; r = fsm_reachableall(z->fsm, i, skip); @@ -740,7 +742,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, fprintf(f, "\n"); fprintf(f, "\t\tswitch (state) {\n"); - for (i = 0; i < z->fsm->statecount; i++) { + for (i = 0; i < statecount; i++) { int r; r = fsm_reachableall(z->fsm, i, skip); @@ -806,7 +808,8 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, fprintf(f, "\tcase NONE: return %sEOF;\n", prefix.tok); - for (i = 0; i < z->fsm->statecount; i++) { + const size_t statecount = fsm_countstates(z->fsm); + for (i = 0; i < statecount; i++) { const struct ast_mapping *m; if (!fsm_isend(z->fsm, i)) { From cb42d58f69bfdf5b48a0c186d86ca1c236fa7e67 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 8 Jul 2025 12:03:27 -0400 Subject: [PATCH 289/373] Makefile: Check '*res*' not 'res*' for tests. This will miss failures in prefixed res files, such as build/tests/lxpos/dyn-fdgetc-getc-res0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7075e432d..6f6d9a2e3 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,6 @@ STAGE_BUILD := ${STAGE_BUILD:Nbin/cvtpcre} .if make(test) .END:: - grep FAIL ${BUILD}/tests/*/res*; [ $$? -ne 0 ] + grep FAIL ${BUILD}/tests/*/*res*; [ $$? -ne 0 ] .endif From 7824e6a223f09a3bb2965b3d45917e7397aa9da1 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 3 Jul 2025 14:29:34 -0400 Subject: [PATCH 290/373] Fix lx by rewriting much of its -l c codegen. Changing the leaf and endleaf callbacks to accept and reject in #485 broke lx, but it went unnoticed for a while. This fixes it. libfsm's normal execution mode evaluates a DFA, character by character, terminating either when the next character isn't a valid edge or the end of input is reached (in which case it checks end state metadata). lx's execution mode is a little different, because it's tokenizing -- instead of reading to the end of input, it should consume as much consecutive input that matches a particular token, then push back the last character read (so it can resume with it as context for the next token), yield the token type, and suspend. lx used to work by breaking abstraction and calling directly into `fsm_print_cfrag` (overriding the leaf behavior to yield token types, and adding an extra 'NONE' state to the generated state machine code), but when the callback interfaces shifted its internals no longer fit what lx expected. Now the reject hook is passed the same state metadata as the accept state, and the reject hook in lx checks whether the end id is associated with a particular AST mapping and token type. This is only implemented for the "c" output format, but similar changes could possibly make others usable without a lot more work. In particular, kate mentioned it'd be good to be able to use "vmc" output instead of "c" moving forward. Most of the code changes happen inside of lx's code generation, but there are a few elsewhere: - The reject hook now has a state_metadata pointer, so update the callers for all the output formats. - libfsm's 'c' output now includes a macro `FSM_ADVANCE_HOOK(C)`, which is called with the next character read in the FSM_IO_STR and FSM_IO_PAIR io modes immediately after advancing. This is used to inform lx's internal bookkeeping about token positions and buffering token names. FSM_IO_GETC doesn't need it, because its getc callback manages the character stream. The macro defaults to a no-op when undefined. - libfsm's 'c' output also includes a flag, `has_consumed_input`, so the code expanded in place from the reject/accept hooks can determine when the state machine input handler loop has consumed any input. This was previously encoded by the extra NONE state. lx's code generation using this flag is a bit cluttered, because the reject hook doesn't know whether it's expanding for the end states, but it's probably not worth changing the reject hook type signature to add another flag. This results in checks for has_consumed_input in code paths where trivial static analysis would show it to be dead code, and some extra unreachable code at the end of the function. --- include/fsm/print.h | 1 + src/libfsm/print.c | 3 +- src/libfsm/print.h | 1 + src/libfsm/print/awk.c | 2 +- src/libfsm/print/c.c | 62 +++++- src/libfsm/print/dot.c | 2 +- src/libfsm/print/fsm.c | 2 +- src/libfsm/print/go.c | 2 +- src/libfsm/print/llvm.c | 2 +- src/libfsm/print/rust.c | 2 +- src/libfsm/print/sh.c | 2 +- src/libfsm/print/vmasm.c | 2 +- src/libfsm/print/vmc.c | 2 +- src/libfsm/print/vmdot.c | 2 +- src/libfsm/print/vmops.c | 2 +- src/lx/main.c | 4 + src/lx/print/c.c | 459 ++++++++++++++++++++++----------------- src/lx/print/dump.c | 20 +- 18 files changed, 348 insertions(+), 224 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index 9f7264e81..6fae2abea 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -75,6 +75,7 @@ struct fsm_hooks { void *lang_opaque, void *hook_opaque); int (*reject)(FILE *, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque); int (*comment)(FILE *, const struct fsm_options *opt, diff --git a/src/libfsm/print.c b/src/libfsm/print.c index e1a425cc0..bdb092d96 100644 --- a/src/libfsm/print.c +++ b/src/libfsm/print.c @@ -111,6 +111,7 @@ int print_hook_reject(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, int (*default_reject)(FILE *f, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque), void *lang_opaque) @@ -124,7 +125,7 @@ print_hook_reject(FILE *f, } if (hooks->reject != NULL) { - return hooks->reject(f, opt, + return hooks->reject(f, opt, state_metadata, lang_opaque, hooks->hook_opaque); } else if (default_reject != NULL) { return default_reject(f, opt, diff --git a/src/libfsm/print.h b/src/libfsm/print.h index 0a8e2e0c1..bfedf0e62 100644 --- a/src/libfsm/print.h +++ b/src/libfsm/print.h @@ -44,6 +44,7 @@ int print_hook_reject(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, int (*default_reject)(FILE *f, const struct fsm_options *opt, void *lang_opaque, void *hook_opaque), void *lang_opaque); diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index bd78e8ef7..4d952cbef 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -154,7 +154,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, hooks, default_reject, NULL); + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); case VM_END_SUCC:; struct fsm_state_metadata state_metadata = { diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 22b03963e..96117f21e 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -210,7 +210,7 @@ print_groups(FILE *f, const struct fsm_options *opt, } static int -print_case(FILE *f, const struct ir *ir, +print_case(FILE *f, const struct ir *ir, fsm_state_t state_id, const struct fsm_options *opt, const struct fsm_hooks *hooks, const char *cp, @@ -222,10 +222,16 @@ print_case(FILE *f, const struct ir *ir, assert(f != NULL); assert(cs != NULL); + assert(state_id < ir->n); + const struct fsm_state_metadata state_metadata = { + .end_ids = ir->states[state_id].endids.ids, + .end_id_count = ir->states[state_id].endids.count, + }; + switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -254,7 +260,7 @@ print_case(FILE *f, const struct ir *ir, print_groups(f, opt, ir_indexof(ir, cs), cs->u.partial.groups, cs->u.partial.n); fprintf(f, "\t\t\tdefault: "); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -285,7 +291,7 @@ print_case(FILE *f, const struct ir *ir, print_ranges(f, opt, cs->u.error.error.ranges, cs->u.error.error.n); fprintf(f, " "); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -398,7 +404,7 @@ print_endstates(FILE *f, /* unexpected EOT */ fprintf(f, "\tdefault: "); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -435,7 +441,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, } fprintf(f, "\n"); - if (-1 == print_case(f, ir, opt, hooks, cp, &ir->states[i])) { + if (-1 == print_case(f, ir, i, opt, hooks, cp, &ir->states[i])) { return -1; } @@ -478,6 +484,41 @@ fsm_print_c_body(FILE *f, const struct ir *ir, } } + /* This flag indicates whether the any of the input stream was + * consumed before getting EOF and skipping over the state and + * character logic expanded here. + * + * lx needs to track this for proper EOF handling. It previously + * generated the state enum itself, so that it could include an + * additional 'NONE' state. Inside the input loop, the default + * state of NONE would be updated to the start state, but if the + * input loop was skipped it would still be NONE. */ + fprintf(f, "\tint has_consumed_input = 0;\n"); + + /* For FSM_IO_STR and FSM_IO_PAIR, define a macro that will be + * called with the new character every time iteration advances. + * This is used by lx's internal bookkeeping to track token + * positions in the input stream. For FSM_IO_GETC, the generated + * getc function handles this directly. + * + * This defaults to a no-op unless defined. */ + switch (opt->io) { + case FSM_IO_GETC: + break; /* nothing to do */ + + case FSM_IO_STR: + fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); + fprintf(f, "#define FSM_ADVANCE_HOOK(C) /* no-op */ (void)C\n"); + fprintf(f, "#endif\n"); + break; + + case FSM_IO_PAIR: + fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); + fprintf(f, "#define FSM_ADVANCE_HOOK(C) /* no-op */ (void)C\n"); + fprintf(f, "#endif\n"); + break; + } + /* enum of states */ print_stateenum(f, ir->n); fprintf(f, "\n"); @@ -489,14 +530,19 @@ fsm_print_c_body(FILE *f, const struct ir *ir, switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\twhile (c = fsm_getc(getc_opaque), c != EOF) {\n"); + fprintf(f, "\t\thas_consumed_input = 1;\n"); break; case FSM_IO_STR: fprintf(f, "\tfor (p = s; *p != '\\0'; p++) {\n"); + fprintf(f, "\t\thas_consumed_input = 1;\n"); + fprintf(f, "\t\tFSM_ADVANCE_HOOK(%s);\n", cp); break; case FSM_IO_PAIR: fprintf(f, "\tfor (p = b; p != e; p++) {\n"); + fprintf(f, "\t\thas_consumed_input = 1;\n"); + fprintf(f, "\t\tFSM_ADVANCE_HOOK(%s);\n", cp); break; } @@ -507,6 +553,10 @@ fsm_print_c_body(FILE *f, const struct ir *ir, fprintf(f, "\t}\n"); fprintf(f, "\n"); + /* Suppress unused variable warning -- this is mainly for lx. */ + fprintf(f, "\t(void)has_consumed_input;\n"); + fprintf(f, "\n"); + /* end states */ if (-1 == print_endstates(f, opt, hooks, ir)) { return -1; diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index c1ac9b875..69cce10aa 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -226,7 +226,7 @@ print_dotfrag(FILE *f, if (!opt->anonymous_states) { fprintf(f, "\t%sS%-2u [ ", prefix, s); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, &s)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, &s)) { return -1; } diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index 06c7e3403..027531514 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -184,7 +184,7 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hook assert(opt != NULL); if (!fsm_isend(fsm, s)) { - if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL, NULL)) { return -1; } } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index 3575c5b75..1f6b7a0dc 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -185,7 +185,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, hooks, default_reject, NULL); + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); case VM_END_SUCC: assert(op->ret >= retlist->a); diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c index 14b116555..98e76d46f 100644 --- a/src/libfsm/print/llvm.c +++ b/src/libfsm/print/llvm.c @@ -713,7 +713,7 @@ fsm_print_llvm(FILE *f, fprintf(f, "\n"); } fprintf(f, "\t "); - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) { return -1; } fprintf(f, "\n"); diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index 682bc051f..c1fcf5bb4 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -174,7 +174,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, hooks, default_reject, NULL); + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); case VM_END_SUCC: assert(op->ret >= retlist->a); diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 3c11c2f1f..5322e4ffc 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -202,7 +202,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, hooks, default_reject, NULL); + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); case VM_END_SUCC:; const struct fsm_state_metadata state_metadata = { diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index e29eda24d..bf5551420 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -44,7 +44,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, switch (end_bits) { case VM_END_FAIL: - if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL, NULL)) { return -1; } break; diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 2fef2da65..9ea06f27a 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -175,7 +175,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, hooks, default_reject, NULL); + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); case VM_END_SUCC:; struct fsm_state_metadata state_metadata = { diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 512cc893b..57344cddb 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -126,7 +126,7 @@ print_end(FILE *f, { switch (end_bits) { case VM_END_FAIL: - return print_hook_reject(f, opt, hooks, default_reject, NULL); + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); case VM_END_SUCC:; struct fsm_state_metadata state_metadata = { diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 59de464f1..74fc6d350 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -146,7 +146,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op, switch (end_bits) { case VM_END_FAIL: - if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) { return -1; } break; diff --git a/src/lx/main.c b/src/lx/main.c index ea16ee70d..92098bfd9 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -695,6 +695,10 @@ main(int argc, char *argv[]) opt.comments = 0; } + if (lang == LX_PRINT_C) { + opt.fragment = 1; + } + { if (print_progress) { fprintf(stderr, "-- parsing:"); diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 4f06475ac..18f6d664a 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -18,18 +18,17 @@ #include -#include "libfsm/print/ir.h" /* XXX */ - #include "lx/lx.h" #include "lx/ast.h" #include "lx/print.h" -/* XXX: abstraction */ -int -fsm_print_cfrag(FILE *f, const struct ir *ir, - const struct fsm_options *opt, - const struct fsm_hooks *hooks, - const char *cp); +struct lx_hook_env { + const struct ast *ast; + /* Name of variable for the current character of input in the + * current scope, which depends on the IO options. */ + const char *cur_char_var; +}; + static int skip(const struct fsm *fsm, fsm_state_t state) @@ -137,6 +136,29 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, return 1; } +static const char * +buf_op_prefix(void) +{ + if (api_tokbuf & API_FIXEDBUF) { + return "fixed"; + } else if (api_tokbuf & API_DYNBUF) { + return "dyn"; + } else { + assert(!"buf is neither fixed nor dyn"); + return NULL; + } +} + +static void +unget_character(FILE *f, bool pop, const char *cur_char_var) +{ + fprintf(f, "%sungetc(lx, %s); ", prefix.api, cur_char_var); + if (pop && (~api_exclude & API_POS)) { + fprintf(f, "%s%spop(lx->buf_opaque); ", + prefix.api, buf_op_prefix()); + } +} + static int accept_c(FILE *f, const struct fsm_options *opt, const struct fsm_state_metadata *state_metadata, @@ -144,6 +166,7 @@ accept_c(FILE *f, const struct fsm_options *opt, { const struct ast *ast; const struct ast_mapping *m; + struct lx_hook_env *env = hook_opaque; assert(f != NULL); assert(opt != NULL); @@ -152,11 +175,20 @@ accept_c(FILE *f, const struct fsm_options *opt, assert(lang_opaque == NULL); assert(hook_opaque != NULL); - ast = hook_opaque; + ast = env->ast; m = ast_getendmappingbyendid(state_metadata->end_ids[0]); - /* XXX: don't need this if complete */ - fprintf(f, "%sungetc(lx, c); ", prefix.api); + /* re-sync before new call into zone */ + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + case FSM_IO_PAIR: + fprintf(f, "lx->p = p; "); + break; + } + fprintf(f, "return "); if (m->to != NULL) { fprintf(f, "lx->z = z%u, ", zindexof(ast, m->to)); @@ -168,12 +200,12 @@ accept_c(FILE *f, const struct fsm_options *opt, fprintf(f, "lx->z(lx)"); } fprintf(f, ";"); - return 0; } static int reject_c(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque) { assert(f != NULL); @@ -182,7 +214,58 @@ reject_c(FILE *f, const struct fsm_options *opt, assert(hook_opaque != NULL); (void) lang_opaque; - (void) hook_opaque; + struct lx_hook_env *env = hook_opaque; + + const struct ast_mapping *m = state_metadata != NULL && state_metadata->end_id_count > 0 + ? ast_getendmappingbyendid(state_metadata->end_ids[0]) + : NULL; + + /* If there is an AST mapping associated with this end state, + * then unget the previous character (in most cases), and + * possibly emit its token type and/or new z state. */ + if (m != NULL) { + const bool has_endids = state_metadata && state_metadata->end_id_count > 0; + if (m->token == NULL && m->to == NULL && !has_endids) { + unget_character(f, true, env->cur_char_var); + } else if (m->token == NULL && m->to == NULL && has_endids) { + unget_character(f, true, env->cur_char_var); + } else if (m->token == NULL && m->to != NULL) { + unget_character(f, true, env->cur_char_var); + } else if (m->token != NULL && m->to == NULL) { + assert(has_endids); + unget_character(f, true, env->cur_char_var); + } else if (m->token != NULL && m->to != NULL) { + unget_character(f, true, env->cur_char_var); + } + + /* re-sync before new call into zone */ + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + case FSM_IO_PAIR: + fprintf(f, "lx->p = p; "); + break; + } + + fprintf(f, "return "); + if (m->to != NULL) { + fprintf(f, "lx->z = z%u, ", zindexof(env->ast, m->to)); + } + if (m->token != NULL) { + fprintf(f, "%s", prefix.tok); + esctok(f, m->token->s); + } else { + fprintf(f, "lx->z(lx)"); + } + fprintf(f, ";"); + return 0; + } else { + fprintf(f, "\n\t\t\t\tif (!has_consumed_input) { return TOK_EOF; }\n"); + fprintf(f, "\t\t\t\t"); + unget_character(f, false, env->cur_char_var); + } /* XXX: don't need this if complete */ switch (opt->io) { @@ -191,16 +274,12 @@ reject_c(FILE *f, const struct fsm_options *opt, break; case FSM_IO_STR: - fprintf(f, "lx->p = NULL; "); - break; - case FSM_IO_PAIR: fprintf(f, "lx->p = NULL; "); break; } fprintf(f, "return %sUNKNOWN;", prefix.tok); - return 0; } @@ -216,20 +295,32 @@ print_proto(FILE *f, const struct ast *ast, const struct ast_zone *z) } static void -print_lgetc(FILE *f) +print_lgetc(FILE *f, const struct fsm_options *opt) { if (api_getc & API_FGETC) { if (print_progress) { fprintf(stderr, " fgetc"); } + if (opt->comments) { + fprintf(f, "/* Get a character from fgetc and push it to the buffer */\n"); + } fprintf(f, "int\n"); fprintf(f, "%sfgetc(struct %slx *lx)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); fprintf(f, "\tassert(lx != NULL);\n"); fprintf(f, "\tassert(lx->getc_opaque != NULL);\n"); fprintf(f, "\n"); - fprintf(f, "\treturn fgetc(lx->getc_opaque);\n"); + + fprintf(f, "\tconst int c = fgetc(lx->getc_opaque);\n"); + fprintf(f, "\tif (c == EOF) {\n"); + fprintf(f, "\t\tlx->c = EOF;\n"); + fprintf(f, "\t\treturn EOF;\n"); + fprintf(f, "\t} else {\n"); + + fprintf(f, "\t\treturn c;\n"); + fprintf(f, "\t}\n"); + fprintf(f, "}\n"); fprintf(f, "\n"); } @@ -346,22 +437,61 @@ print_io(FILE *f, const struct fsm_options *opt) fprintf(stderr, " io"); } - /* TODO: consider passing char *c, and return int 0/-1 for error */ - fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); - fprintf(f, "inline\n"); - fprintf(f, "#endif\n"); fprintf(f, "static int\n"); - fprintf(f, "lx_getc(struct %slx *lx)\n", prefix.lx); + fprintf(f, "lx_advance_end(struct lx *lx, int c)\n"); fprintf(f, "{\n"); - fprintf(f, "\tint c;\n"); - fprintf(f, "\n"); + if (~api_exclude & API_POS) { + fprintf(f, "\tlx->end.byte++;\n"); + fprintf(f, "\tlx->end.col++;\n"); - fprintf(f, "\tassert(lx != NULL);\n"); + fprintf(f, "\tif (c == '\\n') {\n"); + fprintf(f, "\t\tlx->end.line++;\n"); + fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); + fprintf(f, "\t\tlx->end.col = 1;\n"); + + if (opt->io == FSM_IO_STR) { /* ignore terminating '\0' */ + fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); + fprintf(f, "\t\tlx->end.byte--;\n"); + fprintf(f, "\t\tlx->end.col--;\n"); + fprintf(f, "\t}\n"); + } else { + fprintf(f, "\t}\n"); + } + } + + if (~api_exclude & API_BUF) { + fprintf(f, "\tif (lx->push != NULL) {\n"); + fprintf(f, "\t\tif (-1 == lx->push(lx->buf_opaque, (char)c)) {\n"); + fprintf(f, "\t\t\treturn 0;\n"); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t}\n"); + } + + fprintf(f, "\treturn 1;\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); switch (opt->io) { case FSM_IO_GETC: + /* TODO: consider passing char *c, and return int 0/-1 for error */ + if (opt->comments) { + fprintf(f, "/* This wrapper manages one character of lookahead/pushback\n"); + fprintf(f, " * and the line, column, and byte offsets. */\n"); + } + fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); + fprintf(f, "inline\n"); + fprintf(f, "#endif\n"); + fprintf(f, "static int\n"); + fprintf(f, "lx_getc(struct %slx *lx)\n", prefix.lx); + fprintf(f, "{\n"); + fprintf(f, "\tint c;\n"); + fprintf(f, "\n"); + + fprintf(f, "\tassert(lx != NULL);\n"); + fprintf(f, "\tassert(lx->lgetc != NULL);\n"); fprintf(f, "\n"); + fprintf(f, "\tif (lx->c != EOF) {\n"); fprintf(f, "\t\tc = lx->c, lx->c = EOF;\n"); fprintf(f, "\t} else {\n"); @@ -371,54 +501,39 @@ print_io(FILE *f, const struct fsm_options *opt) fprintf(f, "\t\t}\n"); fprintf(f, "\t}\n"); fprintf(f, "\n"); - break; - case FSM_IO_STR: - /* - * For FSM_IO_STR we treat '\0' as the end of input, - * and so there's no need to distinguish it from EOF. - * We return '\0' here to save the assignment. - */ - fprintf(f, "\tassert(lx->p != NULL);\n"); + /* FIXME: This should distinguish between alloc failure + * and EOF, but will require layers of interface changes. */ + fprintf(f, "\tif (!lx_advance_end(lx, c)) { return EOF; }\n"); fprintf(f, "\n"); - fprintf(f, "\tc = *lx->p++;\n"); - fprintf(f, "\n"); - break; - case FSM_IO_PAIR: - fprintf(f, "\tassert(lx->p != NULL);\n"); - fprintf(f, "\n"); - fprintf(f, "\tif (lx->p == lx->e) {\n"); - fprintf(f, "\t\t\treturn EOF;\n"); - fprintf(f, "\t}\n"); - fprintf(f, "\n"); - fprintf(f, "\tc = *lx->p++;\n"); + fprintf(f, "\treturn c;\n"); + fprintf(f, "}\n"); fprintf(f, "\n"); - break; - } - if (~api_exclude & API_POS) { - fprintf(f, "\tlx->end.byte++;\n"); - fprintf(f, "\tlx->end.col++;\n"); + /* Add an implementation of fsm_getc that calls back + * into lx_getc with the lx handle. */ + fprintf(f, "/* This wrapper adapts calling lx_getc to the interface\n"); + fprintf(f, " * in libfsm's generated code. */\n"); + fprintf(f, "static int\n"); + fprintf(f, "fsm_getc(void *getc_opaque)\n"); + fprintf(f, "{\n"); + + fprintf(f, "\treturn lx_getc((struct lx *)getc_opaque);\n"); + fprintf(f, "}\n"); fprintf(f, "\n"); - fprintf(f, "\tif (c == '\\n') {\n"); - fprintf(f, "\t\tlx->end.line++;\n"); - fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); - fprintf(f, "\t\tlx->end.col = 1;\n"); + break; - if (opt->io == FSM_IO_STR) { /* ignore terminating '\0' */ - fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); - fprintf(f, "\t\tlx->end.byte--;\n"); - fprintf(f, "\t\tlx->end.col--;\n"); - fprintf(f, "\t}\n"); - } else { - fprintf(f, "\t}\n"); - } + case FSM_IO_PAIR: + case FSM_IO_STR: + /* When libfsm's generated code advances a character, update + * lx's token name buffer and position bookkeeping. */ + fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); + fprintf(f, "#define FSM_ADVANCE_HOOK(C) if (!lx_advance_end(lx, C)) { return TOK_ERROR; }\n"); + fprintf(f, "#endif\n"); fprintf(f, "\n"); + break; } - fprintf(f, "\treturn c;\n"); - fprintf(f, "}\n"); - fprintf(f, "\n"); fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); fprintf(f, "inline\n"); @@ -431,30 +546,19 @@ print_io(FILE *f, const struct fsm_options *opt) switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tassert(lx->c == EOF);\n"); - fprintf(f, "\n"); fprintf(f, "\tlx->c = c;\n"); - fprintf(f, "\n"); break; case FSM_IO_STR: fprintf(f, "\tassert(lx->p != NULL);\n"); - fprintf(f, "\tassert(*(lx->p - 1) == c);\n"); - fprintf(f, "\n"); - fprintf(f, "\tlx->p--;\n"); - fprintf(f, "\n"); break; case FSM_IO_PAIR: fprintf(f, "\tassert(lx->p != NULL);\n"); - fprintf(f, "\tassert(*(lx->p - 1) == c);\n"); - fprintf(f, "\n"); - fprintf(f, "\tlx->p--;\n"); - fprintf(f, "\n"); break; } if (~api_exclude & API_POS) { - fprintf(f, "\n"); fprintf(f, "\tlx->end.byte--;\n"); fprintf(f, "\tlx->end.col--;\n"); fprintf(f, "\n"); @@ -468,7 +572,7 @@ print_io(FILE *f, const struct fsm_options *opt) } static void -print_buf(FILE *f) +print_buf(FILE *f, const struct fsm_options *opt) { if (api_tokbuf & API_DYNBUF) { if (print_progress) { @@ -518,7 +622,28 @@ print_buf(FILE *f) fprintf(f, "}\n"); fprintf(f, "\n"); + + if (~api_exclude & API_BUF) { + fprintf(f, "static void\n"); + fprintf(f, "%sdynpop(void *buf_opaque)\n", prefix.api); + fprintf(f, "{\n"); + fprintf(f, "\tstruct lx_dynbuf *t = buf_opaque;\n"); + fprintf(f, "\n"); + fprintf(f, "\tassert(t != NULL);\n"); + fprintf(f, "\n"); + + if (opt->io == FSM_IO_GETC) { + fprintf(f, "\tassert(t->p != t->a);\n"); + } + + fprintf(f, "\tt->p--;\n"); + + fprintf(f, "}\n"); + fprintf(f, "\n"); + } + fprintf(f, "int\n"); + /* FIXME: handle error from dynclear */ fprintf(f, "%sdynclear(void *buf_opaque)\n", prefix.api); fprintf(f, "{\n"); fprintf(f, "\tstruct lx_dynbuf *t = buf_opaque;\n"); @@ -542,6 +667,7 @@ print_buf(FILE *f) fprintf(f, "\n"); fprintf(f, "\tt->p = t->a;\n"); fprintf(f, "\n"); + fprintf(f, "\treturn 0;\n"); fprintf(f, "}\n"); fprintf(f, "\n"); @@ -582,6 +708,24 @@ print_buf(FILE *f) fprintf(f, "}\n"); fprintf(f, "\n"); + if (~api_exclude & API_BUF) { + fprintf(f, "static void\n"); + fprintf(f, "%sfixedpop(void *buf_opaque)\n", prefix.api); + fprintf(f, "{\n"); + fprintf(f, "\tstruct lx_fixedbuf *t = buf_opaque;\n"); + fprintf(f, "\n"); + fprintf(f, "\tassert(t != NULL);\n"); + fprintf(f, "\tassert(t->p != NULL);\n"); + fprintf(f, "\tassert(t->a != NULL);\n"); + + if (opt->io == FSM_IO_GETC) { + fprintf(f, "\tassert(t->p > t->a);\n"); + } + fprintf(f, "\tt->p--;\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); + } + fprintf(f, "int\n"); fprintf(f, "%sfixedclear(void *buf_opaque)\n", prefix.api); fprintf(f, "{\n"); @@ -599,50 +743,33 @@ print_buf(FILE *f) } } -static void -print_stateenum(FILE *f, const struct fsm *fsm) -{ - fsm_state_t i; - - fprintf(f, "\tenum {\n"); - fprintf(f, "\t\t"); - - const size_t statecount = fsm_countstates(fsm); - for (i = 0; i < statecount; i++) { - fprintf(f, "S%u, ", i); - - if (i + 1 < statecount && (i + 1) % 10 == 0) { - fprintf(f, "\n"); - fprintf(f, "\t\t"); - } - } - - fprintf(f, "NONE"); - - fprintf(f, "\n"); - fprintf(f, "\t} state;\n"); -} - static int print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, - const struct fsm_options *opt, const char *cp) + const struct fsm_options *opt, const char *cur_char_var) { assert(f != NULL); assert(z != NULL); assert(z->fsm != NULL); assert(fsm_all(z->fsm, fsm_isdfa)); assert(ast != NULL); - assert(cp != NULL); + assert(cur_char_var != NULL); - /* TODO: prerequisite that the FSM is a DFA */ + /* prerequisite that the FSM is a DFA */ + assert(fsm_all(z->fsm, fsm_isdfa)); fprintf(f, "static enum %stoken\n", prefix.api); fprintf(f, "z%u(struct %slx *lx)\n", zindexof(ast, z), prefix.lx); fprintf(f, "{\n"); - fprintf(f, "\tint c;\n"); - fprintf(f, "\n"); - print_stateenum(f, z->fsm); + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "\tint c;\n"); + break; + case FSM_IO_STR: + case FSM_IO_PAIR: + break; + } + fprintf(f, "\n"); fprintf(f, "\tassert(lx != NULL);\n"); @@ -655,9 +782,6 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, fprintf(f, "\n"); } - fprintf(f, "\tstate = NONE;\n"); - fprintf(f, "\n"); - if (~api_exclude & API_POS) { fprintf(f, "\tlx->start = lx->end;\n"); fprintf(f, "\n"); @@ -665,52 +789,37 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "\twhile (c = lx_getc(lx), c != EOF) {\n"); + fprintf(f, "\tvoid *getc_opaque = (void *)lx;\n"); + + /* This must be called with fragment sent, otherwise + * it will generate a nested function definition. */ + assert(opt->fragment); break; case FSM_IO_STR: - fprintf(f, "\twhile (c = lx_getc(lx), c != '\\0') {\n"); + fprintf(f, "const char *s = lx->p;\n"); + fprintf(f, "const char *p;\n"); break; case FSM_IO_PAIR: - fprintf(f, "\twhile (c = lx_getc(lx), c != EOF) {\n"); + fprintf(f, "\tconst char *p, *b = lx->p, *e = lx->e;\n"); break; } - { - fsm_state_t start; - - if (!fsm_getstart(z->fsm, &start)) { - errno = EINVAL; - return -1; - } - - fprintf(f, "\t\tif (state == NONE) {\n"); - fprintf(f, "\t\t\tstate = S%u;\n", start); - fprintf(f, "\t\t}\n"); - fprintf(f, "\n"); - } - { static const struct fsm_hooks defaults; struct fsm_hooks hooks = defaults; - struct ir *ir; - assert(cp != NULL); + struct lx_hook_env hook_env = { + .ast = ast, + .cur_char_var = cur_char_var, + }; hooks.accept = accept_c; hooks.reject = reject_c; - hooks.hook_opaque = (void *) ast; - - ir = make_ir(z->fsm, opt); - if (ir == NULL) { - /* TODO */ - } - - /* XXX: abstraction */ - (void) fsm_print_cfrag(f, ir, opt, &hooks, cp); + hooks.hook_opaque = &hook_env; - free_ir(z->fsm, ir); + fsm_print(f, z->fsm, opt, &hooks, FSM_PRINT_C); } if (~api_exclude & API_BUF) { @@ -762,7 +871,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, fprintf(f, "\t\tdefault:\n"); fprintf(f, "\t\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cp); + fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cur_char_var); fprintf(f, "\t\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t\t}\n"); fprintf(f, "\t\t\t}\n"); @@ -773,20 +882,16 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, } else { fprintf(f, "\n"); fprintf(f, "\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cp); + fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cur_char_var); fprintf(f, "\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t}\n"); fprintf(f, "\t\t}\n"); } } - fprintf(f, "\t}\n"); - fprintf(f, "\n"); { - fsm_state_t i; - switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tlx->lgetc = NULL;\n"); @@ -804,42 +909,10 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, break; } - fprintf(f, "\tswitch (state) {\n"); - - fprintf(f, "\tcase NONE: return %sEOF;\n", prefix.tok); - - const size_t statecount = fsm_countstates(z->fsm); - for (i = 0; i < statecount; i++) { - const struct ast_mapping *m; - - if (!fsm_isend(z->fsm, i)) { - continue; - } - - m = ast_getendmapping(z->fsm, i); - if (LOG()) { - fprintf(stderr, "print_zone: ast_getendmapping for state %d: %p (c)\n", - i, (void *)m); - } - assert(m != NULL); - - fprintf(f, "\tcase S%u: return ", (unsigned) i); - - /* note: no point in changing zone here, because getc is now NULL */ - - if (m->token == NULL) { - fprintf(f, "%sEOF;\n", prefix.tok); - } else { - /* TODO: maybe make a printf-like little language to simplify this */ - fprintf(f, "%s", prefix.tok); - esctok(f, m->token->s); - fprintf(f, ";\n"); - } - } - - fprintf(f, "\tdefault: errno = EINVAL; return %sERROR;\n", prefix.tok); - - fprintf(f, "\t}\n"); + fprintf(f, "\tif (!has_consumed_input) {\n"); + fprintf(f, "\t\treturn TOK_EOF;\n"); + fprintf(f, "\t} \n"); + fprintf(f, "\n"); } fprintf(f, "}\n\n"); @@ -970,8 +1043,8 @@ lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) switch (opt->io) { case FSM_IO_GETC: cp = "c"; break; - case FSM_IO_STR: cp = "c"; break; - case FSM_IO_PAIR: cp = "c"; break; + case FSM_IO_STR: cp = "*p"; break; + case FSM_IO_PAIR: cp = "*p"; break; } for (z = ast->zl; z != NULL; z = z->next) { @@ -1009,9 +1082,9 @@ lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\n"); print_io(f, opt); - print_lgetc(f); + print_lgetc(f, opt); - print_buf(f); + print_buf(f, opt); if (print_progress) { zn = 0; @@ -1044,6 +1117,7 @@ lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "void\n"); fprintf(f, "%sinit(struct %slx *lx)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); + fprintf(f, "\tstatic const struct %slx lx_default;\n", prefix.lx); fprintf(f, "\n"); fprintf(f, "\tassert(lx != NULL);\n"); @@ -1051,16 +1125,8 @@ lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\t*lx = lx_default;\n"); fprintf(f, "\n"); - switch (opt->io) { - case FSM_IO_GETC: + if (opt->io == FSM_IO_GETC) { fprintf(f, "\tlx->c = EOF;\n"); - break; - - case FSM_IO_STR: - break; - - case FSM_IO_PAIR: - break; } fprintf(f, "\tlx->z = z%u;\n", zindexof(ast, ast->global)); @@ -1091,9 +1157,6 @@ lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) break; case FSM_IO_STR: - fprintf(f, "\tif (lx->p == NULL) {\n"); - break; - case FSM_IO_PAIR: fprintf(f, "\tif (lx->p == NULL) {\n"); break; diff --git a/src/lx/print/dump.c b/src/lx/print/dump.c index 162630f88..232b2eb31 100644 --- a/src/lx/print/dump.c +++ b/src/lx/print/dump.c @@ -201,10 +201,12 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\tbuf.len = 0;\n"); fprintf(f, "\n"); - fprintf(f, "\tlx.buf_opaque = &buf;\n"); - fprintf(f, "\tlx.push = lx_dynpush;\n"); - fprintf(f, "\tlx.clear = lx_dynclear;\n"); - fprintf(f, "\tlx.free = lx_dynfree;\n"); + if (~api_exclude & API_BUF) { + fprintf(f, "\tlx.buf_opaque = &buf;\n"); + fprintf(f, "\tlx.push = lx_dynpush;\n"); + fprintf(f, "\tlx.clear = lx_dynclear;\n"); + fprintf(f, "\tlx.free = lx_dynfree;\n"); + } fprintf(f, "\n"); break; @@ -214,10 +216,12 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\tbuf.len = sizeof a;\n"); /* XXX: rename .len to .size */ fprintf(f, "\n"); - fprintf(f, "\tlx.buf_opaque = &buf;\n"); - fprintf(f, "\tlx.push = lx_fixedpush;\n"); - fprintf(f, "\tlx.clear = lx_fixedclear;\n"); - fprintf(f, "\tlx.free = NULL;\n"); + if (~api_exclude & API_BUF) { + fprintf(f, "\tlx.buf_opaque = &buf;\n"); + fprintf(f, "\tlx.push = lx_fixedpush;\n"); + fprintf(f, "\tlx.clear = lx_fixedclear;\n"); + fprintf(f, "\tlx.free = NULL;\n"); + } fprintf(f, "\n"); break; } From 26e2b139468000d151852bc88e1c28f5af249548 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 22 Jul 2025 10:08:07 -0400 Subject: [PATCH 291/373] test/lxpos: Update test data (EOF position info). Instead of having the EOF token occupy the same byte, line, and column position as the last token, it should immediately follow. The new lx codegen behaves this way, and katef and I decided that it made sense to keep it like that, as long as it's consistent. --- tests/lxpos/out0.dump | 2 +- tests/lxpos/out1.dump | 2 +- tests/lxpos/out2.dump | 2 +- tests/lxpos/out3.dump | 2 +- tests/lxpos/out4.dump | 2 +- tests/lxpos/out5.dump | 2 +- tests/lxpos/out6.dump | 2 +- tests/lxpos/out7.dump | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/lxpos/out0.dump b/tests/lxpos/out0.dump index 347bb87dc..8f1a4ac1d 100644 --- a/tests/lxpos/out0.dump +++ b/tests/lxpos/out0.dump @@ -2,4 +2,4 @@ 6-12:1,7-13: 12-13:1-2,13-1: -12-13:1-2,13-1: +13:2,1: diff --git a/tests/lxpos/out1.dump b/tests/lxpos/out1.dump index bf4d9bcb9..f3432e1e3 100644 --- a/tests/lxpos/out1.dump +++ b/tests/lxpos/out1.dump @@ -6,4 +6,4 @@ 19-25:2,7-13: 25-26:2-3,13-1: -25-26:2-3,13-1: +26:3,1: diff --git a/tests/lxpos/out2.dump b/tests/lxpos/out2.dump index cf5aa7013..bdd9e90a6 100644 --- a/tests/lxpos/out2.dump +++ b/tests/lxpos/out2.dump @@ -9,4 +9,4 @@ 12-13:1,13-14: 13-14:1-2,14-1: -13-14:1-2,14-1: +14:2,1: diff --git a/tests/lxpos/out3.dump b/tests/lxpos/out3.dump index 92d9e915a..e7f0f1e70 100644 --- a/tests/lxpos/out3.dump +++ b/tests/lxpos/out3.dump @@ -5,4 +5,4 @@ 7-8:1,8-9: 35-36:1-2,36-1: -35-36:1-2,36-1: +36:2,1: diff --git a/tests/lxpos/out4.dump b/tests/lxpos/out4.dump index 6161924b0..c3bb9bb2d 100644 --- a/tests/lxpos/out4.dump +++ b/tests/lxpos/out4.dump @@ -8,4 +8,4 @@ 27-28:1-2,28-1: 28-29:2,1-2: -28-29:2,1-2: +29:2,2: diff --git a/tests/lxpos/out5.dump b/tests/lxpos/out5.dump index e2506a1df..040b6734a 100644 --- a/tests/lxpos/out5.dump +++ b/tests/lxpos/out5.dump @@ -7,4 +7,4 @@ 4-5:3,1-2: 5-6:3-4,2-1: -5-6:3-4,2-1: +6:4,1: diff --git a/tests/lxpos/out6.dump b/tests/lxpos/out6.dump index 8036b2ad4..395d995be 100644 --- a/tests/lxpos/out6.dump +++ b/tests/lxpos/out6.dump @@ -1,2 +1,2 @@ 0-1:1,1-2: -4-5:1,5-6: +5:1,6: diff --git a/tests/lxpos/out7.dump b/tests/lxpos/out7.dump index 7eeab8bcb..0706b7981 100644 --- a/tests/lxpos/out7.dump +++ b/tests/lxpos/out7.dump @@ -2,4 +2,4 @@ 17-18:2,12-13: 18-19:2-3,13-1: -18-19:2-3,13-1: +19:3,1: From 862a68cb41726de9b3f83e691d77ba5677f8dd08 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 22 Jul 2025 16:19:43 -0400 Subject: [PATCH 292/373] Re-enable lxpos tests. Add `${LX}` as a dependency for the targets using it. Remove the `getcio=${io}` and `io=${io}` arguments to cat. Those may be a merge error? They just produce a warning. --- Makefile | 3 +-- tests/lxpos/Makefile | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6f6d9a2e3..b9e196d7b 100644 --- a/Makefile +++ b/Makefile @@ -124,8 +124,7 @@ SUBDIR += tests/fsm SUBDIR += tests/glob SUBDIR += tests/like SUBDIR += tests/literal -# FIXME: commenting this out for now due to Makefile error -#SUBDIR += tests/lxpos +SUBDIR += tests/lxpos SUBDIR += tests/minimise SUBDIR += tests/native SUBDIR += tests/pcre diff --git a/tests/lxpos/Makefile b/tests/lxpos/Makefile index da2263d24..9f6d464c4 100644 --- a/tests/lxpos/Makefile +++ b/tests/lxpos/Makefile @@ -30,13 +30,13 @@ CFLAGS.${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump += -I ${BUILD}/tests/lxpos .for ext in c h -${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.${ext}: tests/lxpos/lexer.lx +${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.${ext}: tests/lxpos/lexer.lx ${LX} ${LX} -l ${ext} ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ || { rm -f $@; false; } .endfor -${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump.c: tests/lxpos/lexer.lx +${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump.c: tests/lxpos/lexer.lx ${LX} ${LX} -l dump ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ || { rm -f $@; false; } @@ -53,11 +53,11 @@ CLEAN += ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump ${TEST_OUTDIR.tests/lxpos}/${buf}-${getc}-${io}-got${n}.dump: ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump ${TEST_SRCDIR.tests/lxpos}/in${n}.txt .if ${getc} != none - cat ${.ALLSRC:M*.txt} getcio=${io} \ + cat ${.ALLSRC:M*.txt} \ | ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump \ > $@ .else - cat ${.ALLSRC:M*.txt} io=${io} \ + cat ${.ALLSRC:M*.txt} \ | xargs -0 ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump \ > $@ .endif From affca78d8a5164ea3dea91c702d777c1a8951223 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 23 Jul 2025 11:13:37 -0400 Subject: [PATCH 293/373] Use $LX_BIN instead of $LX in lxpos makefile. Some of the CI test matrix builds set LX to 'true; echo lx', but that obviously won't work for tests that actually need to run lx in order to exercise its output. --- tests/lxpos/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/lxpos/Makefile b/tests/lxpos/Makefile index 9f6d464c4..abf11f58a 100644 --- a/tests/lxpos/Makefile +++ b/tests/lxpos/Makefile @@ -4,7 +4,7 @@ TEST.tests/lxpos != ls -1 tests/lxpos/out*.dump TEST_SRCDIR.tests/lxpos = tests/lxpos TEST_OUTDIR.tests/lxpos = ${BUILD}/tests/lxpos -LX?=${BUILD}/bin/lx +LX_BIN?=${BUILD}/bin/lx # for lx -l test LEXER += tests/lxpos/lexer.lx @@ -30,14 +30,14 @@ CFLAGS.${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump += -I ${BUILD}/tests/lxpos .for ext in c h -${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.${ext}: tests/lxpos/lexer.lx ${LX} - ${LX} -l ${ext} ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ +${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.${ext}: tests/lxpos/lexer.lx ${LX_BIN} + ${LX_BIN} -l ${ext} ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ || { rm -f $@; false; } .endfor -${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump.c: tests/lxpos/lexer.lx ${LX} - ${LX} -l dump ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ +${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump.c: tests/lxpos/lexer.lx ${LX_BIN} + ${LX_BIN} -l dump ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ || { rm -f $@; false; } ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump: ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.h From a8f0c590ab83cf285c66827a0e0e43e8f78a0fe7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 23 Jul 2025 12:38:11 -0400 Subject: [PATCH 294/373] lx: Make -l dump's output call lx.free() when using dynamic buffer. --- src/lx/print/dump.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lx/print/dump.c b/src/lx/print/dump.c index 232b2eb31..12cbf9e2e 100644 --- a/src/lx/print/dump.c +++ b/src/lx/print/dump.c @@ -307,6 +307,11 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\t} while (t != TOK_ERROR && t != TOK_EOF && t != TOK_UNKNOWN);\n"); fprintf(f, "\n"); + if (api_tokbuf == API_DYNBUF && (~api_exclude & API_BUF)) { + fprintf(f, "\tlx.free(lx.buf_opaque);\n"); + fprintf(f, "\n"); + } + fprintf(f, "\treturn t == TOK_ERROR;\n"); fprintf(f, "}\n"); } From 552aa01161f3b0f3df980fd002e50a003d68f93f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Jul 2025 10:31:12 -0400 Subject: [PATCH 295/373] lx: Use prefix.tok, not "TOK_". --- src/lx/print/c.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 18f6d664a..fff599a34 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -262,7 +262,7 @@ reject_c(FILE *f, const struct fsm_options *opt, fprintf(f, ";"); return 0; } else { - fprintf(f, "\n\t\t\t\tif (!has_consumed_input) { return TOK_EOF; }\n"); + fprintf(f, "\n\t\t\t\tif (!has_consumed_input) { return %sEOF; }\n", prefix.tok); fprintf(f, "\t\t\t\t"); unget_character(f, false, env->cur_char_var); } @@ -529,7 +529,7 @@ print_io(FILE *f, const struct fsm_options *opt) /* When libfsm's generated code advances a character, update * lx's token name buffer and position bookkeeping. */ fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); - fprintf(f, "#define FSM_ADVANCE_HOOK(C) if (!lx_advance_end(lx, C)) { return TOK_ERROR; }\n"); + fprintf(f, "#define FSM_ADVANCE_HOOK(C) if (!lx_advance_end(lx, C)) { return %sERROR; }\n", prefix.tok); fprintf(f, "#endif\n"); fprintf(f, "\n"); break; @@ -910,7 +910,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, } fprintf(f, "\tif (!has_consumed_input) {\n"); - fprintf(f, "\t\treturn TOK_EOF;\n"); + fprintf(f, "\t\treturn %sEOF;\n", prefix.tok); fprintf(f, "\t} \n"); fprintf(f, "\n"); } From beebd1b57d673fa6599b0dcd9b1f845ed890bfcf Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Jul 2025 15:43:09 -0400 Subject: [PATCH 296/373] lx: return TOK_ERROR if reaching the end of a zone function. --- src/lx/print/c.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index fff599a34..12601de58 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -912,6 +912,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, fprintf(f, "\tif (!has_consumed_input) {\n"); fprintf(f, "\t\treturn %sEOF;\n", prefix.tok); fprintf(f, "\t} \n"); + fprintf(f, "\treturn %sERROR;", prefix.tok); fprintf(f, "\n"); } From ae53e94102890438854f824a0ee093c78265459c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 29 Jul 2025 15:51:35 -0400 Subject: [PATCH 297/373] lx: Handle unexpected EOF in pattern pairs. Add tests, update out6.dump. Add a couple test cases (in8-10.txt) with an unexpected end of input, either in the middle of a pattern, or after matching the first pattern in a .. pair, but without matching the second. Supporting this changes the expected result for in6.txt: Previously it resulted in TOK_EOF, now it leads to TOK_UNKNOWN and produces a "lexically uncategorised" error message for the unexpected end of input. This change is necessary for fixing #386 / #508, and more generally to detect things like unterminated string literals. --- src/lx/print/c.c | 6 ++++++ tests/lxpos/in10.txt | 1 + tests/lxpos/in8.txt | 1 + tests/lxpos/in9.txt | 1 + tests/lxpos/out10.dump | 7 +++++++ tests/lxpos/out6.dump | 2 +- tests/lxpos/out8.dump | 3 +++ tests/lxpos/out9.dump | 7 +++++++ 8 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/lxpos/in10.txt create mode 100644 tests/lxpos/in8.txt create mode 100644 tests/lxpos/in9.txt create mode 100644 tests/lxpos/out10.dump create mode 100644 tests/lxpos/out8.dump create mode 100644 tests/lxpos/out9.dump diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 12601de58..21fbba180 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -192,6 +192,12 @@ accept_c(FILE *f, const struct fsm_options *opt, fprintf(f, "return "); if (m->to != NULL) { fprintf(f, "lx->z = z%u, ", zindexof(ast, m->to)); + } else if (m->to == NULL && m->token == NULL) { + /* If accept-ing here doesn't actually map to a token or + * a different zone, then it's stuck in the middle of a + * pattern pair like `'//' .. /\n/ -> $nl;` with an EOF, + * so tokenization should still fail. */ + fprintf(f, "%sUNKNOWN; ", prefix.tok); } if (m->token != NULL) { fprintf(f, "%s", prefix.tok); diff --git a/tests/lxpos/in10.txt b/tests/lxpos/in10.txt new file mode 100644 index 000000000..52cab2a4f --- /dev/null +++ b/tests/lxpos/in10.txt @@ -0,0 +1 @@ +d: r * t /*distance = rate * time* diff --git a/tests/lxpos/in8.txt b/tests/lxpos/in8.txt new file mode 100644 index 000000000..e5c01f9dd --- /dev/null +++ b/tests/lxpos/in8.txt @@ -0,0 +1 @@ +hello ` diff --git a/tests/lxpos/in9.txt b/tests/lxpos/in9.txt new file mode 100644 index 000000000..81d7792df --- /dev/null +++ b/tests/lxpos/in9.txt @@ -0,0 +1 @@ +d: r * t /*distance = rate * time diff --git a/tests/lxpos/out10.dump b/tests/lxpos/out10.dump new file mode 100644 index 000000000..4314c8beb --- /dev/null +++ b/tests/lxpos/out10.dump @@ -0,0 +1,7 @@ +0-1:1,1-2: +1-2:1,2-3: +3-4:1,4-5: +5-6:1,6-7: +7-8:1,8-9: +34-35:1-2,35-1: lexically uncategorised: ' +' diff --git a/tests/lxpos/out6.dump b/tests/lxpos/out6.dump index 395d995be..c11d09988 100644 --- a/tests/lxpos/out6.dump +++ b/tests/lxpos/out6.dump @@ -1,2 +1,2 @@ 0-1:1,1-2: -5:1,6: +4-5:1,5-6: lexically uncategorised: ' ' diff --git a/tests/lxpos/out8.dump b/tests/lxpos/out8.dump new file mode 100644 index 000000000..3300df2f6 --- /dev/null +++ b/tests/lxpos/out8.dump @@ -0,0 +1,3 @@ +0-5:1,1-6: +6-7:1,7-8: lexically uncategorised: '` +' diff --git a/tests/lxpos/out9.dump b/tests/lxpos/out9.dump new file mode 100644 index 000000000..1f3cf25bc --- /dev/null +++ b/tests/lxpos/out9.dump @@ -0,0 +1,7 @@ +0-1:1,1-2: +1-2:1,2-3: +3-4:1,4-5: +5-6:1,6-7: +7-8:1,8-9: +33-34:1-2,34-1: lexically uncategorised: ' +' From ea9c90b4a2d8ad32e62ca4349231f381b46b19f1 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 30 Jul 2025 10:56:48 -0400 Subject: [PATCH 298/373] lx: Rewrite logic to make the four cases explicit, fix dead code. This was previously ending up with a useless call to the current zone after returning the token ("case S1: return TOK_UNKNOWN; lx->z(lx);"), which led to a warning in CI [-Werror=implicit-fallthrough=]. --- src/lx/print/c.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 21fbba180..a03f4ed8b 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -190,20 +190,28 @@ accept_c(FILE *f, const struct fsm_options *opt, } fprintf(f, "return "); - if (m->to != NULL) { - fprintf(f, "lx->z = z%u, ", zindexof(ast, m->to)); - } else if (m->to == NULL && m->token == NULL) { - /* If accept-ing here doesn't actually map to a token or - * a different zone, then it's stuck in the middle of a - * pattern pair like `'//' .. /\n/ -> $nl;` with an EOF, - * so tokenization should still fail. */ - fprintf(f, "%sUNKNOWN; ", prefix.tok); - } - if (m->token != NULL) { - fprintf(f, "%s", prefix.tok); - esctok(f, m->token->s); + if (m->to == NULL) { + if (m->token == NULL) { + /* If accept-ing here doesn't actually map to a token or + * a different zone, then it's stuck in the middle of a + * pattern pair like `'//' .. /\n/ -> $nl;` with an EOF, + * so tokenization should still fail. */ + fprintf(f, "%sUNKNOWN", prefix.tok); + } else { + /* yield a token */ + fprintf(f, "%s", prefix.tok); + esctok(f, m->token->s); + } } else { - fprintf(f, "lx->z(lx)"); + if (m->token == NULL) { + /* update to a different zone, then call to it */ + fprintf(f, "lx->z = z%u, lx->z(lx)", zindexof(ast, m->to)); + } else { + /* update zone, then yield a token */ + fprintf(f, "lx->z = z%u, ", zindexof(ast, m->to)); + fprintf(f, "%s", prefix.tok); + esctok(f, m->token->s); + } } fprintf(f, ";"); return 0; From 17d415de73f3ed6e397ef187413e04454e6367e0 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 31 Jul 2025 13:25:49 -0400 Subject: [PATCH 299/373] lx: Only gen fixedpop / dynpop & calls to them when buffer mode is set. --- src/lx/print/c.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index a03f4ed8b..bb48fb084 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -637,7 +637,7 @@ print_buf(FILE *f, const struct fsm_options *opt) fprintf(f, "\n"); - if (~api_exclude & API_BUF) { + if ((~api_exclude & API_BUF) && (api_tokbuf & API_DYNBUF)) { fprintf(f, "static void\n"); fprintf(f, "%sdynpop(void *buf_opaque)\n", prefix.api); fprintf(f, "{\n"); @@ -722,7 +722,7 @@ print_buf(FILE *f, const struct fsm_options *opt) fprintf(f, "}\n"); fprintf(f, "\n"); - if (~api_exclude & API_BUF) { + if (~api_exclude & API_BUF && (api_tokbuf & API_FIXEDBUF)) { fprintf(f, "static void\n"); fprintf(f, "%sfixedpop(void *buf_opaque)\n", prefix.api); fprintf(f, "{\n"); From 7ed18b94e2af4fa64440e2fde8adbdfc92af8a55 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 31 Jul 2025 13:29:28 -0400 Subject: [PATCH 300/373] lx: Suppress warning for possibly unused function. These may or may not be called, depending on the input. --- src/lx/print/c.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index bb48fb084..bfc25023e 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -1151,6 +1151,16 @@ lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\tlx->end.line = 1;\n"); fprintf(f, "\tlx->end.col = 1;\n"); } + + /* Suppress warning for possibly unused function */ + if (~api_exclude & API_BUF) { + if (api_tokbuf & API_FIXEDBUF) { + fprintf(f, "\t(void)%sfixedpop;\n", prefix.api); + } else if (api_tokbuf & API_DYNBUF) { + fprintf(f, "\t(void)%sdynpop;\n", prefix.api); + } + } + fprintf(f, "}\n"); fprintf(f, "\n"); } From 1e55db8ae38001dc68b784c6abc07b43941b6fd7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 31 Jul 2025 13:30:59 -0400 Subject: [PATCH 301/373] lx: Ensure prefix.api & prefix.lx are used in the generated code. In some cases this was hardcoding "lx_" in the generated code, which could lead to build failures if 'lx -e' was used to override the default prefix. --- src/lx/print/c.c | 12 ++++++------ src/lx/print/dump.c | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index bfc25023e..5ca179b5a 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -452,7 +452,7 @@ print_io(FILE *f, const struct fsm_options *opt) } fprintf(f, "static int\n"); - fprintf(f, "lx_advance_end(struct lx *lx, int c)\n"); + fprintf(f, "%sadvance_end(struct %slx *lx, int c)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); if (~api_exclude & API_POS) { fprintf(f, "\tlx->end.byte++;\n"); @@ -496,7 +496,7 @@ print_io(FILE *f, const struct fsm_options *opt) fprintf(f, "inline\n"); fprintf(f, "#endif\n"); fprintf(f, "static int\n"); - fprintf(f, "lx_getc(struct %slx *lx)\n", prefix.lx); + fprintf(f, "%sgetc(struct %slx *lx)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); fprintf(f, "\tint c;\n"); fprintf(f, "\n"); @@ -518,7 +518,7 @@ print_io(FILE *f, const struct fsm_options *opt) /* FIXME: This should distinguish between alloc failure * and EOF, but will require layers of interface changes. */ - fprintf(f, "\tif (!lx_advance_end(lx, c)) { return EOF; }\n"); + fprintf(f, "\tif (!%sadvance_end(lx, c)) { return EOF; }\n", prefix.api); fprintf(f, "\n"); fprintf(f, "\treturn c;\n"); @@ -527,13 +527,13 @@ print_io(FILE *f, const struct fsm_options *opt) /* Add an implementation of fsm_getc that calls back * into lx_getc with the lx handle. */ - fprintf(f, "/* This wrapper adapts calling lx_getc to the interface\n"); + fprintf(f, "/* This wrapper adapts calling %sgetc to the interface\n", prefix.api); fprintf(f, " * in libfsm's generated code. */\n"); fprintf(f, "static int\n"); fprintf(f, "fsm_getc(void *getc_opaque)\n"); fprintf(f, "{\n"); - fprintf(f, "\treturn lx_getc((struct lx *)getc_opaque);\n"); + fprintf(f, "\treturn %sgetc((struct %slx *)getc_opaque);\n", prefix.api, prefix.lx); fprintf(f, "}\n"); fprintf(f, "\n"); break; @@ -543,7 +543,7 @@ print_io(FILE *f, const struct fsm_options *opt) /* When libfsm's generated code advances a character, update * lx's token name buffer and position bookkeeping. */ fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); - fprintf(f, "#define FSM_ADVANCE_HOOK(C) if (!lx_advance_end(lx, C)) { return %sERROR; }\n", prefix.tok); + fprintf(f, "#define FSM_ADVANCE_HOOK(C) if (!%sadvance_end(lx, C)) { return %sERROR; }\n", prefix.api, prefix.tok); fprintf(f, "#endif\n"); fprintf(f, "\n"); break; diff --git a/src/lx/print/dump.c b/src/lx/print/dump.c index 12cbf9e2e..e693f9082 100644 --- a/src/lx/print/dump.c +++ b/src/lx/print/dump.c @@ -68,12 +68,12 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "main(int argc, char *argv[])\n"); fprintf(f, "{\n"); - fprintf(f, "\tenum lx_token t;\n"); - fprintf(f, "\tstruct lx lx = { 0 };\n"); + fprintf(f, "\tenum %stoken t;\n", prefix.api); + fprintf(f, "\tstruct %slx lx = { 0 };\n", prefix.lx); switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "\tint (*lgetc)(struct lx *lx);\n"); + fprintf(f, "\tint (*lgetc)(struct %slx *lx);\n", prefix.lx); fprintf(f, "\tvoid *getc_opaque;\n"); break; @@ -135,7 +135,7 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) switch (api_getc) { case API_FGETC: - fprintf(f, "\tlgetc = lx_fgetc;\n"); + fprintf(f, "\tlgetc = %sfgetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = stdin;\n"); fprintf(f, "\n"); break; @@ -144,7 +144,7 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\ts = argv[1];\n"); fprintf(f, "\n"); - fprintf(f, "\tlgetc = lx_sgetc;\n"); + fprintf(f, "\tlgetc = %ssgetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = &s;\n"); fprintf(f, "\n"); break; @@ -154,7 +154,7 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\tarr.len = strlen(arr.p);\n"); fprintf(f, "\n"); - fprintf(f, "\tlgetc = lx_agetc;\n"); + fprintf(f, "\tlgetc = %sagetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = &arr;\n"); fprintf(f, "\n"); break; @@ -167,13 +167,13 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\td.fd = fileno(stdin);\n"); fprintf(f, "\n"); - fprintf(f, "\tlgetc = lx_dgetc;\n"); + fprintf(f, "\tlgetc = %sdgetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = &d;\n"); fprintf(f, "\n"); break; } - fprintf(f, "\tlx_init(&lx);\n"); + fprintf(f, "\t%sinit(&lx);\n", prefix.api); fprintf(f, "\n"); switch (opt->io) { @@ -203,9 +203,9 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) if (~api_exclude & API_BUF) { fprintf(f, "\tlx.buf_opaque = &buf;\n"); - fprintf(f, "\tlx.push = lx_dynpush;\n"); - fprintf(f, "\tlx.clear = lx_dynclear;\n"); - fprintf(f, "\tlx.free = lx_dynfree;\n"); + fprintf(f, "\tlx.push = %sdynpush;\n", prefix.api); + fprintf(f, "\tlx.clear = %sdynclear;\n", prefix.api); + fprintf(f, "\tlx.free = %sdynfree;\n", prefix.api); } fprintf(f, "\n"); break; @@ -218,8 +218,8 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) if (~api_exclude & API_BUF) { fprintf(f, "\tlx.buf_opaque = &buf;\n"); - fprintf(f, "\tlx.push = lx_fixedpush;\n"); - fprintf(f, "\tlx.clear = lx_fixedclear;\n"); + fprintf(f, "\tlx.push = %sfixedpush;\n", prefix.api); + fprintf(f, "\tlx.clear = %sfixedclear;\n", prefix.api); fprintf(f, "\tlx.free = NULL;\n"); } fprintf(f, "\n"); @@ -231,7 +231,7 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\t\tconst char *q;\n"); fprintf(f, "\n"); - fprintf(f, "\t\tt = lx_next(&lx);\n"); + fprintf(f, "\t\tt = %snext(&lx);\n", prefix.api); fprintf(f, "\n"); switch (api_tokbuf) { @@ -278,7 +278,7 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\n"); fprintf(f, "\t\tcase TOK_ERROR:\n"); - fprintf(f, "\t\t\tperror(\"lx_next\");\n"); + fprintf(f, "\t\t\tperror(\"%snext\");\n", prefix.api); fprintf(f, "\t\t\tbreak;\n"); fprintf(f, "\n"); @@ -291,7 +291,7 @@ lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) fprintf(f, "\t\tdefault:\n"); if (~api_exclude & API_NAME) { - fprintf(f, "\t\t\tprintf(\"<%%s\", lx_name(t));\n"); + fprintf(f, "\t\t\tprintf(\"<%%s\", %sname(t));\n", prefix.api); fprintf(f, "\t\t\tdump_buf(q, l);\n"); fprintf(f, "\t\t\tprintf(\">\\n\");\n"); } else { From 4a5ca8458daa410fef43e2303e141f09f4d7b6f7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 4 Aug 2025 10:52:56 -0400 Subject: [PATCH 302/373] Replace FSM_ADVANCE_HOOK macro with optional hooks->advance callback. --- include/fsm/print.h | 6 ++++++ src/libfsm/print/c.c | 36 ++++++++++-------------------------- src/lx/print/c.c | 35 ++++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/include/fsm/print.h b/include/fsm/print.h index 6fae2abea..c9ec7ec0a 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -78,6 +78,12 @@ struct fsm_hooks { const struct fsm_state_metadata *state_metadata, void *lang_opaque, void *hook_opaque); + /* If non-NULL, this will be called to generate code + * in scope immediately after advancing to the + * next character of input. */ + int (*advance)(FILE *, const struct fsm_options *opt, + const char *cur_char_var, void *hook_opaque); + int (*comment)(FILE *, const struct fsm_options *opt, const struct fsm_state_metadata *state_metadata, void *hook_opaque); diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 96117f21e..d8e12eee6 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -495,30 +495,6 @@ fsm_print_c_body(FILE *f, const struct ir *ir, * input loop was skipped it would still be NONE. */ fprintf(f, "\tint has_consumed_input = 0;\n"); - /* For FSM_IO_STR and FSM_IO_PAIR, define a macro that will be - * called with the new character every time iteration advances. - * This is used by lx's internal bookkeeping to track token - * positions in the input stream. For FSM_IO_GETC, the generated - * getc function handles this directly. - * - * This defaults to a no-op unless defined. */ - switch (opt->io) { - case FSM_IO_GETC: - break; /* nothing to do */ - - case FSM_IO_STR: - fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); - fprintf(f, "#define FSM_ADVANCE_HOOK(C) /* no-op */ (void)C\n"); - fprintf(f, "#endif\n"); - break; - - case FSM_IO_PAIR: - fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); - fprintf(f, "#define FSM_ADVANCE_HOOK(C) /* no-op */ (void)C\n"); - fprintf(f, "#endif\n"); - break; - } - /* enum of states */ print_stateenum(f, ir->n); fprintf(f, "\n"); @@ -536,13 +512,21 @@ fsm_print_c_body(FILE *f, const struct ir *ir, case FSM_IO_STR: fprintf(f, "\tfor (p = s; *p != '\\0'; p++) {\n"); fprintf(f, "\t\thas_consumed_input = 1;\n"); - fprintf(f, "\t\tFSM_ADVANCE_HOOK(%s);\n", cp); + if (hooks->advance != NULL) { + if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) { + return -1; + } + } break; case FSM_IO_PAIR: fprintf(f, "\tfor (p = b; p != e; p++) {\n"); fprintf(f, "\t\thas_consumed_input = 1;\n"); - fprintf(f, "\t\tFSM_ADVANCE_HOOK(%s);\n", cp); + if (hooks->advance != NULL) { + if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) { + return -1; + } + } break; } diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 5ca179b5a..97eb8281f 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -297,6 +297,26 @@ reject_c(FILE *f, const struct fsm_options *opt, return 0; } +static int +advance_c(FILE *f, const struct fsm_options *opt, const char *cur_char_var, void *hook_opaque) +{ + (void)hook_opaque; + + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + case FSM_IO_PAIR: + /* When libfsm's generated code advances a character, update + * lx's token name buffer and position bookkeeping. */ + fprintf(f, "\t\tif (!%sadvance_end(lx, %s)) { return %sERROR; }\n", + prefix.api, cur_char_var, prefix.tok); + break; + } + return 0; +} + static void print_proto(FILE *f, const struct ast *ast, const struct ast_zone *z) { @@ -485,8 +505,7 @@ print_io(FILE *f, const struct fsm_options *opt) fprintf(f, "}\n"); fprintf(f, "\n"); - switch (opt->io) { - case FSM_IO_GETC: + if (opt->io == FSM_IO_GETC) { /* TODO: consider passing char *c, and return int 0/-1 for error */ if (opt->comments) { fprintf(f, "/* This wrapper manages one character of lookahead/pushback\n"); @@ -536,17 +555,6 @@ print_io(FILE *f, const struct fsm_options *opt) fprintf(f, "\treturn %sgetc((struct %slx *)getc_opaque);\n", prefix.api, prefix.lx); fprintf(f, "}\n"); fprintf(f, "\n"); - break; - - case FSM_IO_PAIR: - case FSM_IO_STR: - /* When libfsm's generated code advances a character, update - * lx's token name buffer and position bookkeeping. */ - fprintf(f, "#ifndef FSM_ADVANCE_HOOK\n"); - fprintf(f, "#define FSM_ADVANCE_HOOK(C) if (!%sadvance_end(lx, C)) { return %sERROR; }\n", prefix.api, prefix.tok); - fprintf(f, "#endif\n"); - fprintf(f, "\n"); - break; } fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); @@ -831,6 +839,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, hooks.accept = accept_c; hooks.reject = reject_c; + hooks.advance = advance_c; hooks.hook_opaque = &hook_env; fsm_print(f, z->fsm, opt, &hooks, FSM_PRINT_C); From f25e8b7d874d2bea24eaea74bc112a0d1aacde9c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 4 Aug 2025 10:56:05 -0400 Subject: [PATCH 303/373] The advance hook should also be called for FSM_IO_STR. --- src/libfsm/print/c.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index d8e12eee6..86f9855b1 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -512,24 +512,20 @@ fsm_print_c_body(FILE *f, const struct ir *ir, case FSM_IO_STR: fprintf(f, "\tfor (p = s; *p != '\\0'; p++) {\n"); fprintf(f, "\t\thas_consumed_input = 1;\n"); - if (hooks->advance != NULL) { - if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) { - return -1; - } - } break; case FSM_IO_PAIR: fprintf(f, "\tfor (p = b; p != e; p++) {\n"); fprintf(f, "\t\thas_consumed_input = 1;\n"); - if (hooks->advance != NULL) { - if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) { - return -1; - } - } break; } + if (hooks->advance != NULL) { + if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) { + return -1; + } + } + if (-1 == fsm_print_cfrag(f, ir, opt, hooks, cp)) { return -1; } From 051aaf0337b8951383ebbc00defbe85488f56f13 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 4 Aug 2025 10:59:33 -0400 Subject: [PATCH 304/373] Move setting `has_consumed_input` flag into lx's advance hook. This avoids cluttering libfsm's print output with `has_consumed_input`, which is specific to lx. --- src/libfsm/print/c.c | 18 ------------------ src/lx/print/c.c | 13 +++++++++++++ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 86f9855b1..7f9274792 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -484,17 +484,6 @@ fsm_print_c_body(FILE *f, const struct ir *ir, } } - /* This flag indicates whether the any of the input stream was - * consumed before getting EOF and skipping over the state and - * character logic expanded here. - * - * lx needs to track this for proper EOF handling. It previously - * generated the state enum itself, so that it could include an - * additional 'NONE' state. Inside the input loop, the default - * state of NONE would be updated to the start state, but if the - * input loop was skipped it would still be NONE. */ - fprintf(f, "\tint has_consumed_input = 0;\n"); - /* enum of states */ print_stateenum(f, ir->n); fprintf(f, "\n"); @@ -506,17 +495,14 @@ fsm_print_c_body(FILE *f, const struct ir *ir, switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\twhile (c = fsm_getc(getc_opaque), c != EOF) {\n"); - fprintf(f, "\t\thas_consumed_input = 1;\n"); break; case FSM_IO_STR: fprintf(f, "\tfor (p = s; *p != '\\0'; p++) {\n"); - fprintf(f, "\t\thas_consumed_input = 1;\n"); break; case FSM_IO_PAIR: fprintf(f, "\tfor (p = b; p != e; p++) {\n"); - fprintf(f, "\t\thas_consumed_input = 1;\n"); break; } @@ -533,10 +519,6 @@ fsm_print_c_body(FILE *f, const struct ir *ir, fprintf(f, "\t}\n"); fprintf(f, "\n"); - /* Suppress unused variable warning -- this is mainly for lx. */ - fprintf(f, "\t(void)has_consumed_input;\n"); - fprintf(f, "\n"); - /* end states */ if (-1 == print_endstates(f, opt, hooks, ir)) { return -1; diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 97eb8281f..9191202df 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -302,6 +302,8 @@ advance_c(FILE *f, const struct fsm_options *opt, const char *cur_char_var, void { (void)hook_opaque; + fprintf(f, "\t\thas_consumed_input = 1;\n"); + switch (opt->io) { case FSM_IO_GETC: break; @@ -783,6 +785,17 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, fprintf(f, "z%u(struct %slx *lx)\n", zindexof(ast, z), prefix.lx); fprintf(f, "{\n"); + /* This flag indicates whether the any of the input stream was + * consumed before getting EOF and skipping over the state and + * character logic expanded here. + * + * lx needs to track this for proper EOF handling. It previously + * generated the state enum itself, so that it could include an + * additional 'NONE' state. Inside the input loop, the default + * state of NONE would be updated to the start state, but if the + * input loop was skipped it would still be NONE. */ + fprintf(f, "\tint has_consumed_input = 0;\n"); + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tint c;\n"); From 08fd72ccd9fff200c7d5f3921f74f49b7ab0c963 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 18 Aug 2025 17:01:13 -0400 Subject: [PATCH 305/373] lx: Avoid useless call to pop and some other 'unused' warnings. Tested with every combination of (dyn+fgetc, fixed+fgetc, pair, str), with and without '-x buf', '-x pos', or both. --- src/lx/print/c.c | 73 ++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 9191202df..77207403a 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -153,7 +153,7 @@ static void unget_character(FILE *f, bool pop, const char *cur_char_var) { fprintf(f, "%sungetc(lx, %s); ", prefix.api, cur_char_var); - if (pop && (~api_exclude & API_POS)) { + if (pop && (~api_exclude & API_BUF)) { fprintf(f, "%s%spop(lx->buf_opaque); ", prefix.api, buf_op_prefix()); } @@ -312,8 +312,10 @@ advance_c(FILE *f, const struct fsm_options *opt, const char *cur_char_var, void case FSM_IO_PAIR: /* When libfsm's generated code advances a character, update * lx's token name buffer and position bookkeeping. */ - fprintf(f, "\t\tif (!%sadvance_end(lx, %s)) { return %sERROR; }\n", - prefix.api, cur_char_var, prefix.tok); + if (~api_exclude & API_POS) { + fprintf(f, "\t\tif (!%sadvance_end(lx, %s)) { return %sERROR; }\n", + prefix.api, cur_char_var, prefix.tok); + } break; } return 0; @@ -473,40 +475,47 @@ print_io(FILE *f, const struct fsm_options *opt) fprintf(stderr, " io"); } - fprintf(f, "static int\n"); - fprintf(f, "%sadvance_end(struct %slx *lx, int c)\n", prefix.api, prefix.lx); - fprintf(f, "{\n"); - if (~api_exclude & API_POS) { - fprintf(f, "\tlx->end.byte++;\n"); - fprintf(f, "\tlx->end.col++;\n"); + if (opt->io == FSM_IO_GETC || (~api_exclude & API_POS)) { + fprintf(f, "static int\n"); + fprintf(f, "%sadvance_end(struct %slx *lx, int c)\n", prefix.api, prefix.lx); + fprintf(f, "{\n"); - fprintf(f, "\tif (c == '\\n') {\n"); - fprintf(f, "\t\tlx->end.line++;\n"); - fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); - fprintf(f, "\t\tlx->end.col = 1;\n"); - - if (opt->io == FSM_IO_STR) { /* ignore terminating '\0' */ - fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); - fprintf(f, "\t\tlx->end.byte--;\n"); - fprintf(f, "\t\tlx->end.col--;\n"); - fprintf(f, "\t}\n"); + if (api_exclude & API_POS) { + fprintf(f, "\t(void)lx; (void)c;\n"); + } else { + fprintf(f, "\tlx->end.byte++;\n"); + fprintf(f, "\tlx->end.col++;\n"); + + fprintf(f, "\tif (c == '\\n') {\n"); + fprintf(f, "\t\tlx->end.line++;\n"); + fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); + fprintf(f, "\t\tlx->end.col = 1;\n"); + + if (opt->io == FSM_IO_STR) { /* ignore terminating '\0' */ + fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); + fprintf(f, "\t\tlx->end.byte--;\n"); + fprintf(f, "\t\tlx->end.col--;\n"); + fprintf(f, "\t}\n"); + } else { + fprintf(f, "\t}\n"); + } + } + + if (api_exclude & API_BUF) { + fprintf(f, "\t(void)lx; (void)c;\n"); } else { + fprintf(f, "\tif (lx->push != NULL) {\n"); + fprintf(f, "\t\tif (-1 == lx->push(lx->buf_opaque, (char)c)) {\n"); + fprintf(f, "\t\t\treturn 0;\n"); + fprintf(f, "\t\t}\n"); fprintf(f, "\t}\n"); } - } - if (~api_exclude & API_BUF) { - fprintf(f, "\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\tif (-1 == lx->push(lx->buf_opaque, (char)c)) {\n"); - fprintf(f, "\t\t\treturn 0;\n"); - fprintf(f, "\t\t}\n"); - fprintf(f, "\t}\n"); + fprintf(f, "\treturn 1;\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); } - fprintf(f, "\treturn 1;\n"); - fprintf(f, "}\n"); - fprintf(f, "\n"); - if (opt->io == FSM_IO_GETC) { /* TODO: consider passing char *c, and return int 0/-1 for error */ if (opt->comments) { @@ -582,7 +591,9 @@ print_io(FILE *f, const struct fsm_options *opt) break; } - if (~api_exclude & API_POS) { + if (api_exclude & API_POS) { + fprintf(f, "\t(void)lx; (void)c;\n"); + } else { fprintf(f, "\tlx->end.byte--;\n"); fprintf(f, "\tlx->end.col--;\n"); fprintf(f, "\n"); From c853157a6a640e723eaacd3c5f9e7caa07bdfcc6 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 26 Aug 2025 14:54:42 -0400 Subject: [PATCH 306/373] lx: Distinguish between unexpected EOF and EOF in ignored zones. PR #509 introduced a bug: It didn't distinguish between an unexpected end of input and an end of input in a zone that matches but ignores its input. This caused several lxpos tests to fail due to getting a TOK_UNKNOWN rather than a TOK_EOF when the input has trailing whitespace, but I didn't notice until after merging because the normal build doesn't regenerate the code for src/lx/lexer.lx or src/libfsm/lexer.lx. (I had ensured all the libre dialect lexers and parsers were regenerated, but missed those.) Instead of always printing TOK_UNKNOWN, this this inspects the zone mappings to determine whether the current end ID represents a dead end for the zone. If not, it should instead print TOK_EOF. --- src/lx/print/c.c | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 77207403a..3340f0d74 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -159,6 +159,28 @@ unget_character(FILE *f, bool pop, const char *cur_char_var) } } +static bool +endid_represents_dead_end(fsm_end_id_t endid, const struct ast *ast) +{ + const struct ast_mapping *m = ast_getendmappingbyendid(endid); + if (m == NULL) { + return false; + } + + /* For each zone, check if this endid is associated with its z->ml zone. + * If so, that endid is the "dead end" for that zone. + * + * The total number of zones and end ids (each corresponding to mapping) + * should stay small enough that linear search is fine. If this becomes + * prohibitively expensive, then build a bitset of dead-end IDs upfront + * in one pass. */ + for (struct ast_zone *z = ast->zl; z != NULL; z = z->next) { + if (z->ml == m) { return true; } + } + + return false; +} + static int accept_c(FILE *f, const struct fsm_options *opt, const struct fsm_state_metadata *state_metadata, @@ -193,10 +215,21 @@ accept_c(FILE *f, const struct fsm_options *opt, if (m->to == NULL) { if (m->token == NULL) { /* If accept-ing here doesn't actually map to a token or - * a different zone, then it's stuck in the middle of a - * pattern pair like `'//' .. /\n/ -> $nl;` with an EOF, - * so tokenization should still fail. */ - fprintf(f, "%sUNKNOWN", prefix.tok); + * a different zone, then check whether the endid represents + * a dead end. In that case, it's stuck in the middle of a + * pattern pair like `'//' .. /\n/ -> $nl;` with an unexpected + * EOF, so tokenization should still fail (with TOK_UNKNOWN). + * + * An example where the endid doesn't represent a dead end is + * a zone ignoring trailing whitespace in a file, such as + * `/[\r\n\t ]+/;`. In that case, the EOF is valid, so still + * return TOK_EOF. */ + const fsm_end_id_t endid = state_metadata->end_ids[0]; + if (endid_represents_dead_end(endid, ast)) { + fprintf(f, "%sUNKNOWN", prefix.tok); + } else { + fprintf(f, "%sEOF", prefix.tok); + } } else { /* yield a token */ fprintf(f, "%s", prefix.tok); From 9be4aecb1625defde993db68524dd863873a0ffb Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 26 Aug 2025 15:02:42 -0400 Subject: [PATCH 307/373] Generated code. Re-generate lexers and parsers with lx bug fixed. --- src/libfsm/lexer.c | 475 +++++--- src/libfsm/parser.c | 74 +- src/libfsm/parser.h | 4 +- src/libre/dialect/glob/lexer.c | 106 +- src/libre/dialect/glob/parser.c | 38 +- src/libre/dialect/glob/parser.h | 4 +- src/libre/dialect/like/lexer.c | 104 +- src/libre/dialect/like/parser.c | 38 +- src/libre/dialect/like/parser.h | 4 +- src/libre/dialect/literal/lexer.c | 96 +- src/libre/dialect/literal/parser.c | 26 +- src/libre/dialect/literal/parser.h | 4 +- src/libre/dialect/native/lexer.c | 490 +++++--- src/libre/dialect/native/parser.c | 204 ++-- src/libre/dialect/native/parser.h | 4 +- src/libre/dialect/pcre/lexer.c | 1810 +++++++++++++++++----------- src/libre/dialect/pcre/parser.c | 318 ++--- src/libre/dialect/pcre/parser.h | 4 +- src/libre/dialect/sql/lexer.c | 366 ++++-- src/libre/dialect/sql/parser.c | 154 +-- src/libre/dialect/sql/parser.h | 4 +- src/lx/lexer.c | 679 ++++++----- src/lx/parser.c | 182 +-- src/lx/parser.h | 4 +- 24 files changed, 3029 insertions(+), 2163 deletions(-) diff --git a/src/libfsm/lexer.c b/src/libfsm/lexer.c index 8bd374cec..3bf26b3b6 100644 --- a/src/libfsm/lexer.c +++ b/src/libfsm/lexer.c @@ -15,6 +15,26 @@ static enum lx_token z3(struct lx *lx); static enum lx_token z4(struct lx *lx); static enum lx_token z5(struct lx *lx); +static int +lx_advance_end(struct lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif @@ -35,18 +55,19 @@ lx_getc(struct lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_getc((struct lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -55,10 +76,7 @@ lx_ungetc(struct lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -68,13 +86,20 @@ lx_ungetc(struct lx *lx, int c) } } +/* Get a character from fgetc and push it to the buffer */ int lx_fgetc(struct lx *lx) { assert(lx != NULL); assert(lx->getc_opaque != NULL); - return fgetc(lx->getc_opaque); + const int c = fgetc(lx->getc_opaque); + if (c == EOF) { + lx->c = EOF; + return EOF; + } else { + return c; + } } int @@ -119,6 +144,17 @@ lx_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_dynclear(void *buf_opaque) { @@ -158,44 +194,53 @@ lx_dynfree(void *buf_opaque) static enum lx_token z0(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\n': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return lx->z(lx); + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); case S2: /* e.g. "" */ - lx_ungetc(lx, c); return lx->z = z1, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z1, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return lx->z = z1, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S0: @@ -212,44 +257,40 @@ z0(struct lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z1(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '0': case '1': @@ -268,7 +309,9 @@ z1(struct lx *lx) case '\r': case ' ': state = S4; break; case ']': state = S5; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -284,15 +327,15 @@ z1(struct lx *lx) case '7': case '8': case '9': break; - default: lx_ungetc(lx, c); return TOK_ENDID; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ENDID; } break; case S2: /* e.g. "," */ - lx_ungetc(lx, c); return TOK_COMMA; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_COMMA; case S3: /* e.g. "#" */ - lx_ungetc(lx, c); return lx->z = z0, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z0, lx->z(lx); case S4: /* e.g. "\\x09" */ switch ((unsigned char) c) { @@ -300,16 +343,29 @@ z1(struct lx *lx) case '\n': case '\r': case ' ': break; - default: lx_ungetc(lx, c); return lx->z(lx); + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); } break; case S5: /* e.g. "]" */ - lx_ungetc(lx, c); return lx->z = z5, TOK_CLOSEENDIDS; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, TOK_CLOSEENDIDS; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_ENDID; + case S2: return TOK_COMMA; + case S3: return lx->z = z0, lx->z(lx); + case S4: return TOK_EOF; + case S5: return lx->z = z5, TOK_CLOSEENDIDS; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S3: @@ -325,106 +381,105 @@ z1(struct lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_ENDID; - case S2: return TOK_COMMA; - case S3: return TOK_EOF; - case S4: return TOK_EOF; - case S5: return TOK_CLOSEENDIDS; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z2(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\'': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z5, TOK_LABEL; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, TOK_LABEL; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return lx->z = z5, TOK_LABEL; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_LABEL; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z3(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; case '"': state = S3; break; @@ -450,15 +505,15 @@ z3(struct lx *lx) case 'r': case 't': case 'v': state = S6; break; - default: lx_ungetc(lx, c); return TOK_CHAR; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S2: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S2: /* e.g. "\\x00" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z5, TOK_LABEL; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, TOK_LABEL; case S4: /* e.g. "\\x" */ switch ((unsigned char) c) { @@ -484,7 +539,9 @@ z3(struct lx *lx) case 'd': case 'e': case 'f': state = S7; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -498,14 +555,14 @@ z3(struct lx *lx) case '5': case '6': case '7': break; - default: lx_ungetc(lx, c); return TOK_OCT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S6: /* e.g. "\\f" */ - lx_ungetc(lx, c); return TOK_ESC; + case S6: /* e.g. "\\\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ESC; - case S7: /* e.g. "\\xa" */ + case S7: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -529,76 +586,92 @@ z3(struct lx *lx) case 'd': case 'e': case 'f': break; - default: lx_ungetc(lx, c); return TOK_HEX; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_HEX; } break; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; - case S3: return TOK_LABEL; + case S3: return lx->z = z5, TOK_LABEL; case S5: return TOK_OCT; case S6: return TOK_ESC; case S7: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z4(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\n': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return lx->z(lx); + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); case S2: /* e.g. "" */ - lx_ungetc(lx, c); return lx->z = z5, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return lx->z = z5, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S0: @@ -615,46 +688,42 @@ z4(struct lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z5(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case ',': state = S1; break; case ';': state = S2; break; @@ -732,31 +801,35 @@ z5(struct lx *lx) case '\n': case '\r': case ' ': state = S13; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S1: /* e.g. "," */ - lx_ungetc(lx, c); return TOK_COMMA; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_COMMA; case S2: /* e.g. ";" */ - lx_ungetc(lx, c); return TOK_SEP; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_SEP; - case S3: /* e.g. "?" */ - lx_ungetc(lx, c); return TOK_ANY; + case S3: /* e.g. "\077" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ANY; case S4: /* e.g. "-" */ switch ((unsigned char) c) { case '>': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S5: /* e.g. "[" */ - lx_ungetc(lx, c); return lx->z = z1, TOK_OPENENDIDS; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENENDIDS; case S6: /* e.g. "=" */ - lx_ungetc(lx, c); return TOK_EQUALS; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_EQUALS; case S7: /* e.g. "e" */ switch ((unsigned char) c) { @@ -823,7 +896,7 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case 'n': state = S19; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; @@ -892,11 +965,11 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case 't': state = S14; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S9: /* e.g. "a" */ + case S9: /* e.g. "0" */ switch ((unsigned char) c) { case '0': case '1': @@ -961,18 +1034,18 @@ z5(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; case S10: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z2, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z2, lx->z(lx); case S11: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z3, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z3, lx->z(lx); case S12: /* e.g. "#" */ - lx_ungetc(lx, c); return lx->z = z4, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, lx->z(lx); case S13: /* e.g. "\\x09" */ switch ((unsigned char) c) { @@ -980,7 +1053,7 @@ z5(struct lx *lx) case '\n': case '\r': case ' ': break; - default: lx_ungetc(lx, c); return lx->z(lx); + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); } break; @@ -1049,7 +1122,7 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case 'a': state = S15; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; @@ -1118,7 +1191,7 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case 'r': state = S16; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; @@ -1187,7 +1260,7 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case 't': state = S17; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; @@ -1257,12 +1330,12 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case ':': state = S18; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; case S18: /* e.g. "start:" */ - lx_ungetc(lx, c); return TOK_START; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_START; case S19: /* e.g. "en" */ switch ((unsigned char) c) { @@ -1329,7 +1402,7 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case 'd': state = S20; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; @@ -1399,53 +1472,34 @@ z5(struct lx *lx) case 'y': case 'z': state = S9; break; case ':': state = S21; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; case S21: /* e.g. "end:" */ - lx_ungetc(lx, c); return TOK_END; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_END; case S22: /* e.g. "->" */ - lx_ungetc(lx, c); return TOK_TO; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TO; default: ; /* unreached */ } - - switch (state) { - case S10: - case S11: - case S12: - case S13: - break; - - default: - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } - break; - - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_COMMA; case S2: return TOK_SEP; case S3: return TOK_ANY; - case S5: return TOK_OPENENDIDS; + case S5: return lx->z = z1, TOK_OPENENDIDS; case S6: return TOK_EQUALS; case S7: return TOK_IDENT; case S8: return TOK_IDENT; case S9: return TOK_IDENT; - case S10: return TOK_EOF; - case S11: return TOK_EOF; - case S12: return TOK_EOF; + case S10: return lx->z = z2, lx->z(lx); + case S11: return lx->z = z3, lx->z(lx); + case S12: return lx->z = z4, lx->z(lx); case S13: return TOK_EOF; case S14: return TOK_IDENT; case S15: return TOK_IDENT; @@ -1456,8 +1510,34 @@ z5(struct lx *lx) case S20: return TOK_IDENT; case S21: return TOK_END; case S22: return TOK_TO; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + switch (state) { + case S10: + case S11: + case S12: + case S13: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + break; + + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -1640,6 +1720,7 @@ lx_init(struct lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_dynpop; } enum lx_token diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index e4ac8a31b..ec9bf4f78 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 153 "src/libfsm/parser.act" +#line 27 "src/libfsm/parser.act" #include @@ -179,7 +179,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: CHAR */ { -#line 247 "src/libfsm/parser.act" +#line 243 "src/libfsm/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); @@ -196,7 +196,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: ESC */ { -#line 171 "src/libfsm/parser.act" +#line 167 "src/libfsm/parser.act" assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(2 == strlen(lex_state->buf.a)); @@ -224,7 +224,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: HEX */ { -#line 240 "src/libfsm/parser.act" +#line 214 "src/libfsm/parser.act" unsigned long u; char *e; @@ -263,7 +263,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: OCT */ { -#line 211 "src/libfsm/parser.act" +#line 185 "src/libfsm/parser.act" unsigned long u; char *e; @@ -338,7 +338,7 @@ ZL2_items:; case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 252 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIa = xstrdup(lex_state->buf.a); @@ -366,7 +366,7 @@ ZL2_items:; goto ZL2_items; /* END OF INLINE: items */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -451,7 +451,7 @@ ZL2_xend_C_Cend_Hstates:; } /* END OF INLINE: xend::end-states */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -489,7 +489,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-start */ { -#line 404 "src/libfsm/parser.act" +#line 402 "src/libfsm/parser.act" err_expected(lex_state, "'start:'"); @@ -507,7 +507,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 252 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); @@ -530,7 +530,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) } /* BEGINNING OF ACTION: add-state */ { -#line 284 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -588,7 +588,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ { -#line 336 "src/libfsm/parser.act" +#line 335 "src/libfsm/parser.act" fsm_setstart(fsm, (ZIs)); @@ -597,7 +597,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ { -#line 350 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIn)); @@ -639,7 +639,7 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-end */ { -#line 408 "src/libfsm/parser.act" +#line 406 "src/libfsm/parser.act" err_expected(lex_state, "'end:'"); @@ -687,7 +687,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 252 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); @@ -705,7 +705,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 284 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -763,7 +763,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-end */ { -#line 340 "src/libfsm/parser.act" +#line 339 "src/libfsm/parser.act" fsm_setend(fsm, (ZIs), 1); @@ -772,7 +772,7 @@ p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *Z /* END OF ACTION: mark-end */ /* BEGINNING OF ACTION: free */ { -#line 350 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIn)); @@ -810,7 +810,7 @@ p_fsm(fsm fsm, lex_state lex_state, act_state act_state) ADVANCE_LEXER; /* BEGINNING OF ACTION: free-statelist */ { -#line 366 "src/libfsm/parser.act" +#line 353 "src/libfsm/parser.act" struct act_statelist *p; struct act_statelist *next; @@ -834,7 +834,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-syntax */ { -#line 413 "src/libfsm/parser.act" +#line 410 "src/libfsm/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); @@ -865,7 +865,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-comma */ { -#line 400 "src/libfsm/parser.act" +#line 398 "src/libfsm/parser.act" err_expected(lex_state, "','"); @@ -895,7 +895,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-sep */ { -#line 392 "src/libfsm/parser.act" +#line 390 "src/libfsm/parser.act" err_expected(lex_state, "';'"); @@ -923,7 +923,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 252 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); @@ -941,7 +941,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 284 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -999,7 +999,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ { -#line 284 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIb)); @@ -1057,7 +1057,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 350 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((*ZIa)); @@ -1066,7 +1066,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ { -#line 350 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIb)); @@ -1081,7 +1081,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; /* BEGINNING OF ACTION: add-edge-any */ { -#line 376 "src/libfsm/parser.act" +#line 375 "src/libfsm/parser.act" if (!fsm_addedge_any(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_any"); @@ -1104,7 +1104,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) } /* BEGINNING OF ACTION: add-edge-literal */ { -#line 369 "src/libfsm/parser.act" +#line 368 "src/libfsm/parser.act" if (!fsm_addedge_literal(fsm, (ZIx), (ZIy), (ZIc))) { perror("fsm_addedge_literal"); @@ -1120,7 +1120,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: add-edge-epsilon */ { -#line 383 "src/libfsm/parser.act" +#line 382 "src/libfsm/parser.act" if (!fsm_addedge_epsilon(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_epsilon"); @@ -1138,7 +1138,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: err-expected-trans */ { -#line 396 "src/libfsm/parser.act" +#line 394 "src/libfsm/parser.act" err_expected(lex_state, "transition"); @@ -1162,7 +1162,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* BEGINNING OF ACTION: add-state */ { -#line 284 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -1220,7 +1220,7 @@ p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 350 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((*ZIa)); @@ -1269,7 +1269,7 @@ ZL2_xend_C_Cend_Hids:; } /* END OF INLINE: xend::end-ids */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): RESTORE_LEXER; goto ZL1; @@ -1298,7 +1298,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) case (TOK_ENDID): /* BEGINNING OF EXTRACT: ENDID */ { -#line 277 "src/libfsm/parser.act" +#line 255 "src/libfsm/parser.act" unsigned long u; char *e; @@ -1333,7 +1333,7 @@ p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-end-id */ { -#line 344 "src/libfsm/parser.act" +#line 343 "src/libfsm/parser.act" if (!fsm_endid_set(fsm, (ZIs), (ZIid))) { goto ZL1; @@ -1351,7 +1351,7 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 479 "src/libfsm/parser.act" +#line 415 "src/libfsm/parser.act" struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc) { diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index edeebb112..32f562c66 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 163 "src/libfsm/parser.act" +#line 153 "src/libfsm/parser.act" typedef struct lex_state * lex_state; @@ -27,7 +27,7 @@ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 480 "src/libfsm/parser.act" +#line 479 "src/libfsm/parser.act" #line 33 "src/libfsm/parser.h" diff --git a/src/libre/dialect/glob/lexer.c b/src/libre/dialect/glob/lexer.c index 843cedc4e..2a4d33a29 100644 --- a/src/libre/dialect/glob/lexer.c +++ b/src/libre/dialect/glob/lexer.c @@ -10,11 +10,31 @@ static enum lx_glob_token z0(struct lx_glob_lx *lx); +static int +lx_glob_advance_end(struct lx_glob_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_glob_lx *lx) +lx_glob_getc(struct lx_glob_lx *lx) { int c; @@ -30,18 +50,19 @@ lx_getc(struct lx_glob_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_glob_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_glob_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_glob_getc((struct lx_glob_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -50,10 +71,7 @@ lx_glob_ungetc(struct lx_glob_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -105,6 +123,17 @@ lx_glob_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_glob_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_glob_dynclear(void *buf_opaque) { @@ -144,29 +173,28 @@ lx_glob_dynfree(void *buf_opaque) static enum lx_glob_token z0(struct lx_glob_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '*': state = S2; break; case '?': state = S3; break; @@ -175,34 +203,41 @@ z0(struct lx_glob_lx *lx) break; case S1: /* e.g. "\\x00" */ - lx_glob_ungetc(lx, c); return TOK_CHAR; + lx_glob_ungetc(lx, c); lx_glob_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "*" */ - lx_glob_ungetc(lx, c); return TOK_MANY; + lx_glob_ungetc(lx, c); lx_glob_dynpop(lx->buf_opaque); return TOK_MANY; - case S3: /* e.g. "?" */ - lx_glob_ungetc(lx, c); return TOK_ANY; + case S3: /* e.g. "\077" */ + lx_glob_ungetc(lx, c); lx_glob_dynpop(lx->buf_opaque); return TOK_ANY; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_MANY; + case S3: return TOK_ANY; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_glob_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_MANY; - case S3: return TOK_ANY; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -254,6 +289,7 @@ lx_glob_init(struct lx_glob_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_glob_dynpop; } enum lx_glob_token diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index c8f021380..b20798f4f 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 275 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -304,7 +304,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -322,7 +322,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -348,7 +348,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -358,7 +358,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -378,7 +378,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -397,7 +397,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { @@ -418,7 +418,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -428,7 +428,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -440,7 +440,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 809 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -449,7 +449,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -478,7 +478,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 708 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -490,7 +490,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -523,7 +523,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -537,7 +537,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -558,7 +558,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -575,7 +575,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -601,7 +601,7 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 757 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -626,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1207 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index ec618caca..89800e6e9 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 292 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1209 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/like/lexer.c b/src/libre/dialect/like/lexer.c index 4f4dcbdab..2edc365a9 100644 --- a/src/libre/dialect/like/lexer.c +++ b/src/libre/dialect/like/lexer.c @@ -10,11 +10,31 @@ static enum lx_like_token z0(struct lx_like_lx *lx); +static int +lx_like_advance_end(struct lx_like_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_like_lx *lx) +lx_like_getc(struct lx_like_lx *lx) { int c; @@ -30,18 +50,19 @@ lx_getc(struct lx_like_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_like_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_like_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_like_getc((struct lx_like_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -50,10 +71,7 @@ lx_like_ungetc(struct lx_like_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -105,6 +123,17 @@ lx_like_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_like_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_like_dynclear(void *buf_opaque) { @@ -144,29 +173,28 @@ lx_like_dynfree(void *buf_opaque) static enum lx_like_token z0(struct lx_like_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '%': state = S2; break; case '_': state = S3; break; @@ -175,34 +203,41 @@ z0(struct lx_like_lx *lx) break; case S1: /* e.g. "\\x00" */ - lx_like_ungetc(lx, c); return TOK_CHAR; + lx_like_ungetc(lx, c); lx_like_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "%" */ - lx_like_ungetc(lx, c); return TOK_MANY; + lx_like_ungetc(lx, c); lx_like_dynpop(lx->buf_opaque); return TOK_MANY; case S3: /* e.g. "_" */ - lx_like_ungetc(lx, c); return TOK_ANY; + lx_like_ungetc(lx, c); lx_like_dynpop(lx->buf_opaque); return TOK_ANY; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_MANY; + case S3: return TOK_ANY; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_like_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_MANY; - case S3: return TOK_ANY; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -254,6 +289,7 @@ lx_like_init(struct lx_like_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_like_dynpop; } enum lx_like_token diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index 64bfe1078..008bd2b04 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 275 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -304,7 +304,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -322,7 +322,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -348,7 +348,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -358,7 +358,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -378,7 +378,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -397,7 +397,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { @@ -418,7 +418,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -428,7 +428,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -440,7 +440,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 809 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -449,7 +449,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -478,7 +478,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 708 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -490,7 +490,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -523,7 +523,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -537,7 +537,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -558,7 +558,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -575,7 +575,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* END OF INLINE: 119 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -601,7 +601,7 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 757 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -626,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1207 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index f6c87ad7b..5294f9792 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 292 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1209 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/literal/lexer.c b/src/libre/dialect/literal/lexer.c index f4ff77a37..f13bdbc6f 100644 --- a/src/libre/dialect/literal/lexer.c +++ b/src/libre/dialect/literal/lexer.c @@ -10,11 +10,31 @@ static enum lx_literal_token z0(struct lx_literal_lx *lx); +static int +lx_literal_advance_end(struct lx_literal_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_literal_lx *lx) +lx_literal_getc(struct lx_literal_lx *lx) { int c; @@ -30,18 +50,19 @@ lx_getc(struct lx_literal_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_literal_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_literal_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_literal_getc((struct lx_literal_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -50,10 +71,7 @@ lx_literal_ungetc(struct lx_literal_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -105,6 +123,17 @@ lx_literal_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_literal_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_literal_dynclear(void *buf_opaque) { @@ -144,52 +173,58 @@ lx_literal_dynfree(void *buf_opaque) static enum lx_literal_token z0(struct lx_literal_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ state = S1; break; case S1: /* e.g. "" */ - lx_literal_ungetc(lx, c); return TOK_CHAR; + lx_literal_ungetc(lx, c); lx_literal_dynpop(lx->buf_opaque); return TOK_CHAR; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_literal_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -237,6 +272,7 @@ lx_literal_init(struct lx_literal_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_literal_dynpop; } enum lx_literal_token diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 44547716b..5d1dc82f7 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 275 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -304,7 +304,7 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -322,7 +322,7 @@ ZL2_list_Hof_Hatoms:; goto ZL2_list_Hof_Hatoms; /* END OF INLINE: list-of-atoms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -349,7 +349,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -363,7 +363,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -384,7 +384,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -401,7 +401,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ /* END OF INLINE: 117 */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -427,7 +427,7 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 757 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -467,7 +467,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -490,7 +490,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -506,7 +506,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 708 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -518,7 +518,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -539,7 +539,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1207 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index be58db4ea..7f90a15ef 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 292 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1209 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/native/lexer.c b/src/libre/dialect/native/lexer.c index 2399683ac..b18634004 100644 --- a/src/libre/dialect/native/lexer.c +++ b/src/libre/dialect/native/lexer.c @@ -12,11 +12,31 @@ static enum lx_native_token z0(struct lx_native_lx *lx); static enum lx_native_token z1(struct lx_native_lx *lx); static enum lx_native_token z2(struct lx_native_lx *lx); +static int +lx_native_advance_end(struct lx_native_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_native_lx *lx) +lx_native_getc(struct lx_native_lx *lx) { int c; @@ -32,18 +52,19 @@ lx_getc(struct lx_native_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_native_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_native_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_native_getc((struct lx_native_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -52,10 +73,7 @@ lx_native_ungetc(struct lx_native_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -107,6 +125,17 @@ lx_native_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_native_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_native_dynclear(void *buf_opaque) { @@ -146,29 +175,28 @@ lx_native_dynfree(void *buf_opaque) static enum lx_native_token z0(struct lx_native_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '0': case '1': @@ -182,7 +210,9 @@ z0(struct lx_native_lx *lx) case '9': state = S1; break; case ',': state = S2; break; case '}': state = S3; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -198,69 +228,75 @@ z0(struct lx_native_lx *lx) case '7': case '8': case '9': break; - default: lx_native_ungetc(lx, c); return TOK_COUNT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_COUNT; } break; case S2: /* e.g. "," */ - lx_native_ungetc(lx, c); return TOK_SEP; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_SEP; case S3: /* e.g. "}" */ - lx_native_ungetc(lx, c); return lx->z = z2, TOK_CLOSECOUNT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSECOUNT; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COUNT; + case S2: return TOK_SEP; + case S3: return lx->z = z2, TOK_CLOSECOUNT; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_COUNT; - case S2: return TOK_SEP; - case S3: return TOK_CLOSECOUNT; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_native_token z1(struct lx_native_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, - S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, - S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, - S50, S51, S52, S53, S54, S55, S56, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, + S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, + S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, + S50, S51, S52, S53, S54, S55, S56 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '[': state = S1; break; case '\\': state = S3; break; @@ -273,12 +309,12 @@ z1(struct lx_native_lx *lx) case S1: /* e.g. "[" */ switch ((unsigned char) c) { case ':': state = S12; break; - default: lx_native_ungetc(lx, c); return TOK_CHAR; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; } break; case S2: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_CHAR; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\\" */ switch ((unsigned char) c) { @@ -301,25 +337,27 @@ z1(struct lx_native_lx *lx) case '5': case '6': case '7': state = S9; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S4: /* e.g. "-" */ switch ((unsigned char) c) { case ']': state = S6; break; - default: lx_native_ungetc(lx, c); return TOK_RANGE; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_RANGE; } break; case S5: /* e.g. "]" */ - lx_native_ungetc(lx, c); return lx->z = z2, TOK_CLOSEGROUP; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSEGROUP; case S6: /* e.g. "-]" */ - lx_native_ungetc(lx, c); return lx->z = z2, TOK_CLOSEGROUPRANGE; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSEGROUPRANGE; case S7: /* e.g. "\\-" */ - lx_native_ungetc(lx, c); return TOK_ESC; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ESC; case S8: /* e.g. "\\x" */ switch ((unsigned char) c) { @@ -345,7 +383,9 @@ z1(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S10; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -359,7 +399,7 @@ z1(struct lx_native_lx *lx) case '5': case '6': case '7': break; - default: lx_native_ungetc(lx, c); return TOK_OCT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; } break; @@ -387,12 +427,14 @@ z1(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S11; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S11: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_HEX; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_HEX; case S12: /* e.g. "[:" */ switch ((unsigned char) c) { @@ -408,35 +450,45 @@ z1(struct lx_native_lx *lx) case 'w': state = S21; break; case 'u': state = S22; break; case 'p': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S13: /* e.g. "[:d" */ switch ((unsigned char) c) { case 'i': state = S55; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S14: /* e.g. "[:s" */ switch ((unsigned char) c) { case 'p': state = S52; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S15: /* e.g. "[:h" */ switch ((unsigned char) c) { case 's': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S16: /* e.g. "[:g" */ switch ((unsigned char) c) { case 'r': state = S49; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -444,42 +496,54 @@ z1(struct lx_native_lx *lx) switch ((unsigned char) c) { case 's': state = S41; break; case 'l': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S18: /* e.g. "[:c" */ switch ((unsigned char) c) { case 'n': state = S38; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S19: /* e.g. "[:l" */ switch ((unsigned char) c) { case 'o': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S20: /* e.g. "[:x" */ switch ((unsigned char) c) { case 'd': state = S13; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S21: /* e.g. "[:w" */ switch ((unsigned char) c) { case 'o': state = S35; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S22: /* e.g. "[:u" */ switch ((unsigned char) c) { case 'p': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -487,129 +551,165 @@ z1(struct lx_native_lx *lx) switch ((unsigned char) c) { case 'r': state = S24; break; case 'u': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S24: /* e.g. "[:pr" */ switch ((unsigned char) c) { case 'i': state = S31; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S25: /* e.g. "[:pu" */ switch ((unsigned char) c) { case 'n': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S26: /* e.g. "[:pun" */ switch ((unsigned char) c) { case 'c': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S27: /* e.g. "[:digi" */ switch ((unsigned char) c) { case 't': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S28: /* e.g. "[:word" */ switch ((unsigned char) c) { case ':': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S29: /* e.g. "[:word:" */ switch ((unsigned char) c) { case ']': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S30: /* e.g. "[:word:]" */ - lx_native_ungetc(lx, c); return TOK_NAMED__CLASS; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; case S31: /* e.g. "[:pri" */ switch ((unsigned char) c) { case 'n': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S32: /* e.g. "[:up" */ switch ((unsigned char) c) { case 'p': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S33: /* e.g. "[:low" */ switch ((unsigned char) c) { case 'e': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S34: /* e.g. "[:lowe" */ switch ((unsigned char) c) { case 'r': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S35: /* e.g. "[:wo" */ switch ((unsigned char) c) { case 'r': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S36: /* e.g. "[:wor" */ switch ((unsigned char) c) { case 'd': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S37: /* e.g. "[:lo" */ switch ((unsigned char) c) { case 'w': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S38: /* e.g. "[:cn" */ switch ((unsigned char) c) { case 't': state = S39; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S39: /* e.g. "[:cnt" */ switch ((unsigned char) c) { case 'r': state = S40; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S40: /* e.g. "[:cntr" */ switch ((unsigned char) c) { case 'l': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S41: /* e.g. "[:as" */ switch ((unsigned char) c) { case 'c': state = S47; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -617,164 +717,200 @@ z1(struct lx_native_lx *lx) switch ((unsigned char) c) { case 'p': state = S43; break; case 'n': state = S44; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S43: /* e.g. "[:alp" */ switch ((unsigned char) c) { case 'h': state = S46; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S44: /* e.g. "[:aln" */ switch ((unsigned char) c) { case 'u': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S45: /* e.g. "[:alnu" */ switch ((unsigned char) c) { case 'm': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S46: /* e.g. "[:alph" */ switch ((unsigned char) c) { case 'a': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S47: /* e.g. "[:asc" */ switch ((unsigned char) c) { case 'i': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S48: /* e.g. "[:asci" */ switch ((unsigned char) c) { case 'i': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S49: /* e.g. "[:gr" */ switch ((unsigned char) c) { case 'a': state = S50; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S50: /* e.g. "[:gra" */ switch ((unsigned char) c) { case 'p': state = S51; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S51: /* e.g. "[:grap" */ switch ((unsigned char) c) { case 'h': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S52: /* e.g. "[:sp" */ switch ((unsigned char) c) { case 'a': state = S53; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S53: /* e.g. "[:spa" */ switch ((unsigned char) c) { case 'c': state = S54; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S54: /* e.g. "[:spac" */ switch ((unsigned char) c) { case 'e': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S55: /* e.g. "[:di" */ switch ((unsigned char) c) { case 'g': state = S56; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S56: /* e.g. "[:dig" */ switch ((unsigned char) c) { case 'i': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; case S4: return TOK_RANGE; - case S5: return TOK_CLOSEGROUP; - case S6: return TOK_CLOSEGROUPRANGE; + case S5: return lx->z = z2, TOK_CLOSEGROUP; + case S6: return lx->z = z2, TOK_CLOSEGROUPRANGE; case S7: return TOK_ESC; case S9: return TOK_OCT; case S11: return TOK_HEX; case S30: return TOK_NAMED__CLASS; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_native_token z2(struct lx_native_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; case '{': state = S3; break; @@ -820,63 +956,63 @@ z2(struct lx_native_lx *lx) case 'v': case '{': case '|': state = S19; break; - default: lx_native_ungetc(lx, c); return TOK_CHAR; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; } break; case S2: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_CHAR; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "{" */ - lx_native_ungetc(lx, c); return lx->z = z0, TOK_OPENCOUNT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z0, TOK_OPENCOUNT; case S4: /* e.g. "[" */ switch ((unsigned char) c) { case '^': state = S14; break; case ']': state = S15; break; - default: lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUP; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUP; } break; case S5: /* e.g. "|" */ - lx_native_ungetc(lx, c); return TOK_ALT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ALT; case S6: /* e.g. "." */ - lx_native_ungetc(lx, c); return TOK_ANY; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ANY; case S7: /* e.g. "+" */ - lx_native_ungetc(lx, c); return TOK_PLUS; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_PLUS; case S8: /* e.g. "*" */ - lx_native_ungetc(lx, c); return TOK_STAR; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_STAR; - case S9: /* e.g. "?" */ - lx_native_ungetc(lx, c); return TOK_OPT; + case S9: /* e.g. "\077" */ + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OPT; case S10: /* e.g. "$" */ - lx_native_ungetc(lx, c); return TOK_END; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_END; case S11: /* e.g. "^" */ - lx_native_ungetc(lx, c); return TOK_START; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_START; case S12: /* e.g. ")" */ - lx_native_ungetc(lx, c); return TOK_CLOSESUB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CLOSESUB; case S13: /* e.g. "(" */ - lx_native_ungetc(lx, c); return TOK_OPENSUB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OPENSUB; case S14: /* e.g. "[^" */ switch ((unsigned char) c) { case ']': state = S16; break; - default: lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUPINV; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUPINV; } break; case S15: /* e.g. "[]" */ - lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUPCB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUPCB; case S16: /* e.g. "[^]" */ - lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUPINVCB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUPINVCB; case S17: /* e.g. "\\x" */ switch ((unsigned char) c) { @@ -902,7 +1038,9 @@ z2(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -916,12 +1054,12 @@ z2(struct lx_native_lx *lx) case '5': case '6': case '7': state = S20; break; - default: lx_native_ungetc(lx, c); return TOK_OCT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; } break; case S19: /* e.g. "\\$" */ - lx_native_ungetc(lx, c); return TOK_ESC; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ESC; case S20: /* e.g. "\\00" */ switch ((unsigned char) c) { @@ -933,12 +1071,12 @@ z2(struct lx_native_lx *lx) case '5': case '6': case '7': state = S21; break; - default: lx_native_ungetc(lx, c); return TOK_OCT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; } break; case S21: /* e.g. "\\000" */ - lx_native_ungetc(lx, c); return TOK_OCT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; case S22: /* e.g. "\\x0" */ switch ((unsigned char) c) { @@ -964,32 +1102,26 @@ z2(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S23: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_HEX; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_HEX; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; - case S3: return TOK_OPENCOUNT; - case S4: return TOK_OPENGROUP; + case S3: return lx->z = z0, TOK_OPENCOUNT; + case S4: return lx->z = z1, TOK_OPENGROUP; case S5: return TOK_ALT; case S6: return TOK_ANY; case S7: return TOK_PLUS; @@ -999,16 +1131,31 @@ z2(struct lx_native_lx *lx) case S11: return TOK_START; case S12: return TOK_CLOSESUB; case S13: return TOK_OPENSUB; - case S14: return TOK_OPENGROUPINV; - case S15: return TOK_OPENGROUPCB; - case S16: return TOK_OPENGROUPINVCB; + case S14: return lx->z = z1, TOK_OPENGROUPINV; + case S15: return lx->z = z1, TOK_OPENGROUPCB; + case S16: return lx->z = z1, TOK_OPENGROUPINVCB; case S18: return TOK_OCT; case S19: return TOK_ESC; case S20: return TOK_OCT; case S21: return TOK_OCT; case S23: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -1164,6 +1311,7 @@ lx_native_init(struct lx_native_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_native_dynpop; } enum lx_native_token diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 809383bf8..63c5f8cb8 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 275 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -326,7 +326,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -349,7 +349,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 391 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -385,7 +385,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 535 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -443,7 +443,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 492 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -499,7 +499,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 141 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -531,7 +531,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI216 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -545,7 +545,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); @@ -555,7 +555,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -586,7 +586,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 636 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -618,7 +618,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI219 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -636,7 +636,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); @@ -646,7 +646,7 @@ p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -696,7 +696,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -709,7 +709,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -736,7 +736,7 @@ p_154(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI155 = '-'; ZI156 = lex_state->lx.start; @@ -760,7 +760,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 722 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; @@ -795,7 +795,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -815,7 +815,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -849,7 +849,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -875,7 +875,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 391 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -914,7 +914,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 535 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -975,7 +975,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 492 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1031,7 +1031,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 109 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1064,7 +1064,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1096,7 +1096,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 391 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1141,7 +1141,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 535 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1208,7 +1208,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 648 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZI243 = DIALECT_CLASS(lex_state->buf.a); if (ZI243 == NULL) { @@ -1241,7 +1241,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 492 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1325,7 +1325,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 648 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1349,7 +1349,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 845 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); @@ -1389,7 +1389,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 319 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI167 = lex_state->lx.end; @@ -1403,7 +1403,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1430,7 +1430,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 335 "src/libre/parser.act" +#line 334 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI186 = lex_state->lx.end; @@ -1444,7 +1444,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1457,7 +1457,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 886 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1471,7 +1471,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -1493,7 +1493,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 327 "src/libre/parser.act" +#line 326 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI178 = lex_state->lx.end; @@ -1507,7 +1507,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1520,7 +1520,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1577,7 +1577,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 343 "src/libre/parser.act" +#line 342 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI193 = lex_state->lx.end; @@ -1591,7 +1591,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1604,7 +1604,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1647,7 +1647,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 886 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1661,7 +1661,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -1693,7 +1693,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 351 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI200 = ']'; ZI201 = lex_state->lx.start; @@ -1709,7 +1709,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 768 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -1728,7 +1728,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 361 "src/libre/parser.act" +#line 360 "src/libre/parser.act" ZIcrange = '-'; ZI203 = lex_state->lx.start; @@ -1744,7 +1744,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { @@ -1756,7 +1756,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; @@ -1767,7 +1767,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 768 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -1785,7 +1785,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 729 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -1821,7 +1821,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -1842,7 +1842,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { @@ -1865,7 +1865,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); @@ -1875,7 +1875,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI183 = '-'; ZI184 = lex_state->lx.start; @@ -1896,7 +1896,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -1935,7 +1935,7 @@ p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 182 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; @@ -1977,7 +1977,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI149 = lex_state->lx.start; @@ -1993,7 +1993,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -2050,7 +2050,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2088,7 +2088,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2109,7 +2109,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2129,7 +2129,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 715 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -2141,7 +2141,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2170,7 +2170,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { @@ -2194,7 +2194,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); @@ -2204,7 +2204,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI196 = '-'; ZI197 = lex_state->lx.start; @@ -2225,7 +2225,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); @@ -2235,7 +2235,7 @@ p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2294,7 +2294,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -2308,7 +2308,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a } /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -2334,7 +2334,7 @@ p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_a { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 757 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -2422,7 +2422,7 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; @@ -2441,7 +2441,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -2453,7 +2453,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 715 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -2485,7 +2485,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 371 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI263 = lex_state->lx.start; ZI264 = lex_state->lx.end; @@ -2501,7 +2501,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 636 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -2541,7 +2541,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 817 "src/libre/parser.act" +#line 816 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); @@ -2555,7 +2555,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 813 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); @@ -2569,7 +2569,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 809 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -2582,7 +2582,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 821 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -2599,7 +2599,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 701 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; @@ -2611,7 +2611,7 @@ ZL1:; /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 821 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -2640,7 +2640,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -2650,7 +2650,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -2667,7 +2667,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 943 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -2687,7 +2687,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -2701,7 +2701,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { @@ -2725,7 +2725,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 936 "src/libre/parser.act" +#line 935 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { @@ -2765,7 +2765,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 708 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -2777,7 +2777,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -2806,7 +2806,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI243)); if ((ZInode) == NULL) { @@ -2826,7 +2826,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 845 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI243); @@ -2842,7 +2842,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 773 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); @@ -2852,7 +2852,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2908,7 +2908,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2942,7 +2942,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI247)); if ((ZInode) == NULL) { @@ -2962,7 +2962,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI247); @@ -2978,7 +2978,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 773 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); @@ -2988,7 +2988,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -3035,7 +3035,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1207 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index e19648892..5cf04f6c6 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 292 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1209 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/pcre/lexer.c b/src/libre/dialect/pcre/lexer.c index b26096785..da8f825ee 100644 --- a/src/libre/dialect/pcre/lexer.c +++ b/src/libre/dialect/pcre/lexer.c @@ -17,11 +17,31 @@ static enum lx_pcre_token z5(struct lx_pcre_lx *lx); static enum lx_pcre_token z6(struct lx_pcre_lx *lx); static enum lx_pcre_token z7(struct lx_pcre_lx *lx); +static int +lx_pcre_advance_end(struct lx_pcre_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_pcre_lx *lx) +lx_pcre_getc(struct lx_pcre_lx *lx) { int c; @@ -37,18 +57,19 @@ lx_getc(struct lx_pcre_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_pcre_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_pcre_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_pcre_getc((struct lx_pcre_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -57,10 +78,7 @@ lx_pcre_ungetc(struct lx_pcre_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -112,6 +130,17 @@ lx_pcre_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_pcre_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_pcre_dynclear(void *buf_opaque) { @@ -151,32 +180,33 @@ lx_pcre_dynfree(void *buf_opaque) static enum lx_pcre_token z0(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S2; break; } break; @@ -184,19 +214,30 @@ z0(struct lx_pcre_lx *lx) case S1: /* e.g. "\\" */ switch ((unsigned char) c) { case 'E': state = S3; break; - default: lx_pcre_ungetc(lx, c); return TOK_CHAR; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S2: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S2: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z7, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S3: @@ -211,45 +252,40 @@ z0(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_CHAR; - case S3: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z1(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '0': case '1': @@ -263,7 +299,9 @@ z1(struct lx_pcre_lx *lx) case '9': state = S1; break; case ',': state = S2; break; case '}': state = S3; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -279,67 +317,75 @@ z1(struct lx_pcre_lx *lx) case '7': case '8': case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_COUNT; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_COUNT; } break; case S2: /* e.g. "," */ - lx_pcre_ungetc(lx, c); return TOK_SEP; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_SEP; case S3: /* e.g. "}" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSECOUNT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSECOUNT; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COUNT; + case S2: return TOK_SEP; + case S3: return lx->z = z7, TOK_CLOSECOUNT; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_COUNT; - case S2: return TOK_SEP; - case S3: return TOK_CLOSECOUNT; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z2(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S2; break; } break; @@ -347,19 +393,30 @@ z2(struct lx_pcre_lx *lx) case S1: /* e.g. "\\" */ switch ((unsigned char) c) { case 'E': state = S3; break; - default: lx_pcre_ungetc(lx, c); return TOK_CHAR; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S2: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S2: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z = z3, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z3, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S3: @@ -374,24 +431,30 @@ z2(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_CHAR; - case S3: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z3(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + lx->start = lx->end; + + void *getc_opaque = (void *)lx; enum { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, @@ -400,32 +463,23 @@ z3(struct lx_pcre_lx *lx) S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, S50, S51, S52, S53, S54, S55, S56, S57, S58, S59, S60, S61, S62, S63, S64, S65, S66, S67, S68, S69, - S70, S71, S72, NONE + S70, S71, S72 } state; - assert(lx != NULL); - - if (lx->clear != NULL) { - lx->clear(lx->buf_opaque); - } - - state = NONE; - - lx->start = lx->end; - - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; case '[': state = S2; break; case '-': state = S4; break; case ']': state = S5; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S3; break; } break; @@ -441,22 +495,8 @@ z3(struct lx_pcre_lx *lx) case 'h': case 's': case 'v': - case 'w': state = S24; break; - case 'Q': state = S55; break; - case 'E': state = S56; break; - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': state = S57; break; - case 'x': state = S58; break; - case '0': state = S59; break; - case 'o': state = S60; break; - case 'c': state = S61; break; + case 'w': state = S23; break; + case 'c': state = S55; break; case '$': case '(': case '*': @@ -475,384 +515,492 @@ z3(struct lx_pcre_lx *lx) case 'r': case 't': case '{': - case '|': state = S63; break; - default: state = S62; break; + case '|': state = S56; break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': state = S57; break; + case 'Q': state = S58; break; + case 'E': state = S59; break; + case 'o': state = S60; break; + case 'x': state = S62; break; + case '0': state = S63; break; + default: state = S61; break; } break; case S2: /* e.g. "[" */ switch ((unsigned char) c) { case ':': state = S7; break; - default: lx_pcre_ungetc(lx, c); return TOK_CHAR; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S3: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S3: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S4: /* e.g. "-" */ switch ((unsigned char) c) { case ']': state = S6; break; - default: lx_pcre_ungetc(lx, c); return TOK_RANGE; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_RANGE; } break; case S5: /* e.g. "]" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSEGROUP; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSEGROUP; case S6: /* e.g. "-]" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSEGROUPRANGE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSEGROUPRANGE; case S7: /* e.g. "[:" */ switch ((unsigned char) c) { case 'd': state = S8; break; - case 'u': state = S9; break; - case 'w': state = S10; break; - case 'x': state = S11; break; - case 'b': state = S12; break; - case 'c': state = S13; break; - case 'l': state = S14; break; + case 'p': state = S9; break; + case 'x': state = S10; break; + case 'c': state = S11; break; + case 'l': state = S12; break; + case 'g': state = S13; break; + case 's': state = S14; break; case 'a': state = S15; break; - case 's': state = S16; break; - case 'p': state = S17; break; - case 'g': state = S18; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'b': state = S16; break; + case 'u': state = S17; break; + case 'w': state = S18; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S8: /* e.g. "[:d" */ switch ((unsigned char) c) { case 'i': state = S53; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S9: /* e.g. "[:u" */ + case S9: /* e.g. "[:p" */ switch ((unsigned char) c) { - case 'p': state = S52; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S48; break; + case 'u': state = S49; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S10: /* e.g. "[:w" */ + case S10: /* e.g. "[:x" */ switch ((unsigned char) c) { - case 'o': state = S50; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'd': state = S8; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S11: /* e.g. "[:x" */ + case S11: /* e.g. "[:c" */ switch ((unsigned char) c) { - case 'd': state = S8; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S45; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S12: /* e.g. "[:b" */ + case S12: /* e.g. "[:l" */ switch ((unsigned char) c) { - case 'l': state = S47; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'o': state = S44; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S13: /* e.g. "[:c" */ + case S13: /* e.g. "[:g" */ switch ((unsigned char) c) { - case 'n': state = S44; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S41; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S14: /* e.g. "[:l" */ + case S14: /* e.g. "[:s" */ switch ((unsigned char) c) { - case 'o': state = S41; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S38; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S15: /* e.g. "[:a" */ switch ((unsigned char) c) { - case 's': state = S33; break; - case 'l': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 's': state = S30; break; + case 'l': state = S31; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S16: /* e.g. "[:s" */ + case S16: /* e.g. "[:b" */ switch ((unsigned char) c) { - case 'p': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'l': state = S27; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S17: /* e.g. "[:p" */ + case S17: /* e.g. "[:u" */ switch ((unsigned char) c) { - case 'r': state = S25; break; - case 'u': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S24; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S18: /* e.g. "[:g" */ + case S18: /* e.g. "[:w" */ switch ((unsigned char) c) { - case 'r': state = S19; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'o': state = S19; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S19: /* e.g. "[:gr" */ + case S19: /* e.g. "[:wo" */ switch ((unsigned char) c) { - case 'a': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S20; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S20: /* e.g. "[:gra" */ + case S20: /* e.g. "[:wor" */ switch ((unsigned char) c) { - case 'p': state = S21; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'd': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S21: /* e.g. "[:grap" */ + case S21: /* e.g. "[:word" */ switch ((unsigned char) c) { - case 'h': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case ':': state = S22; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S22: /* e.g. "[:word" */ + case S22: /* e.g. "[:word:" */ switch ((unsigned char) c) { - case ':': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case ']': state = S23; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S23: /* e.g. "[:word:" */ + case S23: /* e.g. "\\D" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; + + case S24: /* e.g. "[:up" */ switch ((unsigned char) c) { - case ']': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S25; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S24: /* e.g. "\\d" */ - lx_pcre_ungetc(lx, c); return TOK_NAMED__CLASS; - - case S25: /* e.g. "[:pr" */ + case S25: /* e.g. "[:low" */ switch ((unsigned char) c) { - case 'i': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'e': state = S26; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S26: /* e.g. "[:pu" */ + case S26: /* e.g. "[:lowe" */ switch ((unsigned char) c) { - case 'n': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S27: /* e.g. "[:pun" */ + case S27: /* e.g. "[:bl" */ switch ((unsigned char) c) { - case 'c': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S28; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S28: /* e.g. "[:digi" */ + case S28: /* e.g. "[:bla" */ switch ((unsigned char) c) { - case 't': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S29; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S29: /* e.g. "[:pri" */ + case S29: /* e.g. "[:blan" */ switch ((unsigned char) c) { - case 'n': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'k': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S30: /* e.g. "[:sp" */ + case S30: /* e.g. "[:as" */ switch ((unsigned char) c) { - case 'a': state = S31; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'c': state = S36; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S31: /* e.g. "[:spa" */ + case S31: /* e.g. "[:al" */ switch ((unsigned char) c) { - case 'c': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S32; break; + case 'n': state = S33; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S32: /* e.g. "[:spac" */ + case S32: /* e.g. "[:alp" */ switch ((unsigned char) c) { - case 'e': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'h': state = S35; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S33: /* e.g. "[:as" */ + case S33: /* e.g. "[:aln" */ switch ((unsigned char) c) { - case 'c': state = S39; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'u': state = S34; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S34: /* e.g. "[:al" */ + case S34: /* e.g. "[:alnu" */ switch ((unsigned char) c) { - case 'n': state = S35; break; - case 'p': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'm': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S35: /* e.g. "[:aln" */ + case S35: /* e.g. "[:alph" */ switch ((unsigned char) c) { - case 'u': state = S38; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S36: /* e.g. "[:alp" */ + case S36: /* e.g. "[:asc" */ switch ((unsigned char) c) { - case 'h': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S37; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S37: /* e.g. "[:alph" */ + case S37: /* e.g. "[:asci" */ switch ((unsigned char) c) { - case 'a': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S38: /* e.g. "[:alnu" */ + case S38: /* e.g. "[:sp" */ switch ((unsigned char) c) { - case 'm': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S39; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S39: /* e.g. "[:asc" */ + case S39: /* e.g. "[:spa" */ switch ((unsigned char) c) { - case 'i': state = S40; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'c': state = S40; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S40: /* e.g. "[:asci" */ + case S40: /* e.g. "[:spac" */ switch ((unsigned char) c) { - case 'i': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'e': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S41: /* e.g. "[:lo" */ + case S41: /* e.g. "[:gr" */ switch ((unsigned char) c) { - case 'w': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S42; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S42: /* e.g. "[:low" */ + case S42: /* e.g. "[:gra" */ switch ((unsigned char) c) { - case 'e': state = S43; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S43; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S43: /* e.g. "[:lowe" */ + case S43: /* e.g. "[:grap" */ switch ((unsigned char) c) { - case 'r': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'h': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S44: /* e.g. "[:cn" */ + case S44: /* e.g. "[:lo" */ switch ((unsigned char) c) { - case 't': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'w': state = S25; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S45: /* e.g. "[:cnt" */ + case S45: /* e.g. "[:cn" */ switch ((unsigned char) c) { - case 'r': state = S46; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 't': state = S46; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S46: /* e.g. "[:cntr" */ + case S46: /* e.g. "[:cnt" */ switch ((unsigned char) c) { - case 'l': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S47; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S47: /* e.g. "[:bl" */ + case S47: /* e.g. "[:cntr" */ switch ((unsigned char) c) { - case 'a': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'l': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S48: /* e.g. "[:bla" */ + case S48: /* e.g. "[:pr" */ switch ((unsigned char) c) { - case 'n': state = S49; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S52; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S49: /* e.g. "[:blan" */ + case S49: /* e.g. "[:pu" */ switch ((unsigned char) c) { - case 'k': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S50; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S50: /* e.g. "[:wo" */ + case S50: /* e.g. "[:pun" */ switch ((unsigned char) c) { - case 'r': state = S51; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'c': state = S51; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S51: /* e.g. "[:wor" */ + case S51: /* e.g. "[:digi" */ switch ((unsigned char) c) { - case 'd': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 't': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S52: /* e.g. "[:up" */ + case S52: /* e.g. "[:pri" */ switch ((unsigned char) c) { - case 'p': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S51; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S53: /* e.g. "[:di" */ switch ((unsigned char) c) { case 'g': state = S54; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S54: /* e.g. "[:dig" */ switch ((unsigned char) c) { - case 'i': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S51; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S55: /* e.g. "\\Q" */ - lx_pcre_ungetc(lx, c); return lx->z = z2, lx->z(lx); + case S55: /* e.g. "\\c" */ + state = S72; break; - case S56: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z(lx); + case S56: /* e.g. "\\$" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ESC; case S57: /* e.g. "\\1" */ switch ((unsigned char) c) { @@ -866,13 +1014,29 @@ z3(struct lx_pcre_lx *lx) case '7': case '8': case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; - case S58: /* e.g. "\\x" */ + case S58: /* e.g. "\\Q" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z2, lx->z(lx); + + case S59: /* e.g. "\\E" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z(lx); + + case S60: /* e.g. "\\o" */ switch ((unsigned char) c) { - case '{': state = S69; break; + case '{': state = S70; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + } + break; + + case S61: /* e.g. "\\\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + + case S62: /* e.g. "\\x" */ + switch ((unsigned char) c) { + case '{': state = S66; break; case '0': case '1': case '2': @@ -894,12 +1058,12 @@ z3(struct lx_pcre_lx *lx) case 'c': case 'd': case 'e': - case 'f': state = S70; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + case 'f': state = S67; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S59: /* e.g. "\\0" */ + case S63: /* e.g. "\\0" */ switch ((unsigned char) c) { case '0': case '1': @@ -908,31 +1072,12 @@ z3(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': state = S68; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; + case '7': state = S64; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S60: /* e.g. "\\o" */ - switch ((unsigned char) c) { - case '{': state = S65; break; - default: lx_pcre_ungetc(lx, c); return TOK_NOESC; - } - break; - - case S61: /* e.g. "\\c" */ - state = S64; break; - - case S62: /* e.g. "\\g" */ - lx_pcre_ungetc(lx, c); return TOK_NOESC; - - case S63: /* e.g. "\\a" */ - lx_pcre_ungetc(lx, c); return TOK_ESC; - - case S64: /* e.g. "\\ca" */ - lx_pcre_ungetc(lx, c); return TOK_CONTROL; - - case S65: /* e.g. "\\o{" */ + case S64: /* e.g. "\\00" */ switch ((unsigned char) c) { case '0': case '1': @@ -941,44 +1086,15 @@ z3(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': state = S66; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case '7': state = S65; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S66: /* e.g. "\\o{0" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': break; - case '}': state = S67; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; + case S65: /* e.g. "\\000" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; - case S67: /* e.g. "\\000" */ - lx_pcre_ungetc(lx, c); return TOK_OCT; - - case S68: /* e.g. "\\00" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': state = S67; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; - } - break; - - case S69: /* e.g. "\\x{" */ + case S66: /* e.g. "\\x{" */ switch ((unsigned char) c) { case '0': case '1': @@ -1001,12 +1117,14 @@ z3(struct lx_pcre_lx *lx) case 'c': case 'd': case 'e': - case 'f': state = S72; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'f': state = S69; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S70: /* e.g. "\\xa" */ + case S67: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -1029,17 +1147,17 @@ z3(struct lx_pcre_lx *lx) case 'c': case 'd': case 'e': - case 'f': state = S71; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + case 'f': state = S68; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S71: /* e.g. "\\xaa" */ - lx_pcre_ungetc(lx, c); return TOK_HEX; + case S68: /* e.g. "\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; - case S72: /* e.g. "\\x{a" */ + case S69: /* e.g. "\\x{0" */ switch ((unsigned char) c) { - case '}': state = S71; break; + case '}': state = S68; break; case '0': case '1': case '2': @@ -1062,17 +1180,84 @@ z3(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S70: /* e.g. "\\o{" */ + switch ((unsigned char) c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': state = S71; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; + case S71: /* e.g. "\\o{0" */ + switch ((unsigned char) c) { + case '}': state = S65; break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S72: /* e.g. "\\c\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CONTROL; + default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_INVALID; + case S2: return TOK_CHAR; + case S3: return TOK_CHAR; + case S4: return TOK_RANGE; + case S5: return lx->z = z7, TOK_CLOSEGROUP; + case S6: return lx->z = z7, TOK_CLOSEGROUPRANGE; + case S23: return TOK_NAMED__CLASS; + case S55: return TOK_NOESC; + case S56: return TOK_ESC; + case S57: return TOK_UNSUPPORTED; + case S58: return lx->z = z2, lx->z(lx); + case S59: return TOK_EOF; + case S60: return TOK_NOESC; + case S61: return TOK_NOESC; + case S62: return TOK_HEX; + case S63: return TOK_OCT; + case S64: return TOK_OCT; + case S65: return TOK_OCT; + case S67: return TOK_HEX; + case S68: return TOK_HEX; + case S72: return TOK_CONTROL; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { - case S55: - case S56: + case S58: + case S59: break; default: @@ -1084,64 +1269,41 @@ z3(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_INVALID; - case S2: return TOK_CHAR; - case S3: return TOK_CHAR; - case S4: return TOK_RANGE; - case S5: return TOK_CLOSEGROUP; - case S6: return TOK_CLOSEGROUPRANGE; - case S24: return TOK_NAMED__CLASS; - case S55: return TOK_EOF; - case S56: return TOK_EOF; - case S57: return TOK_UNSUPPORTED; - case S58: return TOK_HEX; - case S59: return TOK_OCT; - case S60: return TOK_NOESC; - case S61: return TOK_NOESC; - case S62: return TOK_NOESC; - case S63: return TOK_ESC; - case S64: return TOK_CONTROL; - case S67: return TOK_OCT; - case S68: return TOK_OCT; - case S70: return TOK_HEX; - case S71: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z4(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '+': case 'R': state = S1; break; @@ -1184,12 +1346,14 @@ z4(struct lx_pcre_lx *lx) case '-': state = S8; break; case ')': state = S9; break; case ':': state = S10; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S1: /* e.g. "R" */ - lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + case S1: /* e.g. "+" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; case S2: /* e.g. "0" */ switch ((unsigned char) c) { @@ -1203,53 +1367,45 @@ z4(struct lx_pcre_lx *lx) case '7': case '8': case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S3: /* e.g. "n" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__IGNORE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__IGNORE; case S4: /* e.g. "x" */ switch ((unsigned char) c) { case 'x': state = S7; break; - default: lx_pcre_ungetc(lx, c); return TOK_FLAG__EXTENDED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__EXTENDED; } break; case S5: /* e.g. "s" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__SINGLE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__SINGLE; case S6: /* e.g. "i" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__INSENSITIVE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__INSENSITIVE; case S7: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__UNKNOWN; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__UNKNOWN; case S8: /* e.g. "-" */ - lx_pcre_ungetc(lx, c); return TOK_NEGATE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NEGATE; case S9: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSE; case S10: /* e.g. ":" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_SUB; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_SUB; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_UNSUPPORTED; case S2: return TOK_UNSUPPORTED; case S3: return TOK_FLAG__IGNORE; @@ -1258,38 +1414,52 @@ z4(struct lx_pcre_lx *lx) case S6: return TOK_FLAG__INSENSITIVE; case S7: return TOK_FLAG__UNKNOWN; case S8: return TOK_NEGATE; - case S9: return TOK_CLOSE; - case S10: return TOK_SUB; - default: errno = EINVAL; return TOK_ERROR; + case S9: return lx->z = z7, TOK_CLOSE; + case S10: return lx->z = z7, TOK_SUB; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z5(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '(': state = S2; break; case ')': state = S3; break; @@ -1297,23 +1467,34 @@ z5(struct lx_pcre_lx *lx) } break; - case S1: /* e.g. "a" */ + case S1: /* e.g. "\\x00" */ switch ((unsigned char) c) { case '(': - case ')': lx_pcre_ungetc(lx, c); return lx->z(lx); + case ')': lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z(lx); default: break; } break; case S2: /* e.g. "(" */ - lx_pcre_ungetc(lx, c); return TOK_INVALID__COMMENT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_INVALID__COMMENT; case S3: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return TOK_INVALID__COMMENT; + case S3: return lx->z = z7, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S1: @@ -1329,24 +1510,30 @@ z5(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_INVALID__COMMENT; - case S3: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z6(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + lx->start = lx->end; + + void *getc_opaque = (void *)lx; enum { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, @@ -1355,26 +1542,15 @@ z6(struct lx_pcre_lx *lx) S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, S50, S51, S52, S53, S54, S55, S56, S57, S58, S59, S60, S61, S62, S63, S64, S65, S66, S67, S68, S69, - S70, S71, S72, S73, S74, S75, S76, S77, S78, NONE + S70, S71, S72, S73, S74, S75, S76, S77, S78 } state; - assert(lx != NULL); - - if (lx->clear != NULL) { - lx->clear(lx->buf_opaque); - } - - state = NONE; - - lx->start = lx->end; - - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case ':': state = S1; break; case 'L': state = S2; break; @@ -1390,13 +1566,17 @@ z6(struct lx_pcre_lx *lx) case 'n': state = S12; break; case 'F': state = S13; break; case ')': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S1: /* e.g. ":" */ switch ((unsigned char) c) { - case ')': lx->lgetc = NULL; return TOK_UNKNOWN; + case ')': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S78; break; } break; @@ -1404,7 +1584,9 @@ z6(struct lx_pcre_lx *lx) case S2: /* e.g. "L" */ switch ((unsigned char) c) { case 'F': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1412,35 +1594,45 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'l': state = S18; break; case 'o': state = S76; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S4: /* e.g. "M" */ switch ((unsigned char) c) { case 'A': state = S74; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S5: /* e.g. "T" */ switch ((unsigned char) c) { case 'H': state = S72; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S6: /* e.g. "S" */ switch ((unsigned char) c) { case 'K': state = S70; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S7: /* e.g. "P" */ switch ((unsigned char) c) { case 'R': state = S67; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1448,21 +1640,27 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'C': state = S60; break; case 'N': state = S61; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S9: /* e.g. "a" */ switch ((unsigned char) c) { case 't': state = S56; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S10: /* e.g. "N" */ switch ((unsigned char) c) { case 'O': state = S46; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1470,7 +1668,9 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'R': state = S41; break; case 'O': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1478,7 +1678,9 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'l': state = S18; break; case 'e': state = S19; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1486,31 +1688,35 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case ':': state = S1; break; case 'A': state = S15; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S14: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, lx->z(lx); case S15: /* e.g. "FA" */ switch ((unsigned char) c) { case 'I': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S16: /* e.g. "FAI" */ switch ((unsigned char) c) { case 'L': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S17: /* e.g. "FAIL" */ switch ((unsigned char) c) { case ':': state = S1; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; @@ -1518,84 +1724,108 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'a': case 'b': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S19: /* e.g. "ne" */ switch ((unsigned char) c) { case 'g': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S20: /* e.g. "neg" */ switch ((unsigned char) c) { case 'a': state = S21; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S21: /* e.g. "nega" */ switch ((unsigned char) c) { case 't': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S22: /* e.g. "negat" */ switch ((unsigned char) c) { case 'i': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S23: /* e.g. "negati" */ switch ((unsigned char) c) { case 'v': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S24: /* e.g. "negativ" */ switch ((unsigned char) c) { case 'e': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S25: /* e.g. "negative" */ switch ((unsigned char) c) { case '_': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S26: /* e.g. "negative_" */ switch ((unsigned char) c) { case 'l': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S27: /* e.g. "negative_l" */ switch ((unsigned char) c) { case 'o': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S28: /* e.g. "negative_lo" */ switch ((unsigned char) c) { case 'o': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S29: /* e.g. "negative_loo" */ switch ((unsigned char) c) { case 'k': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1603,338 +1833,428 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'b': state = S31; break; case 'a': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S31: /* e.g. "negative_lookb" */ switch ((unsigned char) c) { case 'e': state = S38; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S32: /* e.g. "negative_looka" */ switch ((unsigned char) c) { case 'h': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S33: /* e.g. "negative_lookah" */ switch ((unsigned char) c) { case 'e': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S34: /* e.g. "negative_lookahe" */ switch ((unsigned char) c) { case 'a': state = S35; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S35: /* e.g. "negative_lookahea" */ switch ((unsigned char) c) { case 'd': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S36: /* e.g. "nla" */ switch ((unsigned char) c) { case ':': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S37: /* e.g. "LF" */ - lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; case S38: /* e.g. "negative_lookbe" */ switch ((unsigned char) c) { case 'h': state = S39; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S39: /* e.g. "negative_lookbeh" */ switch ((unsigned char) c) { case 'i': state = S40; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S40: /* e.g. "negative_lookbehi" */ switch ((unsigned char) c) { case 'n': state = S35; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S41: /* e.g. "CR" */ switch ((unsigned char) c) { case 'L': state = S2; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S42: /* e.g. "CO" */ switch ((unsigned char) c) { case 'M': state = S43; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S43: /* e.g. "COM" */ switch ((unsigned char) c) { case 'M': state = S44; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S44: /* e.g. "COMM" */ switch ((unsigned char) c) { case 'I': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S45: /* e.g. "ACCEP" */ switch ((unsigned char) c) { case 'T': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S46: /* e.g. "NO" */ switch ((unsigned char) c) { case '_': state = S47; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S47: /* e.g. "NO_" */ switch ((unsigned char) c) { case 'S': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S48: /* e.g. "NO_S" */ switch ((unsigned char) c) { case 'T': state = S49; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S49: /* e.g. "NO_ST" */ switch ((unsigned char) c) { case 'A': state = S50; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S50: /* e.g. "NO_STA" */ switch ((unsigned char) c) { case 'R': state = S51; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S51: /* e.g. "NO_STAR" */ switch ((unsigned char) c) { case 'T': state = S52; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S52: /* e.g. "NO_START" */ switch ((unsigned char) c) { case '_': state = S53; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S53: /* e.g. "NO_START_" */ switch ((unsigned char) c) { case 'O': state = S54; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S54: /* e.g. "NO_START_O" */ switch ((unsigned char) c) { case 'P': state = S55; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S55: /* e.g. "NO_START_OP" */ switch ((unsigned char) c) { case 'T': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S56: /* e.g. "at" */ switch ((unsigned char) c) { case 'o': state = S57; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S57: /* e.g. "ato" */ switch ((unsigned char) c) { case 'm': state = S58; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S58: /* e.g. "atom" */ switch ((unsigned char) c) { case 'i': state = S59; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S59: /* e.g. "atomi" */ switch ((unsigned char) c) { case 'c': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S60: /* e.g. "AC" */ switch ((unsigned char) c) { case 'C': state = S65; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S61: /* e.g. "AN" */ switch ((unsigned char) c) { case 'Y': state = S62; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S62: /* e.g. "ANY" */ switch ((unsigned char) c) { case 'C': state = S63; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S63: /* e.g. "ANYC" */ switch ((unsigned char) c) { case 'R': state = S64; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S64: /* e.g. "ANYCR" */ switch ((unsigned char) c) { case 'L': state = S2; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S65: /* e.g. "ACC" */ switch ((unsigned char) c) { case 'E': state = S66; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S66: /* e.g. "ACCE" */ switch ((unsigned char) c) { case 'P': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S67: /* e.g. "PR" */ switch ((unsigned char) c) { case 'U': state = S68; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S68: /* e.g. "PRU" */ switch ((unsigned char) c) { case 'N': state = S69; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S69: /* e.g. "PRUN" */ switch ((unsigned char) c) { case 'E': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S70: /* e.g. "SK" */ switch ((unsigned char) c) { case 'I': state = S71; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S71: /* e.g. "SKI" */ switch ((unsigned char) c) { case 'P': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S72: /* e.g. "TH" */ switch ((unsigned char) c) { case 'E': state = S73; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S73: /* e.g. "THE" */ switch ((unsigned char) c) { case 'N': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S74: /* e.g. "MA" */ switch ((unsigned char) c) { case 'R': state = S75; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S75: /* e.g. "MAR" */ switch ((unsigned char) c) { case 'K': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S76: /* e.g. "po" */ switch ((unsigned char) c) { case 's': state = S77; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S77: /* e.g. "pos" */ switch ((unsigned char) c) { case 'i': state = S21; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S78: /* e.g. ":a" */ + case S78: /* e.g. ":\\x00" */ switch ((unsigned char) c) { - case ')': lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + case ')': lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; default: break; } break; @@ -1942,6 +2262,21 @@ z6(struct lx_pcre_lx *lx) default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S13: return TOK_UNSUPPORTED; + case S14: return lx->z = z7, lx->z(lx); + case S17: return TOK_UNSUPPORTED; + case S37: return TOK_UNSUPPORTED; + case S41: return TOK_UNSUPPORTED; + case S62: return TOK_UNSUPPORTED; + case S78: return TOK_UNSUPPORTED; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S14: @@ -1956,53 +2291,44 @@ z6(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S13: return TOK_UNSUPPORTED; - case S14: return TOK_EOF; - case S17: return TOK_UNSUPPORTED; - case S37: return TOK_UNSUPPORTED; - case S41: return TOK_UNSUPPORTED; - case S62: return TOK_UNSUPPORTED; - case S78: return TOK_UNSUPPORTED; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z7(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, - S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, - S40, S41, S42, S43, S44, S45, S46, S47, S48, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, + S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, + S40, S41, S42, S43, S44, S45, S46, S47, S48 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S2; break; case '\n': @@ -2023,13 +2349,15 @@ z7(struct lx_pcre_lx *lx) case '$': state = S14; break; case '^': state = S15; break; case ')': state = S16; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "\\" */ switch ((unsigned char) c) { @@ -2042,24 +2370,7 @@ z7(struct lx_pcre_lx *lx) case 'X': case 'b': case 'g': - case 'k': state = S20; break; - case 'Q': state = S28; break; - case 'o': state = S29; break; - case 'c': state = S30; break; - case 'x': state = S32; break; - case '0': state = S33; break; - case 'R': state = S34; break; - case 'D': - case 'H': - case 'N': - case 'S': - case 'V': - case 'W': - case 'd': - case 'h': - case 's': - case 'v': - case 'w': state = S35; break; + case 'k': state = S21; break; case '$': case '(': case ')': @@ -2077,8 +2388,9 @@ z7(struct lx_pcre_lx *lx) case 'r': case 't': case '{': - case '|': state = S36; break; - case 'E': state = S37; break; + case '|': state = S28; break; + case 'E': state = S29; break; + case 'z': state = S30; break; case '1': case '2': case '3': @@ -2087,29 +2399,45 @@ z7(struct lx_pcre_lx *lx) case '6': case '7': case '8': - case '9': state = S38; break; - case 'z': state = S39; break; - default: state = S31; break; + case '9': state = S31; break; + case 'Q': state = S32; break; + case 'c': state = S33; break; + case 'o': state = S34; break; + case 'x': state = S36; break; + case 'D': + case 'H': + case 'N': + case 'S': + case 'V': + case 'W': + case 'd': + case 'h': + case 's': + case 'v': + case 'w': state = S37; break; + case 'R': state = S38; break; + case '0': state = S39; break; + default: state = S35; break; } break; case S3: /* e.g. "\\x0a" */ - lx_pcre_ungetc(lx, c); return TOK_NEWLINE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NEWLINE; case S4: /* e.g. "\\x09" */ - lx_pcre_ungetc(lx, c); return TOK_WHITESPACE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_WHITESPACE; case S5: /* e.g. "#" */ - lx_pcre_ungetc(lx, c); return TOK_MAYBE_COMMENT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_MAYBE_COMMENT; case S6: /* e.g. "{" */ - lx_pcre_ungetc(lx, c); return lx->z = z1, TOK_OPENCOUNT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENCOUNT; case S7: /* e.g. "[" */ switch ((unsigned char) c) { case '^': state = S25; break; case ']': state = S26; break; - default: lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUP; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUP; } break; @@ -2117,66 +2445,56 @@ z7(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case '?': state = S17; break; case '*': state = S18; break; - default: lx_pcre_ungetc(lx, c); return TOK_OPEN; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OPEN; } break; case S9: /* e.g. "|" */ - lx_pcre_ungetc(lx, c); return TOK_ALT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ALT; case S10: /* e.g. "." */ - lx_pcre_ungetc(lx, c); return TOK_ANY; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ANY; case S11: /* e.g. "+" */ - lx_pcre_ungetc(lx, c); return TOK_PLUS; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_PLUS; case S12: /* e.g. "*" */ - lx_pcre_ungetc(lx, c); return TOK_STAR; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_STAR; - case S13: /* e.g. "?" */ - lx_pcre_ungetc(lx, c); return TOK_OPT; + case S13: /* e.g. "\077" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OPT; case S14: /* e.g. "$" */ - lx_pcre_ungetc(lx, c); return TOK_END__NL; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_END__NL; case S15: /* e.g. "^" */ - lx_pcre_ungetc(lx, c); return TOK_START; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_START; case S16: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return TOK_CLOSE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CLOSE; - case S17: /* e.g. "(?" */ + case S17: /* e.g. "(\077" */ switch ((unsigned char) c) { case '#': state = S19; break; + case '<': state = S20; break; case '!': case '&': - case '=': state = S20; break; - case 'P': state = S21; break; - case '<': state = S22; break; - default: lx_pcre_ungetc(lx, c); return lx->z = z4, TOK_FLAGS; + case '=': state = S21; break; + case 'P': state = S22; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z4, TOK_FLAGS; } break; case S18: /* e.g. "(*" */ - lx_pcre_ungetc(lx, c); return lx->z = z6, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z6, lx->z(lx); - case S19: /* e.g. "(?#" */ - lx_pcre_ungetc(lx, c); return lx->z = z5, lx->z(lx); + case S19: /* e.g. "(\077#" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z5, lx->z(lx); - case S20: /* e.g. "\\b" */ - lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; - - case S21: /* e.g. "(?P" */ - switch ((unsigned char) c) { - case '>': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; - - case S22: /* e.g. "(?<" */ + case S20: /* e.g. "(\077<" */ switch ((unsigned char) c) { case '!': - case '=': state = S20; break; + case '=': state = S21; break; case 'A': case 'B': case 'C': @@ -2230,11 +2548,25 @@ z7(struct lx_pcre_lx *lx) case 'x': case 'y': case 'z': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S21: /* e.g. "\\B" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; + + case S22: /* e.g. "(\077P" */ + switch ((unsigned char) c) { + case '>': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S23: /* e.g. "(?': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S24: /* e.g. "(?" */ - lx_pcre_ungetc(lx, c); return TOK_OPENCAPTURE; + case S24: /* e.g. "(\077" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OPENCAPTURE; case S25: /* e.g. "[^" */ switch ((unsigned char) c) { case ']': state = S27; break; - default: lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUPINV; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUPINV; } break; case S26: /* e.g. "[]" */ - lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUPCB; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUPCB; case S27: /* e.g. "[^]" */ - lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUPINVCB; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUPINVCB; - case S28: /* e.g. "\\Q" */ - lx_pcre_ungetc(lx, c); return lx->z = z0, lx->z(lx); + case S28: /* e.g. "\\$" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ESC; - case S29: /* e.g. "\\o" */ + case S29: /* e.g. "\\E" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z(lx); + + case S30: /* e.g. "\\z" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_END; + + case S31: /* e.g. "\\1" */ switch ((unsigned char) c) { - case '{': state = S47; break; - default: lx_pcre_ungetc(lx, c); return TOK_NOESC; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; - case S30: /* e.g. "\\c" */ - state = S46; break; + case S32: /* e.g. "\\Q" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z0, lx->z(lx); - case S31: /* e.g. "\\i" */ - lx_pcre_ungetc(lx, c); return TOK_NOESC; + case S33: /* e.g. "\\c" */ + state = S48; break; - case S32: /* e.g. "\\x" */ + case S34: /* e.g. "\\o" */ + switch ((unsigned char) c) { + case '{': state = S46; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + } + break; + + case S35: /* e.g. "\\\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + + case S36: /* e.g. "\\x" */ switch ((unsigned char) c) { case '{': state = S42; break; case '0': @@ -2361,37 +2720,17 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': state = S43; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S33: /* e.g. "\\0" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': state = S40; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; - } - break; - - case S34: /* e.g. "\\R" */ - lx_pcre_ungetc(lx, c); return TOK_EOL; - - case S35: /* e.g. "\\d" */ - lx_pcre_ungetc(lx, c); return TOK_NAMED__CLASS; - - case S36: /* e.g. "\\a" */ - lx_pcre_ungetc(lx, c); return TOK_ESC; + case S37: /* e.g. "\\D" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; - case S37: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z(lx); + case S38: /* e.g. "\\R" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_EOL; - case S38: /* e.g. "\\1" */ + case S39: /* e.g. "\\0" */ switch ((unsigned char) c) { case '0': case '1': @@ -2400,16 +2739,11 @@ z7(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': - case '8': - case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + case '7': state = S40; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S39: /* e.g. "\\z" */ - lx_pcre_ungetc(lx, c); return TOK_END; - case S40: /* e.g. "\\00" */ switch ((unsigned char) c) { case '0': @@ -2420,12 +2754,12 @@ z7(struct lx_pcre_lx *lx) case '5': case '6': case '7': state = S41; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; case S41: /* e.g. "\\000" */ - lx_pcre_ungetc(lx, c); return TOK_OCT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; case S42: /* e.g. "\\x{" */ switch ((unsigned char) c) { @@ -2451,11 +2785,13 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S43: /* e.g. "\\xa" */ + case S43: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -2479,14 +2815,14 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': state = S44; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S44: /* e.g. "\\xaa" */ - lx_pcre_ungetc(lx, c); return TOK_HEX; + case S44: /* e.g. "\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; - case S45: /* e.g. "\\x{a" */ + case S45: /* e.g. "\\x{0" */ switch ((unsigned char) c) { case '}': state = S44; break; case '0': @@ -2511,14 +2847,13 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S46: /* e.g. "\\ca" */ - lx_pcre_ungetc(lx, c); return TOK_CONTROL; - - case S47: /* e.g. "\\o{" */ + case S46: /* e.g. "\\o{" */ switch ((unsigned char) c) { case '0': case '1': @@ -2527,12 +2862,14 @@ z7(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case '7': state = S47; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S48: /* e.g. "\\o{0" */ + case S47: /* e.g. "\\o{0" */ switch ((unsigned char) c) { case '}': state = S41; break; case '0': @@ -2543,43 +2880,29 @@ z7(struct lx_pcre_lx *lx) case '5': case '6': case '7': break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - default: - ; /* unreached */ - } - - switch (state) { - case S18: - case S19: - case S28: - case S37: - break; + case S48: /* e.g. "\\c\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CONTROL; default: - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } - break; - + ; /* unreached */ } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_INVALID; case S3: return TOK_NEWLINE; case S4: return TOK_WHITESPACE; case S5: return TOK_MAYBE_COMMENT; - case S6: return TOK_OPENCOUNT; - case S7: return TOK_OPENGROUP; + case S6: return lx->z = z1, TOK_OPENCOUNT; + case S7: return lx->z = z3, TOK_OPENGROUP; case S8: return TOK_OPEN; case S9: return TOK_ALT; case S10: return TOK_ANY; @@ -2589,33 +2912,59 @@ z7(struct lx_pcre_lx *lx) case S14: return TOK_END__NL; case S15: return TOK_START; case S16: return TOK_CLOSE; - case S17: return TOK_FLAGS; - case S18: return TOK_EOF; - case S19: return TOK_EOF; - case S20: return TOK_UNSUPPORTED; + case S17: return lx->z = z4, TOK_FLAGS; + case S18: return lx->z = z6, lx->z(lx); + case S19: return lx->z = z5, lx->z(lx); + case S21: return TOK_UNSUPPORTED; case S24: return TOK_OPENCAPTURE; - case S25: return TOK_OPENGROUPINV; - case S26: return TOK_OPENGROUPCB; - case S27: return TOK_OPENGROUPINVCB; - case S28: return TOK_EOF; - case S29: return TOK_NOESC; - case S30: return TOK_NOESC; - case S31: return TOK_NOESC; - case S32: return TOK_HEX; - case S33: return TOK_OCT; - case S34: return TOK_EOL; - case S35: return TOK_NAMED__CLASS; - case S36: return TOK_ESC; - case S37: return TOK_EOF; - case S38: return TOK_UNSUPPORTED; - case S39: return TOK_END; + case S25: return lx->z = z3, TOK_OPENGROUPINV; + case S26: return lx->z = z3, TOK_OPENGROUPCB; + case S27: return lx->z = z3, TOK_OPENGROUPINVCB; + case S28: return TOK_ESC; + case S29: return TOK_EOF; + case S30: return TOK_END; + case S31: return TOK_UNSUPPORTED; + case S32: return lx->z = z0, lx->z(lx); + case S33: return TOK_NOESC; + case S34: return TOK_NOESC; + case S35: return TOK_NOESC; + case S36: return TOK_HEX; + case S37: return TOK_NAMED__CLASS; + case S38: return TOK_EOL; + case S39: return TOK_OCT; case S40: return TOK_OCT; case S41: return TOK_OCT; case S43: return TOK_HEX; case S44: return TOK_HEX; - case S46: return TOK_CONTROL; - default: errno = EINVAL; return TOK_ERROR; + case S48: return TOK_CONTROL; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + switch (state) { + case S18: + case S19: + case S29: + case S32: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + break; + + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -3092,6 +3441,7 @@ lx_pcre_init(struct lx_pcre_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_pcre_dynpop; } enum lx_pcre_token diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index 78c11d795..e59fee020 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 275 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -325,7 +325,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 666 "src/libre/parser.act" +#line 665 "src/libre/parser.act" ZIc = RE_EXTENDED; @@ -335,7 +335,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 801 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); @@ -356,7 +356,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 662 "src/libre/parser.act" +#line 661 "src/libre/parser.act" ZIc = RE_ICASE; @@ -366,7 +366,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 801 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); @@ -381,7 +381,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 670 "src/libre/parser.act" +#line 669 "src/libre/parser.act" ZIc = RE_SINGLE; @@ -391,7 +391,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 801 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); @@ -406,7 +406,7 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 743 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; @@ -451,7 +451,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -474,7 +474,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 448 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -522,7 +522,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -538,7 +538,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 391 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -574,7 +574,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 535 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -632,7 +632,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 492 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -685,7 +685,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 429 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -707,7 +707,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -726,7 +726,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags /* END OF INLINE: 155 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -762,7 +762,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; @@ -775,7 +775,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -822,7 +822,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -851,7 +851,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -874,7 +874,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI290)); if ((ZInode) == NULL) { @@ -894,7 +894,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 845 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI290); @@ -910,7 +910,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 773 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI291)); mark(&act_state->rangeend, &(ZIend)); @@ -920,7 +920,7 @@ p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -980,7 +980,7 @@ p_168(flags flags, lex_state lex_state, act_state act_state, err err) case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI169 = '-'; ZI170 = lex_state->lx.start; @@ -1004,7 +1004,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 722 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; @@ -1043,7 +1043,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1069,7 +1069,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: CONTROL */ { -#line 448 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1124,7 +1124,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: ESC */ { -#line 391 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1163,7 +1163,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: HEX */ { -#line 535 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1224,7 +1224,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: NOESC */ { -#line 417 "src/libre/parser.act" +#line 412 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1251,7 +1251,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: OCT */ { -#line 492 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1307,7 +1307,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 429 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1329,7 +1329,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -1348,7 +1348,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err /* END OF INLINE: 111 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1381,7 +1381,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1413,7 +1413,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CONTROL */ { -#line 448 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1461,7 +1461,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -1486,7 +1486,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: ESC */ { -#line 391 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1531,7 +1531,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: HEX */ { -#line 535 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1598,7 +1598,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 648 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZI290 = DIALECT_CLASS(lex_state->buf.a); if (ZI290 == NULL) { @@ -1631,7 +1631,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: NOESC */ { -#line 417 "src/libre/parser.act" +#line 412 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1651,7 +1651,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1671,7 +1671,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: OCT */ { -#line 492 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1733,7 +1733,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 429 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1755,7 +1755,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -1801,7 +1801,7 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 687 "src/libre/parser.act" +#line 683 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; @@ -1835,7 +1835,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 648 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1859,7 +1859,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 845 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); @@ -1899,7 +1899,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 319 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI181 = lex_state->lx.end; @@ -1913,7 +1913,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1940,7 +1940,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 335 "src/libre/parser.act" +#line 334 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI200 = lex_state->lx.end; @@ -1954,7 +1954,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1967,7 +1967,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 886 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -1981,7 +1981,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -2003,7 +2003,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 327 "src/libre/parser.act" +#line 326 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI192 = lex_state->lx.end; @@ -2017,7 +2017,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2030,7 +2030,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -2087,7 +2087,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 343 "src/libre/parser.act" +#line 342 "src/libre/parser.act" ZIstart = lex_state->lx.start; ZI207 = lex_state->lx.end; @@ -2101,7 +2101,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2114,7 +2114,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -2157,7 +2157,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 886 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; @@ -2171,7 +2171,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; @@ -2203,7 +2203,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 351 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI214 = ']'; ZI215 = lex_state->lx.start; @@ -2219,7 +2219,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 768 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -2238,7 +2238,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 361 "src/libre/parser.act" +#line 360 "src/libre/parser.act" ZIcrange = '-'; ZI217 = lex_state->lx.start; @@ -2254,7 +2254,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { @@ -2266,7 +2266,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; @@ -2277,7 +2277,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 768 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); @@ -2295,7 +2295,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 729 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -2338,7 +2338,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI163 = lex_state->lx.start; @@ -2354,7 +2354,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (ZIc); @@ -2401,7 +2401,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI314)); if ((ZInode) == NULL) { @@ -2421,7 +2421,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI314); @@ -2437,7 +2437,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 773 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI315)); mark(&act_state->rangeend, &(ZIend)); @@ -2447,7 +2447,7 @@ p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2522,7 +2522,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* BEGINNING OF ACTION: count-one */ { -#line 821 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -2531,7 +2531,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: count-one */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2578,7 +2578,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI256 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2592,7 +2592,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); @@ -2602,7 +2602,7 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2656,7 +2656,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2677,7 +2677,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 715 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -2689,7 +2689,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2722,7 +2722,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2736,7 +2736,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); @@ -2746,7 +2746,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ { -#line 805 "src/libre/parser.act" +#line 804 "src/libre/parser.act" (ZIn) = AST_COUNT_UNBOUNDED; @@ -2755,7 +2755,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2783,7 +2783,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* BEGINNING OF EXTRACT: COUNT */ { -#line 636 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -2811,7 +2811,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI259 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -2829,7 +2829,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); @@ -2839,7 +2839,7 @@ p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -2885,7 +2885,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; @@ -2906,7 +2906,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { @@ -2929,7 +2929,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (ZIc); @@ -2939,7 +2939,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI197 = '-'; ZI198 = lex_state->lx.start; @@ -2960,7 +2960,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -2999,7 +2999,7 @@ p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF INLINE: 196 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; @@ -3044,7 +3044,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 797 "src/libre/parser.act" +#line 796 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; @@ -3053,7 +3053,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 797 "src/libre/parser.act" +#line 796 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; @@ -3110,7 +3110,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3124,7 +3124,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -3144,7 +3144,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 919 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIflags) = *flags; @@ -3153,7 +3153,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 931 "src/libre/parser.act" +#line 926 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3172,7 +3172,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 923 "src/libre/parser.act" +#line 922 "src/libre/parser.act" *flags = (ZIflags); @@ -3197,7 +3197,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 750 "src/libre/parser.act" +#line 746 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; @@ -3209,7 +3209,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -3250,7 +3250,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3275,7 +3275,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -3292,7 +3292,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 764 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; @@ -3384,7 +3384,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 648 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3408,7 +3408,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { @@ -3439,7 +3439,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { @@ -3463,7 +3463,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); @@ -3473,7 +3473,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI210 = '-'; ZI211 = lex_state->lx.start; @@ -3494,7 +3494,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); @@ -3504,7 +3504,7 @@ p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -3566,7 +3566,7 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; @@ -3585,7 +3585,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -3597,7 +3597,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 715 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -3629,7 +3629,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 371 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI318 = lex_state->lx.start; ZI319 = lex_state->lx.end; @@ -3645,7 +3645,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 636 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -3685,7 +3685,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 817 "src/libre/parser.act" +#line 816 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); @@ -3699,7 +3699,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 813 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); @@ -3713,7 +3713,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 809 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -3732,7 +3732,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 701 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; @@ -3744,7 +3744,7 @@ ZL1:; /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 821 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -3774,7 +3774,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -3788,7 +3788,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -3814,7 +3814,7 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 757 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -3850,7 +3850,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -3860,7 +3860,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -3877,7 +3877,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 943 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3894,7 +3894,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 950 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3922,7 +3922,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 789 "src/libre/parser.act" +#line 787 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; @@ -3932,7 +3932,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { @@ -3944,7 +3944,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { @@ -3956,7 +3956,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 890 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIcr) = '\r'; @@ -3965,7 +3965,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { @@ -3977,7 +3977,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 894 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZInl) = '\n'; @@ -3986,7 +3986,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { @@ -3998,7 +3998,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; @@ -4009,7 +4009,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; @@ -4020,7 +4020,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -4032,7 +4032,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; @@ -4043,7 +4043,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; @@ -4063,7 +4063,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 919 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIflags) = *flags; @@ -4072,7 +4072,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -4086,7 +4086,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 923 "src/libre/parser.act" +#line 922 "src/libre/parser.act" *flags = (ZIflags); @@ -4095,7 +4095,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { @@ -4119,7 +4119,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 936 "src/libre/parser.act" +#line 935 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { @@ -4178,7 +4178,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 708 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -4190,7 +4190,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -4223,7 +4223,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -4244,7 +4244,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -4287,7 +4287,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -4299,7 +4299,7 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; @@ -4319,7 +4319,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1207 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index c0cfbabe3..84ef34223 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 292 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1209 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/sql/lexer.c b/src/libre/dialect/sql/lexer.c index 6c35bf800..87459e786 100644 --- a/src/libre/dialect/sql/lexer.c +++ b/src/libre/dialect/sql/lexer.c @@ -12,11 +12,31 @@ static enum lx_sql_token z0(struct lx_sql_lx *lx); static enum lx_sql_token z1(struct lx_sql_lx *lx); static enum lx_sql_token z2(struct lx_sql_lx *lx); +static int +lx_sql_advance_end(struct lx_sql_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_sql_lx *lx) +lx_sql_getc(struct lx_sql_lx *lx) { int c; @@ -32,18 +52,19 @@ lx_getc(struct lx_sql_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_sql_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_sql_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_sql_getc((struct lx_sql_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -52,10 +73,7 @@ lx_sql_ungetc(struct lx_sql_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -107,6 +125,17 @@ lx_sql_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_sql_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_sql_dynclear(void *buf_opaque) { @@ -146,29 +175,28 @@ lx_sql_dynfree(void *buf_opaque) static enum lx_sql_token z0(struct lx_sql_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case ',': state = S1; break; case '0': @@ -182,12 +210,14 @@ z0(struct lx_sql_lx *lx) case '8': case '9': state = S2; break; case '}': state = S3; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S1: /* e.g. "," */ - lx_sql_ungetc(lx, c); return TOK_SEP; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_SEP; case S2: /* e.g. "0" */ switch ((unsigned char) c) { @@ -201,64 +231,70 @@ z0(struct lx_sql_lx *lx) case '7': case '8': case '9': break; - default: lx_sql_ungetc(lx, c); return TOK_COUNT; + default: lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_COUNT; } break; case S3: /* e.g. "}" */ - lx_sql_ungetc(lx, c); return lx->z = z2, TOK_CLOSECOUNT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSECOUNT; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_SEP; + case S2: return TOK_COUNT; + case S3: return lx->z = z2, TOK_CLOSECOUNT; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_SEP; - case S2: return TOK_COUNT; - case S3: return TOK_CLOSECOUNT; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_sql_token z1(struct lx_sql_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, - S30, S31, S32, S33, S34, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, + S30, S31, S32, S33, S34 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '[': state = S1; break; case '-': state = S3; break; @@ -271,21 +307,21 @@ z1(struct lx_sql_lx *lx) case S1: /* e.g. "[" */ switch ((unsigned char) c) { case ':': state = S6; break; - default: lx_sql_ungetc(lx, c); return TOK_CHAR; + default: lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CHAR; } break; case S2: /* e.g. "\\x00" */ - lx_sql_ungetc(lx, c); return TOK_CHAR; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "-" */ - lx_sql_ungetc(lx, c); return TOK_RANGE; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_RANGE; case S4: /* e.g. "^" */ - lx_sql_ungetc(lx, c); return TOK_INVERT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_INVERT; case S5: /* e.g. "]" */ - lx_sql_ungetc(lx, c); return lx->z = z2, TOK_CLOSEGROUP; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSEGROUP; case S6: /* e.g. "[:" */ switch ((unsigned char) c) { @@ -295,94 +331,120 @@ z1(struct lx_sql_lx *lx) case 'A': state = S10; break; case 'L': state = S11; break; case 'U': state = S12; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S7: /* e.g. "[:S" */ switch ((unsigned char) c) { case 'P': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S8: /* e.g. "[:W" */ switch ((unsigned char) c) { case 'H': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S9: /* e.g. "[:D" */ switch ((unsigned char) c) { case 'I': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S10: /* e.g. "[:A" */ switch ((unsigned char) c) { case 'L': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S11: /* e.g. "[:L" */ switch ((unsigned char) c) { case 'O': state = S19; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S12: /* e.g. "[:U" */ switch ((unsigned char) c) { case 'P': state = S13; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S13: /* e.g. "[:UP" */ switch ((unsigned char) c) { case 'P': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S14: /* e.g. "[:LOW" */ switch ((unsigned char) c) { case 'E': state = S15; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S15: /* e.g. "[:LOWE" */ switch ((unsigned char) c) { case 'R': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S16: /* e.g. "[:ALPHA" */ switch ((unsigned char) c) { case ':': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S17: /* e.g. "[:ALPHA:" */ switch ((unsigned char) c) { case ']': state = S18; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S18: /* e.g. "[:ALPHA:]" */ - lx_sql_ungetc(lx, c); return TOK_NAMED__CLASS; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; case S19: /* e.g. "[:LO" */ switch ((unsigned char) c) { case 'W': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -390,160 +452,196 @@ z1(struct lx_sql_lx *lx) switch ((unsigned char) c) { case 'N': state = S21; break; case 'P': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S21: /* e.g. "[:ALN" */ switch ((unsigned char) c) { case 'U': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S22: /* e.g. "[:ALP" */ switch ((unsigned char) c) { case 'H': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S23: /* e.g. "[:ALPH" */ switch ((unsigned char) c) { case 'A': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S24: /* e.g. "[:ALNU" */ switch ((unsigned char) c) { case 'M': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S25: /* e.g. "[:DI" */ switch ((unsigned char) c) { case 'G': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S26: /* e.g. "[:DIG" */ switch ((unsigned char) c) { case 'I': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S27: /* e.g. "[:DIGI" */ switch ((unsigned char) c) { case 'T': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S28: /* e.g. "[:WH" */ switch ((unsigned char) c) { case 'I': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S29: /* e.g. "[:WHI" */ switch ((unsigned char) c) { case 'T': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S30: /* e.g. "[:WHIT" */ switch ((unsigned char) c) { case 'E': state = S31; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S31: /* e.g. "[:WHITE" */ switch ((unsigned char) c) { case 'S': state = S7; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S32: /* e.g. "[:SP" */ switch ((unsigned char) c) { case 'A': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S33: /* e.g. "[:SPA" */ switch ((unsigned char) c) { case 'C': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S34: /* e.g. "[:SPAC" */ switch ((unsigned char) c) { case 'E': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; case S3: return TOK_RANGE; case S4: return TOK_INVERT; - case S5: return TOK_CLOSEGROUP; + case S5: return lx->z = z2, TOK_CLOSEGROUP; case S18: return TOK_NAMED__CLASS; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_sql_token z2(struct lx_sql_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '{': state = S2; break; case '[': state = S3; break; @@ -560,56 +658,48 @@ z2(struct lx_sql_lx *lx) break; case S1: /* e.g. "\\x00" */ - lx_sql_ungetc(lx, c); return TOK_CHAR; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "{" */ - lx_sql_ungetc(lx, c); return lx->z = z0, TOK_OPENCOUNT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z0, TOK_OPENCOUNT; case S3: /* e.g. "[" */ - lx_sql_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUP; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUP; case S4: /* e.g. "|" */ - lx_sql_ungetc(lx, c); return TOK_ALT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_ALT; case S5: /* e.g. "+" */ - lx_sql_ungetc(lx, c); return TOK_PLUS; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_PLUS; case S6: /* e.g. "*" */ - lx_sql_ungetc(lx, c); return TOK_STAR; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_STAR; - case S7: /* e.g. "?" */ - lx_sql_ungetc(lx, c); return TOK_OPT; + case S7: /* e.g. "\077" */ + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_OPT; case S8: /* e.g. ")" */ - lx_sql_ungetc(lx, c); return TOK_CLOSESUB; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CLOSESUB; case S9: /* e.g. "(" */ - lx_sql_ungetc(lx, c); return TOK_OPENSUB; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_OPENSUB; case S10: /* e.g. "%" */ - lx_sql_ungetc(lx, c); return TOK_MANY; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_MANY; case S11: /* e.g. "_" */ - lx_sql_ungetc(lx, c); return TOK_ANY; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_ANY; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; - case S2: return TOK_OPENCOUNT; - case S3: return TOK_OPENGROUP; + case S2: return lx->z = z0, TOK_OPENCOUNT; + case S3: return lx->z = z1, TOK_OPENGROUP; case S4: return TOK_ALT; case S5: return TOK_PLUS; case S6: return TOK_STAR; @@ -618,8 +708,23 @@ z2(struct lx_sql_lx *lx) case S9: return TOK_OPENSUB; case S10: return TOK_MANY; case S11: return TOK_ANY; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -747,6 +852,7 @@ lx_sql_init(struct lx_sql_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_sql_dynpop; } enum lx_sql_token diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index d380c5b7d..e7a4c2e75 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 275 "src/libre/parser.act" +#line 22 "src/libre/parser.act" #include @@ -311,7 +311,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: INVERT */ { -#line 303 "src/libre/parser.act" +#line 302 "src/libre/parser.act" ZI203 = '^'; @@ -337,7 +337,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI114 = lex_state->lx.start; @@ -353,7 +353,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -365,7 +365,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -401,7 +401,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -415,7 +415,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ } /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { @@ -441,7 +441,7 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 757 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; @@ -484,7 +484,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; @@ -499,7 +499,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 694 "src/libre/parser.act" +#line 690 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; @@ -522,7 +522,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; goto ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms; /* END OF INLINE: expr::character-class::list-of-class-terms */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -560,7 +560,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1041 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; @@ -579,7 +579,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL2_expr_C_Clist_Hof_Hpieces; /* END OF INLINE: expr::list-of-pieces */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -606,7 +606,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -670,7 +670,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 319 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -688,7 +688,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { @@ -713,7 +713,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 351 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI154 = ']'; ZIclose__start = lex_state->lx.start; @@ -729,7 +729,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 768 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); @@ -748,7 +748,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* BEGINNING OF EXTRACT: INVERT */ { -#line 303 "src/libre/parser.act" +#line 302 "src/libre/parser.act" ZI158 = '^'; @@ -760,7 +760,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { @@ -789,7 +789,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 351 "src/libre/parser.act" +#line 350 "src/libre/parser.act" ZI163 = ']'; ZIclose__start = lex_state->lx.start; @@ -809,7 +809,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 768 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); @@ -823,7 +823,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 729 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -839,7 +839,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF INLINE: 162 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 960 "src/libre/parser.act" +#line 959 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { @@ -862,7 +862,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 729 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; @@ -874,7 +874,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -917,7 +917,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -954,7 +954,7 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 868 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -975,7 +975,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 715 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -987,7 +987,7 @@ ZL1:; /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -1019,7 +1019,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZI117 = lex_state->lx.start; @@ -1035,7 +1035,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { @@ -1047,7 +1047,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; @@ -1058,7 +1058,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1105,7 +1105,7 @@ p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 995 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1167,7 +1167,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI205)); if ((ZInode) == NULL) { @@ -1192,7 +1192,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; (ZIa).u.literal.c = (unsigned char) (*ZI205); @@ -1202,7 +1202,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 309 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZI136 = '-'; ZI137 = lex_state->lx.start; @@ -1220,7 +1220,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1243,7 +1243,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 840 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; (ZIz).u.literal.c = (unsigned char) (ZIcz); @@ -1253,7 +1253,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 773 "src/libre/parser.act" +#line 772 "src/libre/parser.act" mark(&act_state->rangestart, &(*ZI206)); mark(&act_state->rangeend, &(ZIend)); @@ -1263,7 +1263,7 @@ p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1007 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" unsigned char lower, upper; @@ -1321,7 +1321,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI176 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1335,7 +1335,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); @@ -1345,7 +1345,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1376,7 +1376,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 636 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -1408,7 +1408,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 379 "src/libre/parser.act" +#line 378 "src/libre/parser.act" ZI179 = lex_state->lx.start; ZIend = lex_state->lx.end; @@ -1426,7 +1426,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 778 "src/libre/parser.act" +#line 777 "src/libre/parser.act" mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); @@ -1436,7 +1436,7 @@ p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 825 "src/libre/parser.act" +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; @@ -1486,7 +1486,7 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1047 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; @@ -1505,7 +1505,7 @@ ZL2_expr_C_Clist_Hof_Halts:; goto ZL2_expr_C_Clist_Hof_Halts; /* END OF INLINE: expr::list-of-alts */ } - /*UNREACHED*/ + /* UNREACHED */ default: break; } @@ -1517,7 +1517,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 715 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; @@ -1549,7 +1549,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 371 "src/libre/parser.act" +#line 370 "src/libre/parser.act" ZI209 = lex_state->lx.start; ZI210 = lex_state->lx.end; @@ -1565,7 +1565,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 636 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -1605,7 +1605,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 817 "src/libre/parser.act" +#line 816 "src/libre/parser.act" (ZIc) = ast_make_count(0, 1); @@ -1619,7 +1619,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 813 "src/libre/parser.act" +#line 812 "src/libre/parser.act" (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); @@ -1633,7 +1633,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 809 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -1646,7 +1646,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 821 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -1663,7 +1663,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 701 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; @@ -1675,7 +1675,7 @@ ZL1:; /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 821 "src/libre/parser.act" +#line 820 "src/libre/parser.act" (ZIc) = ast_make_count(1, 1); @@ -1704,7 +1704,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -1714,7 +1714,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -1734,7 +1734,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* BEGINNING OF EXTRACT: CHAR */ { -#line 579 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -1753,7 +1753,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 875 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { @@ -1774,7 +1774,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 784 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -1784,7 +1784,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { @@ -1796,7 +1796,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 809 "src/libre/parser.act" +#line 808 "src/libre/parser.act" (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); @@ -1805,7 +1805,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 898 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1832,7 +1832,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 882 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; @@ -1846,7 +1846,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 912 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { @@ -1884,7 +1884,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 708 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; @@ -1896,7 +1896,7 @@ ZL1:; /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { @@ -1932,7 +1932,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 648 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1956,7 +1956,7 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1034 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { @@ -1986,7 +1986,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 861 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2007,7 +2007,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 854 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { @@ -2032,7 +2032,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1207 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index c5e885439..7825ae3af 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 292 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1209 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/lx/lexer.c b/src/lx/lexer.c index 03bfbf463..ed7ca6d78 100644 --- a/src/lx/lexer.c +++ b/src/lx/lexer.c @@ -14,6 +14,26 @@ static enum lx_token z2(struct lx *lx); static enum lx_token z3(struct lx *lx); static enum lx_token z4(struct lx *lx); +static int +lx_advance_end(struct lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif @@ -34,18 +54,19 @@ lx_getc(struct lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_getc((struct lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -54,10 +75,7 @@ lx_ungetc(struct lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -67,13 +85,20 @@ lx_ungetc(struct lx *lx, int c) } } +/* Get a character from fgetc and push it to the buffer */ int lx_fgetc(struct lx *lx) { assert(lx != NULL); assert(lx->getc_opaque != NULL); - return fgetc(lx->getc_opaque); + const int c = fgetc(lx->getc_opaque); + if (c == EOF) { + lx->c = EOF; + return EOF; + } else { + return c; + } } int @@ -118,6 +143,17 @@ lx_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_dynclear(void *buf_opaque) { @@ -157,37 +193,36 @@ lx_dynfree(void *buf_opaque) static enum lx_token z0(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '/': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "\057" */ switch ((unsigned char) c) { @@ -243,79 +278,73 @@ z0(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return lx->z = z4, TOK_RE; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, TOK_RE; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return lx->z = z4, TOK_RE; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_RE; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z1(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '"': state = S2; break; - case '\\': state = S3; break; - default: state = S1; break; + case '\\': state = S1; break; + case '"': state = S3; break; + default: state = S2; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; - - case S2: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z4, TOK_STR; - - case S3: /* e.g. "\\" */ + case S1: /* e.g. "\\" */ switch ((unsigned char) c) { - case '"': - case '\\': - case 'f': - case 'n': - case 'r': - case 't': - case 'v': state = S4; break; + case 'x': state = S4; break; case '0': case '1': case '2': @@ -324,29 +353,24 @@ z1(struct lx *lx) case '5': case '6': case '7': state = S5; break; - case 'x': state = S6; break; - default: lx_ungetc(lx, c); return TOK_CHAR; + case '"': + case '\\': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': state = S6; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S4: /* e.g. "\\f" */ - lx_ungetc(lx, c); return TOK_ESC; + case S2: /* e.g. "\\x00" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; - case S5: /* e.g. "\\0" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': break; - default: lx_ungetc(lx, c); return TOK_OCT; - } - break; + case S3: /* e.g. "\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, TOK_STR; - case S6: /* e.g. "\\x" */ + case S4: /* e.g. "\\x" */ switch ((unsigned char) c) { case '0': case '1': @@ -370,11 +394,30 @@ z1(struct lx *lx) case 'd': case 'e': case 'f': state = S7; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S5: /* e.g. "\\0" */ + switch ((unsigned char) c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S7: /* e.g. "\\xa" */ + case S6: /* e.g. "\\\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ESC; + + case S7: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -398,135 +441,157 @@ z1(struct lx *lx) case 'd': case 'e': case 'f': break; - default: lx_ungetc(lx, c); return TOK_HEX; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_HEX; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z4, TOK_STR; + case S5: return TOK_OCT; + case S6: return TOK_ESC; + case S7: return TOK_HEX; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_STR; - case S3: return TOK_CHAR; - case S4: return TOK_ESC; - case S5: return TOK_OCT; - case S7: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z2(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\'': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z4, TOK_STR; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, TOK_STR; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return lx->z = z4, TOK_STR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_STR; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z3(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\n': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return lx->z(lx); + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); case S2: /* e.g. "" */ - lx_ungetc(lx, c); return lx->z = z4, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return lx->z = z4, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S0: @@ -536,75 +601,52 @@ z3(struct lx *lx) default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z4(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '\t': - case '\n': - case '\r': - case ' ': state = S1; break; - case '!': state = S2; break; - case '"': state = S3; break; - case '#': state = S4; break; - case '$': state = S5; break; - case '&': state = S6; break; - case '\'': state = S7; break; - case '(': state = S8; break; - case ')': state = S9; break; - case '*': state = S10; break; - case '+': state = S11; break; - case ',': state = S12; break; - case '-': state = S13; break; - case '.': state = S14; break; - case '/': state = S15; break; - case ';': state = S16; break; - case '=': state = S17; break; - case '?': state = S18; break; + case ',': state = S1; break; + case '$': state = S2; break; case 'A': case 'B': case 'C': @@ -657,37 +699,42 @@ z4(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S19; break; - case '\\': state = S20; break; - case '^': state = S21; break; - case '{': state = S22; break; - case '|': state = S23; break; - case '}': state = S24; break; - case '~': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; - - case S1: /* e.g. "\\x09" */ - switch ((unsigned char) c) { + case 'z': state = S3; break; + case '&': state = S4; break; + case '|': state = S5; break; + case '.': state = S6; break; + case '-': state = S7; break; + case '\\': state = S8; break; + case '^': state = S9; break; + case '!': state = S10; break; + case '~': state = S11; break; + case '?': state = S12; break; + case '+': state = S13; break; + case '*': state = S14; break; + case ')': state = S15; break; + case '(': state = S16; break; + case '}': state = S17; break; + case '{': state = S18; break; + case ';': state = S19; break; + case '=': state = S20; break; + case '/': state = S21; break; + case '"': state = S22; break; + case '\'': state = S23; break; + case '#': state = S24; break; case '\t': case '\n': case '\r': - case ' ': break; - default: lx_ungetc(lx, c); return lx->z(lx); + case ' ': state = S25; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S2: /* e.g. "!" */ - lx_ungetc(lx, c); return TOK_BANG; - - case S3: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z1, lx->z(lx); - - case S4: /* e.g. "#" */ - lx_ungetc(lx, c); return lx->z = z3, lx->z(lx); + case S1: /* e.g. "," */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_COMMA; - case S5: /* e.g. "$" */ + case S2: /* e.g. "$" */ switch ((unsigned char) c) { case 'A': case 'B': @@ -742,58 +789,13 @@ z4(struct lx *lx) case 'x': case 'y': case 'z': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S6: /* e.g. "&" */ - lx_ungetc(lx, c); return TOK_AND; - - case S7: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z2, lx->z(lx); - - case S8: /* e.g. "(" */ - lx_ungetc(lx, c); return TOK_LPAREN; - - case S9: /* e.g. ")" */ - lx_ungetc(lx, c); return TOK_RPAREN; - - case S10: /* e.g. "*" */ - lx_ungetc(lx, c); return TOK_STAR; - - case S11: /* e.g. "+" */ - lx_ungetc(lx, c); return TOK_CROSS; - - case S12: /* e.g. "," */ - lx_ungetc(lx, c); return TOK_COMMA; - - case S13: /* e.g. "-" */ - switch ((unsigned char) c) { - case '>': state = S27; break; - default: lx_ungetc(lx, c); return TOK_DASH; - } - break; - - case S14: /* e.g. "." */ - switch ((unsigned char) c) { - case '.': state = S26; break; - default: lx_ungetc(lx, c); return TOK_DOT; - } - break; - - case S15: /* e.g. "\057" */ - lx_ungetc(lx, c); return lx->z = z0, lx->z(lx); - - case S16: /* e.g. ";" */ - lx_ungetc(lx, c); return TOK_SEMI; - - case S17: /* e.g. "=" */ - lx_ungetc(lx, c); return TOK_BIND; - - case S18: /* e.g. "?" */ - lx_ungetc(lx, c); return TOK_QMARK; - - case S19: /* e.g. "a" */ + case S3: /* e.g. "A" */ switch ((unsigned char) c) { case '0': case '1': @@ -858,35 +860,98 @@ z4(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S20: /* e.g. "\\" */ - lx_ungetc(lx, c); return TOK_DASH; + case S4: /* e.g. "&" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_AND; + + case S5: /* e.g. "|" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_PIPE; + + case S6: /* e.g. "." */ + switch ((unsigned char) c) { + case '.': state = S27; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_DOT; + } + break; + + case S7: /* e.g. "-" */ + switch ((unsigned char) c) { + case '>': state = S26; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_DASH; + } + break; + + case S8: /* e.g. "\\" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_DASH; + + case S9: /* e.g. "^" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_HAT; - case S21: /* e.g. "^" */ - lx_ungetc(lx, c); return TOK_HAT; + case S10: /* e.g. "!" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_BANG; - case S22: /* e.g. "{" */ - lx_ungetc(lx, c); return TOK_OPEN; + case S11: /* e.g. "~" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TILDE; - case S23: /* e.g. "|" */ - lx_ungetc(lx, c); return TOK_PIPE; + case S12: /* e.g. "\077" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_QMARK; - case S24: /* e.g. "}" */ - lx_ungetc(lx, c); return TOK_CLOSE; + case S13: /* e.g. "+" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CROSS; - case S25: /* e.g. "~" */ - lx_ungetc(lx, c); return TOK_TILDE; + case S14: /* e.g. "*" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_STAR; - case S26: /* e.g. ".." */ - lx_ungetc(lx, c); return TOK_TO; + case S15: /* e.g. ")" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_RPAREN; - case S27: /* e.g. "->" */ - lx_ungetc(lx, c); return TOK_MAP; + case S16: /* e.g. "(" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_LPAREN; - case S28: /* e.g. "$a" */ + case S17: /* e.g. "}" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CLOSE; + + case S18: /* e.g. "{" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_OPEN; + + case S19: /* e.g. ";" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_SEMI; + + case S20: /* e.g. "=" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_BIND; + + case S21: /* e.g. "\057" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z0, lx->z(lx); + + case S22: /* e.g. "\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z1, lx->z(lx); + + case S23: /* e.g. "'" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z2, lx->z(lx); + + case S24: /* e.g. "#" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z3, lx->z(lx); + + case S25: /* e.g. "\\x09" */ + switch ((unsigned char) c) { + case '\t': + case '\n': + case '\r': + case ' ': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); + } + break; + + case S26: /* e.g. "->" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_MAP; + + case S27: /* e.g. ".." */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TO; + + case S28: /* e.g. "$A" */ switch ((unsigned char) c) { case '0': case '1': @@ -951,66 +1016,73 @@ z4(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return TOK_TOKEN; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TOKEN; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COMMA; + case S3: return TOK_IDENT; + case S4: return TOK_AND; + case S5: return TOK_PIPE; + case S6: return TOK_DOT; + case S7: return TOK_DASH; + case S8: return TOK_DASH; + case S9: return TOK_HAT; + case S10: return TOK_BANG; + case S11: return TOK_TILDE; + case S12: return TOK_QMARK; + case S13: return TOK_CROSS; + case S14: return TOK_STAR; + case S15: return TOK_RPAREN; + case S16: return TOK_LPAREN; + case S17: return TOK_CLOSE; + case S18: return TOK_OPEN; + case S19: return TOK_SEMI; + case S20: return TOK_BIND; + case S21: return lx->z = z0, lx->z(lx); + case S22: return lx->z = z1, lx->z(lx); + case S23: return lx->z = z2, lx->z(lx); + case S24: return lx->z = z3, lx->z(lx); + case S25: return TOK_EOF; + case S26: return TOK_MAP; + case S27: return TOK_TO; + case S28: return TOK_TOKEN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { - case S1: - case S3: - case S4: - case S7: - case S15: + case S21: + case S22: + case S23: + case S24: + case S25: break; default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_BANG; - case S3: return TOK_EOF; - case S4: return TOK_EOF; - case S6: return TOK_AND; - case S7: return TOK_EOF; - case S8: return TOK_LPAREN; - case S9: return TOK_RPAREN; - case S10: return TOK_STAR; - case S11: return TOK_CROSS; - case S12: return TOK_COMMA; - case S13: return TOK_DASH; - case S14: return TOK_DOT; - case S15: return TOK_EOF; - case S16: return TOK_SEMI; - case S17: return TOK_BIND; - case S18: return TOK_QMARK; - case S19: return TOK_IDENT; - case S20: return TOK_DASH; - case S21: return TOK_HAT; - case S22: return TOK_OPEN; - case S23: return TOK_PIPE; - case S24: return TOK_CLOSE; - case S25: return TOK_TILDE; - case S26: return TOK_TO; - case S27: return TOK_MAP; - case S28: return TOK_TOKEN; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -1238,6 +1310,7 @@ lx_init(struct lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_dynpop; } enum lx_token diff --git a/src/lx/parser.c b/src/lx/parser.c index f6f759693..4ab69ce21 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 127 "src/lx/parser.act" +#line 27 "src/lx/parser.act" #include @@ -182,7 +182,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-list */ { -#line 816 "src/lx/parser.act" +#line 814 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); @@ -204,7 +204,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: IDENT */ { -#line 228 "src/lx/parser.act" +#line 227 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); @@ -214,7 +214,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-var */ { -#line 280 "src/lx/parser.act" +#line 277 "src/lx/parser.act" struct ast_zone *z; @@ -252,7 +252,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: TOKEN */ { -#line 224 "src/lx/parser.act" +#line 222 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ @@ -263,7 +263,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 308 "src/lx/parser.act" +#line 304 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -374,7 +374,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-reverse */ { -#line 677 "src/lx/parser.act" +#line 676 "src/lx/parser.act" assert((ZI210) != NULL); @@ -398,7 +398,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI231) != NULL); @@ -449,7 +449,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complete */ { -#line 668 "src/lx/parser.act" +#line 667 "src/lx/parser.act" assert((ZI210) != NULL); @@ -473,7 +473,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI239) != NULL); @@ -509,7 +509,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: IDENT */ { -#line 228 "src/lx/parser.act" +#line 227 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); @@ -563,7 +563,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI248) != NULL); @@ -614,7 +614,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complement */ { -#line 659 "src/lx/parser.act" +#line 658 "src/lx/parser.act" assert((ZI210) != NULL); @@ -638,7 +638,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI223) != NULL); @@ -685,7 +685,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: TOKEN */ { -#line 224 "src/lx/parser.act" +#line 222 "src/lx/parser.act" /* TODO: submatch addressing */ ZI253 = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ @@ -696,7 +696,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 308 "src/lx/parser.act" +#line 304 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -769,7 +769,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI259) != NULL); @@ -829,7 +829,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI268) != NULL); @@ -869,7 +869,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-thing */ { -#line 812 "src/lx/parser.act" +#line 810 "src/lx/parser.act" err_expected(lex_state, "mapping, binding or zone"); @@ -899,7 +899,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-open */ { -#line 800 "src/lx/parser.act" +#line 798 "src/lx/parser.act" err_expected(lex_state, "'{'"); @@ -929,7 +929,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-close */ { -#line 804 "src/lx/parser.act" +#line 802 "src/lx/parser.act" err_expected(lex_state, "'}'"); @@ -955,7 +955,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: CHAR */ { -#line 219 "src/lx/parser.act" +#line 215 "src/lx/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); @@ -972,7 +972,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: ESC */ { -#line 149 "src/lx/parser.act" +#line 143 "src/lx/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1000,7 +1000,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: HEX */ { -#line 212 "src/lx/parser.act" +#line 188 "src/lx/parser.act" unsigned long u; char *e; @@ -1037,7 +1037,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: OCT */ { -#line 185 "src/lx/parser.act" +#line 161 "src/lx/parser.act" unsigned long u; char *e; @@ -1077,7 +1077,7 @@ ZL2_pattern_C_Cbody:; /* END OF INLINE: 84 */ /* BEGINNING OF ACTION: pattern-char */ { -#line 249 "src/lx/parser.act" +#line 247 "src/lx/parser.act" /* TODO */ *lex_state->p++ = (ZIc); @@ -1089,7 +1089,7 @@ ZL2_pattern_C_Cbody:; goto ZL2_pattern_C_Cbody; /* END OF INLINE: pattern::body */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -1124,7 +1124,7 @@ ZL2_174:; } /* END OF INLINE: 174 */ } - /*UNREACHED*/ + /* UNREACHED */ default: { ZI171 = ZI168; @@ -1211,7 +1211,7 @@ ZL2_180:; } /* BEGINNING OF ACTION: op-alt */ { -#line 725 "src/lx/parser.act" +#line 724 "src/lx/parser.act" assert((ZI177) != NULL); assert((ZIb) != NULL); @@ -1230,7 +1230,7 @@ ZL2_180:; goto ZL2_180; /* END OF INLINE: 180 */ } - /*UNREACHED*/ + /* UNREACHED */ default: { ZI178 = ZI176; @@ -1291,7 +1291,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-reverse */ { -#line 677 "src/lx/parser.act" +#line 676 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1315,7 +1315,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complete */ { -#line 668 "src/lx/parser.act" +#line 667 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1339,7 +1339,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complement */ { -#line 659 "src/lx/parser.act" +#line 658 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1398,7 +1398,7 @@ ZL2_186:; } /* BEGINNING OF ACTION: op-intersect */ { -#line 714 "src/lx/parser.act" +#line 713 "src/lx/parser.act" assert((ZI183) != NULL); assert((ZIb) != NULL); @@ -1417,7 +1417,7 @@ ZL2_186:; goto ZL2_186; /* END OF INLINE: 186 */ } - /*UNREACHED*/ + /* UNREACHED */ default: { ZI184 = ZI182; @@ -1455,7 +1455,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ } /* BEGINNING OF ACTION: op-alt */ { -#line 725 "src/lx/parser.act" +#line 724 "src/lx/parser.act" assert((ZIold_Hexit) != NULL); assert((ZInew_Hexit) != NULL); @@ -1473,7 +1473,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ goto ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_Hzone_Hto_Hmappings_Hx; /* END OF INLINE: list-of-things::zone-thing::list-of-zone-to-mappings::list-of-zone-to-mappings-x */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -1554,7 +1554,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-cross */ { -#line 575 "src/lx/parser.act" +#line 568 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1607,13 +1607,13 @@ ZL2_197:; goto ZL2_197; /* END OF INLINE: 197 */ } - /*UNREACHED*/ + /* UNREACHED */ case (TOK_QMARK): { ADVANCE_LEXER; /* BEGINNING OF ACTION: op-qmark */ { -#line 620 "src/lx/parser.act" +#line 613 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1666,13 +1666,13 @@ ZL2_197:; goto ZL2_197; /* END OF INLINE: 197 */ } - /*UNREACHED*/ + /* UNREACHED */ case (TOK_STAR): { ADVANCE_LEXER; /* BEGINNING OF ACTION: op-star */ { -#line 525 "src/lx/parser.act" +#line 518 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1730,14 +1730,14 @@ ZL2_197:; goto ZL2_197; /* END OF INLINE: 197 */ } - /*UNREACHED*/ + /* UNREACHED */ default: goto ZL1; } } /* END OF INLINE: 272 */ } - /*UNREACHED*/ + /* UNREACHED */ default: { ZI196 = ZI191; @@ -1795,7 +1795,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) } /* BEGINNING OF ACTION: op-subtract */ { -#line 703 "src/lx/parser.act" +#line 702 "src/lx/parser.act" assert((*ZI202) != NULL); assert((ZIb) != NULL); @@ -1870,7 +1870,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) } /* BEGINNING OF ACTION: op-concat */ { -#line 686 "src/lx/parser.act" +#line 685 "src/lx/parser.act" assert((*ZI206) != NULL); assert((ZIb) != NULL); @@ -1920,7 +1920,7 @@ p_212(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI210, fsm *ZOq) } /* BEGINNING OF ACTION: op-product */ { -#line 698 "src/lx/parser.act" +#line 696 "src/lx/parser.act" fprintf(stderr, "unimplemented\n"); (ZIq) = NULL; @@ -1960,7 +1960,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* BEGINNING OF EXTRACT: RE */ { -#line 236 "src/lx/parser.act" +#line 231 "src/lx/parser.act" assert(lex_state->buf.a[0] == '/'); @@ -1980,7 +1980,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 263 "src/lx/parser.act" +#line 252 "src/lx/parser.act" size_t len; @@ -2010,7 +2010,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-regex */ { -#line 379 "src/lx/parser.act" +#line 376 "src/lx/parser.act" struct re_err err; @@ -2030,7 +2030,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* END OF ACTION: compile-regex */ /* BEGINNING OF ACTION: free-arr */ { -#line 766 "src/lx/parser.act" +#line 765 "src/lx/parser.act" free((ZIa)); @@ -2046,7 +2046,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 263 "src/lx/parser.act" +#line 252 "src/lx/parser.act" size_t len; @@ -2076,7 +2076,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-literal */ { -#line 364 "src/lx/parser.act" +#line 361 "src/lx/parser.act" struct re_err err; @@ -2096,7 +2096,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* END OF ACTION: compile-literal */ /* BEGINNING OF ACTION: free-arr */ { -#line 766 "src/lx/parser.act" +#line 765 "src/lx/parser.act" free((ZIa)); @@ -2164,7 +2164,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2189,7 +2189,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 453 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2229,7 +2229,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho goto ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Hof_Hzone_Hfrom_Hmappings_Hx; /* END OF INLINE: list-of-things::zone-thing::list-of-zone-from-mappings::list-of-zone-from-mappings-x */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -2256,7 +2256,7 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* BEGINNING OF ACTION: no-zone */ { -#line 515 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIparent) = NULL; @@ -2265,7 +2265,7 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: make-ast */ { -#line 424 "src/lx/parser.act" +#line 423 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2278,7 +2278,7 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: make-zone */ { -#line 432 "src/lx/parser.act" +#line 431 "src/lx/parser.act" assert((ZIa) != NULL); @@ -2301,7 +2301,7 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: no-exit */ { -#line 511 "src/lx/parser.act" +#line 510 "src/lx/parser.act" (ZIexit) = NULL; @@ -2310,7 +2310,7 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: set-globalzone */ { -#line 500 "src/lx/parser.act" +#line 499 "src/lx/parser.act" assert((ZIa) != NULL); assert((ZIz) != NULL); @@ -2341,7 +2341,7 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 808 "src/lx/parser.act" +#line 806 "src/lx/parser.act" err_expected(lex_state, "EOF"); @@ -2358,7 +2358,7 @@ ZL1:; { /* BEGINNING OF ACTION: make-ast */ { -#line 424 "src/lx/parser.act" +#line 423 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2371,7 +2371,7 @@ ZL1:; /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: err-syntax */ { -#line 776 "src/lx/parser.act" +#line 773 "src/lx/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); @@ -2433,7 +2433,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ } /* BEGINNING OF ACTION: add-mapping */ { -#line 453 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2471,7 +2471,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ /* END OF ACTION: add-mapping */ /* BEGINNING OF ACTION: clone */ { -#line 756 "src/lx/parser.act" +#line 755 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2517,7 +2517,7 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: err-expected-map */ { -#line 784 "src/lx/parser.act" +#line 782 "src/lx/parser.act" err_expected(lex_state, "'->'"); @@ -2532,7 +2532,7 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) case (TOK_TOKEN): /* BEGINNING OF EXTRACT: TOKEN */ { -#line 224 "src/lx/parser.act" +#line 222 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ @@ -2551,7 +2551,7 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: no-token */ { -#line 507 "src/lx/parser.act" +#line 506 "src/lx/parser.act" (ZIt) = NULL; @@ -2644,7 +2644,7 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-semi */ { -#line 792 "src/lx/parser.act" +#line 790 "src/lx/parser.act" err_expected(lex_state, "';'"); @@ -2695,7 +2695,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 515 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIto) = NULL; @@ -2704,7 +2704,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 453 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2749,7 +2749,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: make-zone */ { -#line 432 "src/lx/parser.act" +#line 431 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -2772,7 +2772,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 453 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2829,7 +2829,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-to */ { -#line 796 "src/lx/parser.act" +#line 794 "src/lx/parser.act" err_expected(lex_state, "'..'"); @@ -2858,7 +2858,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 515 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIx) = NULL; @@ -2867,7 +2867,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: no-token */ { -#line 507 "src/lx/parser.act" +#line 506 "src/lx/parser.act" (ZIy) = NULL; @@ -2876,7 +2876,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: no-token */ /* BEGINNING OF ACTION: regex-any */ { -#line 395 "src/lx/parser.act" +#line 392 "src/lx/parser.act" fsm_state_t start, end; @@ -2912,7 +2912,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: regex-any */ /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZIw) != NULL); @@ -2937,7 +2937,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 453 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2997,7 +2997,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 816 "src/lx/parser.act" +#line 814 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); @@ -3017,7 +3017,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: no-exit */ { -#line 511 "src/lx/parser.act" +#line 510 "src/lx/parser.act" (ZIr2) = NULL; @@ -3026,7 +3026,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: make-zone */ { -#line 432 "src/lx/parser.act" +#line 431 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -3049,7 +3049,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 453 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -3101,7 +3101,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 816 "src/lx/parser.act" +#line 814 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); @@ -3149,7 +3149,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-bind */ { -#line 788 "src/lx/parser.act" +#line 786 "src/lx/parser.act" err_expected(lex_state, "'='"); @@ -3168,7 +3168,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: add-binding */ { -#line 485 "src/lx/parser.act" +#line 482 "src/lx/parser.act" struct var *v; @@ -3211,7 +3211,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: deref-var */ { -#line 280 "src/lx/parser.act" +#line 277 "src/lx/parser.act" struct ast_zone *z; @@ -3254,7 +3254,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: subtract-exit */ { -#line 736 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI278) != NULL); @@ -3297,7 +3297,7 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 880 "src/lx/parser.act" +#line 818 "src/lx/parser.act" struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc) { diff --git a/src/lx/parser.h b/src/lx/parser.h index fdaff9879..947b194b5 100644 --- a/src/lx/parser.h +++ b/src/lx/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 139 "src/lx/parser.act" +#line 127 "src/lx/parser.act" #include @@ -29,7 +29,7 @@ extern void p_lx(lex_state, act_state, ast *); /* BEGINNING OF TRAILER */ -#line 882 "src/lx/parser.act" +#line 880 "src/lx/parser.act" #line 36 "src/lx/parser.h" From 051c362412961984bcad53fdfd25202a98048df9 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 28 Aug 2025 21:27:27 +0100 Subject: [PATCH 308/373] Typo. --- man/fsm.1/fsm.1.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/fsm.1/fsm.1.xml b/man/fsm.1/fsm.1.xml index 776aaaab9..caec5309f 100644 --- a/man/fsm.1/fsm.1.xml +++ b/man/fsm.1/fsm.1.xml @@ -22,6 +22,7 @@ io"> iterations"> length"> + limit"> charset"> -a"> @@ -33,7 +34,7 @@ -G &length.arg;"> -k &io.arg;"> -i &iterations.arg;"> - -S &limit.arg;"> + -S &limit.arg;"> -U &charset.arg;"> -X"> From 87f6df26550658d51da875e286efc4781cd73f75 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 28 Aug 2025 21:28:34 +0100 Subject: [PATCH 309/373] Stray const. --- src/libfsm/print/c.c | 2 +- src/libfsm/print/vmc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index 7f9274792..f55b2e748 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -581,7 +581,7 @@ fsm_print_c(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, ",\n"); - fprintf(f, "\tconst unsigned *id"); + fprintf(f, "\tunsigned *id"); break; case AMBIG_MULTIPLE: diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index 9ea06f27a..b427a9cb8 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -588,7 +588,7 @@ fsm_print_vmc(FILE *f, case AMBIG_ERROR: case AMBIG_EARLIEST: fprintf(f, ",\n"); - fprintf(f, "\tconst unsigned *id"); + fprintf(f, "\tunsigned *id"); break; case AMBIG_MULTIPLE: From 664b32cd567cbb884a25bc9d1f15e3411183e5e8 Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Thu, 11 Sep 2025 01:04:25 +0500 Subject: [PATCH 310/373] Update to Unicode 17.0 --- share/ucd/CaseFolding.txt | 40 ++- share/ucd/Makefile | 2 +- share/ucd/Scripts.txt | 142 ++++++--- share/ucd/UnicodeData.txt | 489 ++++++++++++++++++++++++++++- src/libre/class.h | 4 + src/libre/class/utf8_Arabic.c | 15 +- src/libre/class/utf8_Beria_Erfe.c | 14 + src/libre/class/utf8_Common.c | 30 +- src/libre/class/utf8_Han.c | 9 +- src/libre/class/utf8_Inherited.c | 3 +- src/libre/class/utf8_Kannada.c | 2 +- src/libre/class/utf8_L.c | 40 ++- src/libre/class/utf8_Latin.c | 7 +- src/libre/class/utf8_Ll.c | 4 +- src/libre/class/utf8_Lm.c | 6 +- src/libre/class/utf8_Lo.c | 32 +- src/libre/class/utf8_Lu.c | 4 + src/libre/class/utf8_M.c | 10 +- src/libre/class/utf8_Mc.c | 3 + src/libre/class/utf8_Mn.c | 12 +- src/libre/class/utf8_N.c | 2 + src/libre/class/utf8_Nd.c | 1 + src/libre/class/utf8_Nl.c | 3 +- src/libre/class/utf8_P.c | 1 + src/libre/class/utf8_Po.c | 1 + src/libre/class/utf8_S.c | 36 ++- src/libre/class/utf8_Sc.c | 2 +- src/libre/class/utf8_Sharada.c | 3 +- src/libre/class/utf8_Sidetic.c | 13 + src/libre/class/utf8_Sm.c | 4 +- src/libre/class/utf8_So.c | 32 +- src/libre/class/utf8_Tai_Yo.c | 15 + src/libre/class/utf8_Tangut.c | 6 +- src/libre/class/utf8_Telugu.c | 2 +- src/libre/class/utf8_Tolong_Siki.c | 14 + src/libre/class/utf8_assigned.c | 83 ++--- src/libre/class_name.c | 4 + 37 files changed, 881 insertions(+), 209 deletions(-) create mode 100644 src/libre/class/utf8_Beria_Erfe.c create mode 100644 src/libre/class/utf8_Sidetic.c create mode 100644 src/libre/class/utf8_Tai_Yo.c create mode 100644 src/libre/class/utf8_Tolong_Siki.c diff --git a/share/ucd/CaseFolding.txt b/share/ucd/CaseFolding.txt index 1b7a9c156..a0b0f07fd 100644 --- a/share/ucd/CaseFolding.txt +++ b/share/ucd/CaseFolding.txt @@ -1,6 +1,6 @@ -# CaseFolding-16.0.0.txt -# Date: 2024-04-30, 21:48:11 GMT -# © 2024 Unicode®, Inc. +# CaseFolding-17.0.0.txt +# Date: 2025-07-30, 23:54:36 GMT +# © 2025 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use and license, see https://www.unicode.org/terms_of_use.html # @@ -18,15 +18,15 @@ # The data supports both implementations that require simple case foldings # (where string lengths don't change), and implementations that allow full case folding # (where string lengths may grow). Note that where they can be supported, the -# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# full case foldings are superior: for example, they allow "FUSS" and "Fuß" to match. # # All code points not listed in this file map to themselves. # # NOTE: case folding does not preserve normalization formats! # # For information on case folding, including how to have case folding -# preserve normalization formats, see Section 3.13 Default Case Algorithms in -# The Unicode Standard. +# preserve normalization formats, see the +# "Conformance" / "Default Case Algorithms" section of the core specification. # # ================================================================================ # Format @@ -1243,7 +1243,10 @@ A7C7; C; A7C8; # LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY A7C9; C; A7CA; # LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY A7CB; C; 0264; # LATIN CAPITAL LETTER RAMS HORN A7CC; C; A7CD; # LATIN CAPITAL LETTER S WITH DIAGONAL STROKE +A7CE; C; A7CF; # LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE A7D0; C; A7D1; # LATIN CAPITAL LETTER CLOSED INSULAR G +A7D2; C; A7D3; # LATIN CAPITAL LETTER DOUBLE THORN +A7D4; C; A7D5; # LATIN CAPITAL LETTER DOUBLE WYNN A7D6; C; A7D7; # LATIN CAPITAL LETTER MIDDLE SCOTS S A7D8; C; A7D9; # LATIN CAPITAL LETTER SIGMOID S A7DA; C; A7DB; # LATIN CAPITAL LETTER LAMBDA @@ -1616,6 +1619,31 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z 16E5D; C; 16E7D; # MEDEFAIDRIN CAPITAL LETTER O 16E5E; C; 16E7E; # MEDEFAIDRIN CAPITAL LETTER AI 16E5F; C; 16E7F; # MEDEFAIDRIN CAPITAL LETTER Y +16EA0; C; 16EBB; # BERIA ERFE CAPITAL LETTER ARKAB +16EA1; C; 16EBC; # BERIA ERFE CAPITAL LETTER BASIGNA +16EA2; C; 16EBD; # BERIA ERFE CAPITAL LETTER DARBAI +16EA3; C; 16EBE; # BERIA ERFE CAPITAL LETTER EH +16EA4; C; 16EBF; # BERIA ERFE CAPITAL LETTER FITKO +16EA5; C; 16EC0; # BERIA ERFE CAPITAL LETTER GOWAY +16EA6; C; 16EC1; # BERIA ERFE CAPITAL LETTER HIRDEABO +16EA7; C; 16EC2; # BERIA ERFE CAPITAL LETTER I +16EA8; C; 16EC3; # BERIA ERFE CAPITAL LETTER DJAI +16EA9; C; 16EC4; # BERIA ERFE CAPITAL LETTER KOBO +16EAA; C; 16EC5; # BERIA ERFE CAPITAL LETTER LAKKO +16EAB; C; 16EC6; # BERIA ERFE CAPITAL LETTER MERI +16EAC; C; 16EC7; # BERIA ERFE CAPITAL LETTER NINI +16EAD; C; 16EC8; # BERIA ERFE CAPITAL LETTER GNA +16EAE; C; 16EC9; # BERIA ERFE CAPITAL LETTER NGAY +16EAF; C; 16ECA; # BERIA ERFE CAPITAL LETTER OI +16EB0; C; 16ECB; # BERIA ERFE CAPITAL LETTER PI +16EB1; C; 16ECC; # BERIA ERFE CAPITAL LETTER ERIGO +16EB2; C; 16ECD; # BERIA ERFE CAPITAL LETTER ERIGO TAMURA +16EB3; C; 16ECE; # BERIA ERFE CAPITAL LETTER SERI +16EB4; C; 16ECF; # BERIA ERFE CAPITAL LETTER SHEP +16EB5; C; 16ED0; # BERIA ERFE CAPITAL LETTER TATASOUE +16EB6; C; 16ED1; # BERIA ERFE CAPITAL LETTER UI +16EB7; C; 16ED2; # BERIA ERFE CAPITAL LETTER WASSE +16EB8; C; 16ED3; # BERIA ERFE CAPITAL LETTER AY 1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF 1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI 1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM diff --git a/share/ucd/Makefile b/share/ucd/Makefile index 833910330..22aa0a869 100644 --- a/share/ucd/Makefile +++ b/share/ucd/Makefile @@ -1,4 +1,4 @@ -UCD_URL ?= https://www.unicode.org/Public/16.0.0/ucd/ +UCD_URL ?= https://www.unicode.org/Public/17.0.0/ucd/ WGET ?= wget diff --git a/share/ucd/Scripts.txt b/share/ucd/Scripts.txt index 443a6d2dd..5574fdd6a 100644 --- a/share/ucd/Scripts.txt +++ b/share/ucd/Scripts.txt @@ -1,6 +1,6 @@ -# Scripts-16.0.0.txt -# Date: 2024-04-30, 21:48:40 GMT -# © 2024 Unicode®, Inc. +# Scripts-17.0.0.txt +# Date: 2025-07-24, 13:28:55 GMT +# © 2025 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use and license, see https://www.unicode.org/terms_of_use.html # @@ -154,7 +154,7 @@ 208A..208C ; Common # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN 208D ; Common # Ps SUBSCRIPT LEFT PARENTHESIS 208E ; Common # Pe SUBSCRIPT RIGHT PARENTHESIS -20A0..20C0 ; Common # Sc [33] EURO-CURRENCY SIGN..SOM SIGN +20A0..20C1 ; Common # Sc [34] EURO-CURRENCY SIGN..SAUDI RIYAL SIGN 2100..2101 ; Common # So [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT 2102 ; Common # L& DOUBLE-STRUCK CAPITAL C 2103..2106 ; Common # So [4] DEGREE CELSIUS..CADA UNA @@ -306,8 +306,7 @@ 2B45..2B46 ; Common # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW 2B47..2B4C ; Common # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR 2B4D..2B73 ; Common # So [39] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR -2B76..2B95 ; Common # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW -2B97..2BFF ; Common # So [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL +2B76..2BFF ; Common # So [138] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..HELLSCHREIBER PAUSE SYMBOL 2E00..2E01 ; Common # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER 2E02 ; Common # Pi LEFT SUBSTITUTION BRACKET 2E03 ; Common # Pf RIGHT SUBSTITUTION BRACKET @@ -524,7 +523,11 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 1BCA0..1BCA3 ; Common # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP 1CC00..1CCEF ; Common # So [240] UP-POINTING GO-KART..OUTLINED LATIN CAPITAL LETTER Z 1CCF0..1CCF9 ; Common # Nd [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE +1CCFA..1CCFC ; Common # So [3] SNAKE SYMBOL..NOSE SYMBOL 1CD00..1CEB3 ; Common # So [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET +1CEBA..1CED0 ; Common # So [23] FRAGILE SYMBOL..LEUKOTHEA +1CEE0..1CEEF ; Common # So [16] GEOMANTIC FIGURE POPULUS..GEOMANTIC FIGURE VIA +1CEF0 ; Common # Sm MEDIUM SMALL WHITE CIRCLE WITH HORIZONTAL BAR 1CF50..1CFC3 ; Common # So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK 1D000..1D0F5 ; Common # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO 1D100..1D126 ; Common # So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 @@ -605,11 +608,10 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 1F260..1F265 ; Common # So [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI 1F300..1F3FA ; Common # So [251] CYCLONE..AMPHORA 1F3FB..1F3FF ; Common # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 -1F400..1F6D7 ; Common # So [728] RAT..ELEVATOR +1F400..1F6D8 ; Common # So [729] RAT..LANDSLIDE 1F6DC..1F6EC ; Common # So [17] WIRELESS..AIRPLANE ARRIVING 1F6F0..1F6FC ; Common # So [13] SATELLITE..ROLLER SKATE -1F700..1F776 ; Common # So [119] ALCHEMICAL SYMBOL FOR QUINTESSENCE..LUNAR ECLIPSE -1F77B..1F7D9 ; Common # So [95] HAUMEA..NINE POINTED WHITE STAR +1F700..1F7D9 ; Common # So [218] ALCHEMICAL SYMBOL FOR QUINTESSENCE..NINE POINTED WHITE STAR 1F7E0..1F7EB ; Common # So [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE 1F7F0 ; Common # So HEAVY EQUALS SIGN 1F800..1F80B ; Common # So [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD @@ -619,21 +621,24 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 1F890..1F8AD ; Common # So [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS 1F8B0..1F8BB ; Common # So [12] ARROW POINTING UPWARDS THEN NORTH WEST..SOUTH WEST ARROW FROM BAR 1F8C0..1F8C1 ; Common # So [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW -1F900..1FA53 ; Common # So [340] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS KNIGHT-BISHOP +1F8D0..1F8D8 ; Common # Sm [9] LONG RIGHTWARDS ARROW OVER LONG LEFTWARDS ARROW..LONG LEFT RIGHT ARROW WITH DEPENDENT LOBE +1F900..1FA57 ; Common # So [344] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS ALFIL 1FA60..1FA6D ; Common # So [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER 1FA70..1FA7C ; Common # So [13] BALLET SHOES..CRUTCH -1FA80..1FA89 ; Common # So [10] YO-YO..HARP -1FA8F..1FAC6 ; Common # So [56] SHOVEL..FINGERPRINT -1FACE..1FADC ; Common # So [15] MOOSE..ROOT VEGETABLE -1FADF..1FAE9 ; Common # So [11] SPLATTER..FACE WITH BAGS UNDER EYES -1FAF0..1FAF8 ; Common # So [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND +1FA80..1FA8A ; Common # So [11] YO-YO..TROMBONE +1FA8E..1FAC6 ; Common # So [57] TREASURE CHEST..FINGERPRINT +1FAC8 ; Common # So HAIRY CREATURE +1FACD..1FADC ; Common # So [16] ORCA..ROOT VEGETABLE +1FADF..1FAEA ; Common # So [12] SPLATTER..DISTORTED FACE +1FAEF..1FAF8 ; Common # So [10] FIGHT CLOUD..RIGHTWARDS PUSHING HAND 1FB00..1FB92 ; Common # So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK 1FB94..1FBEF ; Common # So [92] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE 1FBF0..1FBF9 ; Common # Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE +1FBFA ; Common # So ALARM BELL SYMBOL E0001 ; Common # Cf LANGUAGE TAG E0020..E007F ; Common # Cf [96] TAG SPACE..CANCEL TAG -# Total code points: 9053 +# Total code points: 9123 # ================================================ @@ -648,8 +653,8 @@ E0020..E007F ; Common # Cf [96] TAG SPACE..CANCEL TAG 01BC..01BF ; Latin # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN 01C0..01C3 ; Latin # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK 01C4..0293 ; Latin # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL -0294 ; Latin # Lo LATIN LETTER GLOTTAL STOP -0295..02AF ; Latin # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +0294..0295 ; Latin # Lo [2] LATIN LETTER GLOTTAL STOP..LATIN LETTER PHARYNGEAL VOICED FRICATIVE +0296..02AF ; Latin # L& [26] LATIN LETTER INVERTED GLOTTAL STOP..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL 02B0..02B8 ; Latin # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y 02E0..02E4 ; Latin # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP 1D00..1D25 ; Latin # L& [38] LATIN LETTER SMALL CAPITAL A..LATIN LETTER AIN @@ -676,11 +681,8 @@ A770 ; Latin # Lm MODIFIER LETTER US A771..A787 ; Latin # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T A78B..A78E ; Latin # L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT A78F ; Latin # Lo LATIN LETTER SINOLOGICAL DOT -A790..A7CD ; Latin # L& [62] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH DIAGONAL STROKE -A7D0..A7D1 ; Latin # L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G -A7D3 ; Latin # L& LATIN SMALL LETTER DOUBLE THORN -A7D5..A7DC ; Latin # L& [8] LATIN SMALL LETTER DOUBLE WYNN..LATIN CAPITAL LETTER LAMBDA WITH STROKE -A7F2..A7F4 ; Latin # Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q +A790..A7DC ; Latin # L& [77] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN CAPITAL LETTER LAMBDA WITH STROKE +A7F1..A7F4 ; Latin # Lm [4] MODIFIER LETTER CAPITAL S..MODIFIER LETTER CAPITAL Q A7F5..A7F6 ; Latin # L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H A7F7 ; Latin # Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I A7F8..A7F9 ; Latin # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE @@ -702,7 +704,7 @@ FF41..FF5A ; Latin # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN 1DF0B..1DF1E ; Latin # L& [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL 1DF25..1DF2A ; Latin # L& [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK -# Total code points: 1487 +# Total code points: 1492 # ================================================ @@ -869,7 +871,7 @@ FB46..FB4F ; Hebrew # Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATU 0750..077F ; Arabic # Lo [48] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE 0870..0887 ; Arabic # Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT 0888 ; Arabic # Sk ARABIC RAISED ROUND DOT -0889..088E ; Arabic # Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL +0889..088F ; Arabic # Lo [7] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC LETTER NOON WITH RING ABOVE 0890..0891 ; Arabic # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE 0897..089F ; Arabic # Mn [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA 08A0..08C8 ; Arabic # Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF @@ -878,11 +880,13 @@ FB46..FB4F ; Hebrew # Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATU 08E3..08FF ; Arabic # Mn [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA FB50..FBB1 ; Arabic # Lo [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM FBB2..FBC2 ; Arabic # Sk [17] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL WASLA ABOVE +FBC3..FBD2 ; Arabic # So [16] ARABIC LIGATURE JALLA WA-ALAA..ARABIC LIGATURE ALAYHI AR-RAHMAH FBD3..FD3D ; Arabic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM FD40..FD4F ; Arabic # So [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH FD50..FD8F ; Arabic # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD90..FD91 ; Arabic # So [2] ARABIC LIGATURE RAHMATU ALLAAHI ALAYH..ARABIC LIGATURE RAHMATU ALLAAHI ALAYHAA FD92..FDC7 ; Arabic # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM -FDCF ; Arabic # So ARABIC LIGATURE SALAAMUHU ALAYNAA +FDC8..FDCF ; Arabic # So [8] ARABIC LIGATURE RAHIMAHU ALLAAH TAAALAA..ARABIC LIGATURE SALAAMUHU ALAYNAA FDF0..FDFB ; Arabic # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU FDFC ; Arabic # Sc RIAL SIGN FDFD..FDFF ; Arabic # So [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL @@ -890,7 +894,11 @@ FE70..FE74 ; Arabic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN FE76..FEFC ; Arabic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM 10E60..10E7E ; Arabic # No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS 10EC2..10EC4 ; Arabic # Lo [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW -10EFC..10EFF ; Arabic # Mn [4] ARABIC COMBINING ALEF OVERLAY..ARABIC SMALL LOW WORD MADDA +10EC5 ; Arabic # Lm ARABIC SMALL YEH BARREE WITH TWO DOTS BELOW +10EC6..10EC7 ; Arabic # Lo [2] ARABIC LETTER THIN NOON..ARABIC LETTER YEH WITH FOUR DOTS BELOW +10ED0 ; Arabic # Po ARABIC BIBLICAL END OF VERSE +10ED1..10ED8 ; Arabic # So [8] ARABIC LIGATURE ALAYHAA AS-SALAATU WAS-SALAAM..ARABIC LIGATURE NAWWARA ALLAAHU MARQADAH +10EFA..10EFF ; Arabic # Mn [6] ARABIC DOUBLE VERTICAL BAR BELOW..ARABIC SMALL LOW WORD MADDA 1EE00..1EE03 ; Arabic # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL 1EE05..1EE1F ; Arabic # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF 1EE21..1EE22 ; Arabic # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM @@ -926,7 +934,7 @@ FE76..FEFC ; Arabic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LA 1EEAB..1EEBB ; Arabic # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN 1EEF0..1EEF1 ; Arabic # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL -# Total code points: 1373 +# Total code points: 1413 # ================================================ @@ -1155,7 +1163,7 @@ A8FF ; Devanagari # Mn DEVANAGARI VOWEL SIGN AY 0C4A..0C4D ; Telugu # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA 0C55..0C56 ; Telugu # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK 0C58..0C5A ; Telugu # Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA -0C5D ; Telugu # Lo TELUGU LETTER NAKAARA POLLU +0C5C..0C5D ; Telugu # Lo [2] TELUGU ARCHAIC SHRII..TELUGU LETTER NAKAARA POLLU 0C60..0C61 ; Telugu # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL 0C62..0C63 ; Telugu # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL 0C66..0C6F ; Telugu # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE @@ -1163,7 +1171,7 @@ A8FF ; Devanagari # Mn DEVANAGARI VOWEL SIGN AY 0C78..0C7E ; Telugu # No [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR 0C7F ; Telugu # So TELUGU SIGN TUUMU -# Total code points: 100 +# Total code points: 101 # ================================================ @@ -1186,14 +1194,14 @@ A8FF ; Devanagari # Mn DEVANAGARI VOWEL SIGN AY 0CCA..0CCB ; Kannada # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO 0CCC..0CCD ; Kannada # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA 0CD5..0CD6 ; Kannada # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK -0CDD..0CDE ; Kannada # Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA +0CDC..0CDE ; Kannada # Lo [3] KANNADA ARCHAIC SHRII..KANNADA LETTER FA 0CE0..0CE1 ; Kannada # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL 0CE2..0CE3 ; Kannada # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL 0CE6..0CEF ; Kannada # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE 0CF1..0CF2 ; Kannada # Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA 0CF3 ; Kannada # Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT -# Total code points: 91 +# Total code points: 92 # ================================================ @@ -1594,17 +1602,18 @@ FA70..FAD9 ; Han # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILI 16FE2 ; Han # Po OLD CHINESE HOOK MARK 16FE3 ; Han # Lm OLD CHINESE ITERATION MARK 16FF0..16FF1 ; Han # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +16FF2..16FF3 ; Han # Lm [2] CHINESE SMALL SIMPLIFIED ER..CHINESE SMALL TRADITIONAL ER +16FF4..16FF6 ; Han # Nl [3] YANGQIN SIGN SLOW ONE BEAT..YANGQIN SIGN SLOW TWO BEATS 20000..2A6DF ; Han # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF -2A700..2B739 ; Han # Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 -2B740..2B81D ; Han # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D -2B820..2CEA1 ; Han # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2A700..2B81D ; Han # Lo [4382] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEAD ; Han # Lo [5774] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEAD 2CEB0..2EBE0 ; Han # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 2EBF0..2EE5D ; Han # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D 2F800..2FA1D ; Han # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D 30000..3134A ; Han # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A -31350..323AF ; Han # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF +31350..33479 ; Han # Lo [8490] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-33479 -# Total code points: 99030 +# Total code points: 103351 # ================================================ @@ -1647,7 +1656,8 @@ A490..A4C6 ; Yi # So [55] YI RADICAL QOT..YI RADICAL KE 0951..0954 ; Inherited # Mn [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT 1AB0..1ABD ; Inherited # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW 1ABE ; Inherited # Me COMBINING PARENTHESES OVERLAY -1ABF..1ACE ; Inherited # Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T +1ABF..1ADD ; Inherited # Mn [31] COMBINING LATIN SMALL LETTER W BELOW..COMBINING DOT-AND-RING BELOW +1AE0..1AEB ; Inherited # Mn [12] COMBINING LEFT TACK ABOVE..COMBINING DOUBLE RIGHTWARDS ARROW ABOVE 1CD0..1CD2 ; Inherited # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA 1CD4..1CE0 ; Inherited # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA 1CE2..1CE8 ; Inherited # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL @@ -1676,7 +1686,7 @@ FE20..FE2D ; Inherited # Mn [14] COMBINING LIGATURE LEFT HALF..COMBINING CON 1D1AA..1D1AD ; Inherited # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO E0100..E01EF ; Inherited # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -# Total code points: 657 +# Total code points: 684 # ================================================ @@ -2347,8 +2357,14 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI 111DB ; Sharada # Po SHARADA SIGN SIDDHAM 111DC ; Sharada # Lo SHARADA HEADSTROKE 111DD..111DF ; Sharada # Po [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 +11B60 ; Sharada # Mn SHARADA VOWEL SIGN OE +11B61 ; Sharada # Mc SHARADA VOWEL SIGN OOE +11B62..11B64 ; Sharada # Mn [3] SHARADA VOWEL SIGN UE..SHARADA VOWEL SIGN SHORT E +11B65 ; Sharada # Mc SHARADA VOWEL SIGN SHORT O +11B66 ; Sharada # Mn SHARADA VOWEL SIGN CANDRA E +11B67 ; Sharada # Mc SHARADA VOWEL SIGN CANDRA O -# Total code points: 96 +# Total code points: 104 # ================================================ @@ -2756,11 +2772,11 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # ================================================ 16FE0 ; Tangut # Lm TANGUT ITERATION MARK -17000..187F7 ; Tangut # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 -18800..18AFF ; Tangut # Lo [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768 -18D00..18D08 ; Tangut # Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +17000..18AFF ; Tangut # Lo [6912] TANGUT IDEOGRAPH-17000..TANGUT COMPONENT-768 +18D00..18D1E ; Tangut # Lo [31] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D1E +18D80..18DF2 ; Tangut # Lo [115] TANGUT COMPONENT-769..TANGUT COMPONENT-883 -# Total code points: 6914 +# Total code points: 7059 # ================================================ @@ -3125,4 +3141,42 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # Total code points: 80 +# ================================================ + +10940..10959 ; Sidetic # Lo [26] SIDETIC LETTER N01..SIDETIC LETTER N26 + +# Total code points: 26 + +# ================================================ + +1E6C0..1E6DE ; Tai_Yo # Lo [31] TAI YO LETTER LOW KO..TAI YO LETTER HIGH KVO +1E6E0..1E6E2 ; Tai_Yo # Lo [3] TAI YO LETTER AA..TAI YO LETTER UE +1E6E3 ; Tai_Yo # Mn TAI YO SIGN UE +1E6E4..1E6E5 ; Tai_Yo # Lo [2] TAI YO LETTER U..TAI YO LETTER AE +1E6E6 ; Tai_Yo # Mn TAI YO SIGN AU +1E6E7..1E6ED ; Tai_Yo # Lo [7] TAI YO LETTER O..TAI YO LETTER AUE +1E6EE..1E6EF ; Tai_Yo # Mn [2] TAI YO SIGN AY..TAI YO SIGN ANG +1E6F0..1E6F4 ; Tai_Yo # Lo [5] TAI YO LETTER AN..TAI YO LETTER AP +1E6F5 ; Tai_Yo # Mn TAI YO SIGN OM +1E6FE ; Tai_Yo # Lo TAI YO SYMBOL MUEANG +1E6FF ; Tai_Yo # Lm TAI YO XAM LAI + +# Total code points: 55 + +# ================================================ + +11DB0..11DD8 ; Tolong_Siki # Lo [41] TOLONG SIKI LETTER I..TOLONG SIKI LETTER RRH +11DD9 ; Tolong_Siki # Lm TOLONG SIKI SIGN SELA +11DDA..11DDB ; Tolong_Siki # Lo [2] TOLONG SIKI SIGN HECAKA..TOLONG SIKI UNGGA +11DE0..11DE9 ; Tolong_Siki # Nd [10] TOLONG SIKI DIGIT ZERO..TOLONG SIKI DIGIT NINE + +# Total code points: 54 + +# ================================================ + +16EA0..16EB8 ; Beria_Erfe # L& [25] BERIA ERFE CAPITAL LETTER ARKAB..BERIA ERFE CAPITAL LETTER AY +16EBB..16ED3 ; Beria_Erfe # L& [25] BERIA ERFE SMALL LETTER ARKAB..BERIA ERFE SMALL LETTER AY + +# Total code points: 50 + # EOF diff --git a/share/ucd/UnicodeData.txt b/share/ucd/UnicodeData.txt index 64258a373..fca68e3e1 100644 --- a/share/ucd/UnicodeData.txt +++ b/share/ucd/UnicodeData.txt @@ -659,7 +659,7 @@ 0292;LATIN SMALL LETTER EZH;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH;;01B7;;01B7 0293;LATIN SMALL LETTER EZH WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH CURL;;;; 0294;LATIN LETTER GLOTTAL STOP;Lo;0;L;;;;;N;;;;; -0295;LATIN LETTER PHARYNGEAL VOICED FRICATIVE;Ll;0;L;;;;;N;LATIN LETTER REVERSED GLOTTAL STOP;;;; +0295;LATIN LETTER PHARYNGEAL VOICED FRICATIVE;Lo;0;L;;;;;N;LATIN LETTER REVERSED GLOTTAL STOP;;;; 0296;LATIN LETTER INVERTED GLOTTAL STOP;Ll;0;L;;;;;N;;;;; 0297;LATIN LETTER STRETCHED C;Ll;0;L;;;;;N;;;;; 0298;LATIN LETTER BILABIAL CLICK;Ll;0;L;;;;;N;LATIN LETTER BULLSEYE;;;; @@ -2121,6 +2121,7 @@ 088C;ARABIC LETTER TAH WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;;;;; 088D;ARABIC LETTER KEHEH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 088E;ARABIC VERTICAL TAIL;Lo;0;AL;;;;;N;;;;; +088F;ARABIC LETTER NOON WITH RING ABOVE;Lo;0;AL;;;;;N;;;;; 0890;ARABIC POUND MARK ABOVE;Cf;0;AN;;;;;N;;;;; 0891;ARABIC PIASTRE MARK ABOVE;Cf;0;AN;;;;;N;;;;; 0897;ARABIC PEPET;Mn;230;NSM;;;;;N;;;;; @@ -2862,6 +2863,7 @@ 0C58;TELUGU LETTER TSA;Lo;0;L;;;;;N;;;;; 0C59;TELUGU LETTER DZA;Lo;0;L;;;;;N;;;;; 0C5A;TELUGU LETTER RRRA;Lo;0;L;;;;;N;;;;; +0C5C;TELUGU ARCHAIC SHRII;Lo;0;L;;;;;N;;;;; 0C5D;TELUGU LETTER NAKAARA POLLU;Lo;0;L;;;;;N;;;;; 0C60;TELUGU LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; 0C61;TELUGU LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; @@ -2958,6 +2960,7 @@ 0CCD;KANNADA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; 0CD5;KANNADA LENGTH MARK;Mc;0;L;;;;;N;;;;; 0CD6;KANNADA AI LENGTH MARK;Mc;0;L;;;;;N;;;;; +0CDC;KANNADA ARCHAIC SHRII;Lo;0;L;;;;;N;;;;; 0CDD;KANNADA LETTER NAKAARA POLLU;Lo;0;L;;;;;N;;;;; 0CDE;KANNADA LETTER FA;Lo;0;L;;;;;N;;;;; 0CE0;KANNADA LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; @@ -6137,6 +6140,33 @@ 1ACC;COMBINING LATIN SMALL LETTER INSULAR G;Mn;230;NSM;;;;;N;;;;; 1ACD;COMBINING LATIN SMALL LETTER INSULAR R;Mn;230;NSM;;;;;N;;;;; 1ACE;COMBINING LATIN SMALL LETTER INSULAR T;Mn;230;NSM;;;;;N;;;;; +1ACF;COMBINING DOUBLE CARON;Mn;230;NSM;;;;;N;;;;; +1AD0;COMBINING VERTICAL-LINE-ACUTE;Mn;230;NSM;;;;;N;;;;; +1AD1;COMBINING GRAVE-VERTICAL-LINE;Mn;230;NSM;;;;;N;;;;; +1AD2;COMBINING VERTICAL-LINE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1AD3;COMBINING ACUTE-VERTICAL-LINE;Mn;230;NSM;;;;;N;;;;; +1AD4;COMBINING VERTICAL-LINE-MACRON;Mn;230;NSM;;;;;N;;;;; +1AD5;COMBINING MACRON-VERTICAL-LINE;Mn;230;NSM;;;;;N;;;;; +1AD6;COMBINING VERTICAL-LINE-ACUTE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1AD7;COMBINING VERTICAL-LINE-GRAVE-ACUTE;Mn;230;NSM;;;;;N;;;;; +1AD8;COMBINING MACRON-ACUTE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1AD9;COMBINING SHARP SIGN;Mn;230;NSM;;;;;N;;;;; +1ADA;COMBINING FLAT SIGN;Mn;230;NSM;;;;;N;;;;; +1ADB;COMBINING DOWN TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1ADC;COMBINING DIAERESIS WITH RAISED LEFT DOT;Mn;230;NSM;;;;;N;;;;; +1ADD;COMBINING DOT-AND-RING BELOW;Mn;220;NSM;;;;;N;;;;; +1AE0;COMBINING LEFT TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE1;COMBINING RIGHT TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE2;COMBINING MINUS SIGN ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE3;COMBINING INVERTED BRIDGE ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE4;COMBINING SQUARE ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE5;COMBINING SEAGULL ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE6;COMBINING DOUBLE ARCH BELOW;Mn;220;NSM;;;;;N;;;;; +1AE7;COMBINING DOUBLE ARCH ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE8;COMBINING EQUALS SIGN ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE9;COMBINING LEFT ANGLE CENTRED ABOVE;Mn;230;NSM;;;;;N;;;;; +1AEA;COMBINING UPWARDS ARROW ABOVE;Mn;230;NSM;;;;;N;;;;; +1AEB;COMBINING DOUBLE RIGHTWARDS ARROW ABOVE;Mn;234;NSM;;;;;N;;;;; 1B00;BALINESE SIGN ULU RICEM;Mn;0;NSM;;;;;N;;;;; 1B01;BALINESE SIGN ULU CANDRA;Mn;0;NSM;;;;;N;;;;; 1B02;BALINESE SIGN CECEK;Mn;0;NSM;;;;;N;;;;; @@ -7545,6 +7575,7 @@ 20BE;LARI SIGN;Sc;0;ET;;;;;N;;;;; 20BF;BITCOIN SIGN;Sc;0;ET;;;;;N;;;;; 20C0;SOM SIGN;Sc;0;ET;;;;;N;;;;; +20C1;SAUDI RIYAL SIGN;Sc;0;ET;;;;;N;;;;; 20D0;COMBINING LEFT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT HARPOON ABOVE;;;; 20D1;COMBINING RIGHT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RIGHT HARPOON ABOVE;;;; 20D2;COMBINING LONG VERTICAL LINE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG VERTICAL BAR OVERLAY;;;; @@ -10239,6 +10270,7 @@ 2B93;NEWLINE RIGHT;So;0;ON;;;;;N;;;;; 2B94;FOUR CORNER ARROWS CIRCLING ANTICLOCKWISE;So;0;ON;;;;;N;;;;; 2B95;RIGHTWARDS BLACK ARROW;So;0;ON;;;;;N;;;;; +2B96;EQUALS SIGN WITH INFINITY ABOVE;So;0;ON;;;;;N;;;;; 2B97;SYMBOL FOR TYPE A ELECTRONICS;So;0;ON;;;;;N;;;;; 2B98;THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD;So;0;ON;;;;;N;;;;; 2B99;THREE-D RIGHT-LIGHTED UPWARDS EQUILATERAL ARROWHEAD;So;0;ON;;;;;N;;;;; @@ -14274,10 +14306,14 @@ A7CA;LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY;Ll;0;L;;;;;N;;;A7C9;;A7C9 A7CB;LATIN CAPITAL LETTER RAMS HORN;Lu;0;L;;;;;N;;;;0264; A7CC;LATIN CAPITAL LETTER S WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A7CD; A7CD;LATIN SMALL LETTER S WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;A7CC;;A7CC +A7CE;LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE;Lu;0;L;;;;;N;;;;A7CF; +A7CF;LATIN SMALL LETTER PHARYNGEAL VOICED FRICATIVE;Ll;0;L;;;;;N;;;A7CE;;A7CE A7D0;LATIN CAPITAL LETTER CLOSED INSULAR G;Lu;0;L;;;;;N;;;;A7D1; A7D1;LATIN SMALL LETTER CLOSED INSULAR G;Ll;0;L;;;;;N;;;A7D0;;A7D0 -A7D3;LATIN SMALL LETTER DOUBLE THORN;Ll;0;L;;;;;N;;;;; -A7D5;LATIN SMALL LETTER DOUBLE WYNN;Ll;0;L;;;;;N;;;;; +A7D2;LATIN CAPITAL LETTER DOUBLE THORN;Lu;0;L;;;;;N;;;;A7D3; +A7D3;LATIN SMALL LETTER DOUBLE THORN;Ll;0;L;;;;;N;;;A7D2;;A7D2 +A7D4;LATIN CAPITAL LETTER DOUBLE WYNN;Lu;0;L;;;;;N;;;;A7D5; +A7D5;LATIN SMALL LETTER DOUBLE WYNN;Ll;0;L;;;;;N;;;A7D4;;A7D4 A7D6;LATIN CAPITAL LETTER MIDDLE SCOTS S;Lu;0;L;;;;;N;;;;A7D7; A7D7;LATIN SMALL LETTER MIDDLE SCOTS S;Ll;0;L;;;;;N;;;A7D6;;A7D6 A7D8;LATIN CAPITAL LETTER SIGMOID S;Lu;0;L;;;;;N;;;;A7D9; @@ -14285,6 +14321,7 @@ A7D9;LATIN SMALL LETTER SIGMOID S;Ll;0;L;;;;;N;;;A7D8;;A7D8 A7DA;LATIN CAPITAL LETTER LAMBDA;Lu;0;L;;;;;N;;;;A7DB; A7DB;LATIN SMALL LETTER LAMBDA;Ll;0;L;;;;;N;;;A7DA;;A7DA A7DC;LATIN CAPITAL LETTER LAMBDA WITH STROKE;Lu;0;L;;;;;N;;;;019B; +A7F1;MODIFIER LETTER CAPITAL S;Lm;0;L; 0053;;;;N;;;;; A7F2;MODIFIER LETTER CAPITAL C;Lm;0;L; 0043;;;;N;;;;; A7F3;MODIFIER LETTER CAPITAL F;Lm;0;L; 0046;;;;N;;;;; A7F4;MODIFIER LETTER CAPITAL Q;Lm;0;L; 0051;;;;N;;;;; @@ -15925,6 +15962,22 @@ FBBF;ARABIC SYMBOL RING;Sk;0;AL;;;;;N;;;;; FBC0;ARABIC SYMBOL SMALL TAH ABOVE;Sk;0;AL;;;;;N;;;;; FBC1;ARABIC SYMBOL SMALL TAH BELOW;Sk;0;AL;;;;;N;;;;; FBC2;ARABIC SYMBOL WASLA ABOVE;Sk;0;AL;;;;;N;;;;; +FBC3;ARABIC LIGATURE JALLA WA-ALAA;So;0;ON;;;;;N;;;;; +FBC4;ARABIC LIGATURE DAAMAT BARAKAATUHUM;So;0;ON;;;;;N;;;;; +FBC5;ARABIC LIGATURE RAHMATU ALLAAHI TAAALAA ALAYH;So;0;ON;;;;;N;;;;; +FBC6;ARABIC LIGATURE RAHMATU ALLAAHI ALAYHIM;So;0;ON;;;;;N;;;;; +FBC7;ARABIC LIGATURE RAHMATU ALLAAHI ALAYHIMAA;So;0;ON;;;;;N;;;;; +FBC8;ARABIC LIGATURE RAHIMAHUM ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBC9;ARABIC LIGATURE RAHIMAHUMAA ALLAAH;So;0;ON;;;;;N;;;;; +FBCA;ARABIC LIGATURE RAHIMAHUMAA ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBCB;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANHUM;So;0;ON;;;;;N;;;;; +FBCC;ARABIC LIGATURE HAFIZAHU ALLAAH;So;0;ON;;;;;N;;;;; +FBCD;ARABIC LIGATURE HAFIZAHU ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBCE;ARABIC LIGATURE HAFIZAHUM ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBCF;ARABIC LIGATURE HAFIZAHUMAA ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBD0;ARABIC LIGATURE SALLALLAAHU TAAALAA ALAYHI WA-SALLAM;So;0;ON;;;;;N;;;;; +FBD1;ARABIC LIGATURE AJJAL ALLAAHU FARAJAHU ASH-SHAREEF;So;0;ON;;;;;N;;;;; +FBD2;ARABIC LIGATURE ALAYHI AR-RAHMAH;So;0;ON;;;;;N;;;;; FBD3;ARABIC LETTER NG ISOLATED FORM;Lo;0;AL; 06AD;;;;N;;;;; FBD4;ARABIC LETTER NG FINAL FORM;Lo;0;AL; 06AD;;;;N;;;;; FBD5;ARABIC LETTER NG INITIAL FORM;Lo;0;AL; 06AD;;;;N;;;;; @@ -16370,6 +16423,8 @@ FD8C;ARABIC LIGATURE MEEM WITH JEEM WITH HAH INITIAL FORM;Lo;0;AL; 0645 FD8D;ARABIC LIGATURE MEEM WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062C 0645;;;;N;;;;; FD8E;ARABIC LIGATURE MEEM WITH KHAH WITH JEEM INITIAL FORM;Lo;0;AL; 0645 062E 062C;;;;N;;;;; FD8F;ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062E 0645;;;;N;;;;; +FD90;ARABIC LIGATURE RAHMATU ALLAAHI ALAYH;So;0;ON;;;;;N;;;;; +FD91;ARABIC LIGATURE RAHMATU ALLAAHI ALAYHAA;So;0;ON;;;;;N;;;;; FD92;ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM;Lo;0;AL; 0645 062C 062E;;;;N;;;;; FD93;ARABIC LIGATURE HEH WITH MEEM WITH JEEM INITIAL FORM;Lo;0;AL; 0647 0645 062C;;;;N;;;;; FD94;ARABIC LIGATURE HEH WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0647 0645 0645;;;;N;;;;; @@ -16424,6 +16479,13 @@ FDC4;ARABIC LIGATURE AIN WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0639 FDC5;ARABIC LIGATURE SAD WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0635 0645 0645;;;;N;;;;; FDC6;ARABIC LIGATURE SEEN WITH KHAH WITH YEH FINAL FORM;Lo;0;AL; 0633 062E 064A;;;;N;;;;; FDC7;ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 0646 062C 064A;;;;N;;;;; +FDC8;ARABIC LIGATURE RAHIMAHU ALLAAH TAAALAA;So;0;ON;;;;;N;;;;; +FDC9;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANH;So;0;ON;;;;;N;;;;; +FDCA;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANHAA;So;0;ON;;;;;N;;;;; +FDCB;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANHUMAA;So;0;ON;;;;;N;;;;; +FDCC;ARABIC LIGATURE SALLALLAHU ALAYHI WA-ALAA AALIHEE WA-SALLAM;So;0;ON;;;;;N;;;;; +FDCD;ARABIC LIGATURE AJJAL ALLAAHU TAAALAA FARAJAHU ASH-SHAREEF;So;0;ON;;;;;N;;;;; +FDCE;ARABIC LIGATURE KARRAMA ALLAAHU WAJHAH;So;0;ON;;;;;N;;;;; FDCF;ARABIC LIGATURE SALAAMUHU ALAYNAA;So;0;ON;;;;;N;;;;; FDF0;ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM;Lo;0;AL; 0635 0644 06D2;;;;N;;;;; FDF1;ARABIC LIGATURE QALA USED AS KORANIC STOP SIGN ISOLATED FORM;Lo;0;AL; 0642 0644 06D2;;;;N;;;;; @@ -18708,6 +18770,32 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10938;LYDIAN LETTER NN;Lo;0;R;;;;;N;;;;; 10939;LYDIAN LETTER C;Lo;0;R;;;;;N;;;;; 1093F;LYDIAN TRIANGULAR MARK;Po;0;R;;;;;N;;;;; +10940;SIDETIC LETTER N01;Lo;0;R;;;;;N;;;;; +10941;SIDETIC LETTER N02;Lo;0;R;;;;;N;;;;; +10942;SIDETIC LETTER N03;Lo;0;R;;;;;N;;;;; +10943;SIDETIC LETTER N04;Lo;0;R;;;;;N;;;;; +10944;SIDETIC LETTER N05;Lo;0;R;;;;;N;;;;; +10945;SIDETIC LETTER N06;Lo;0;R;;;;;N;;;;; +10946;SIDETIC LETTER N07;Lo;0;R;;;;;N;;;;; +10947;SIDETIC LETTER N08;Lo;0;R;;;;;N;;;;; +10948;SIDETIC LETTER N09;Lo;0;R;;;;;N;;;;; +10949;SIDETIC LETTER N10;Lo;0;R;;;;;N;;;;; +1094A;SIDETIC LETTER N11;Lo;0;R;;;;;N;;;;; +1094B;SIDETIC LETTER N12;Lo;0;R;;;;;N;;;;; +1094C;SIDETIC LETTER N13;Lo;0;R;;;;;N;;;;; +1094D;SIDETIC LETTER N14;Lo;0;R;;;;;N;;;;; +1094E;SIDETIC LETTER N15;Lo;0;R;;;;;N;;;;; +1094F;SIDETIC LETTER N16;Lo;0;R;;;;;N;;;;; +10950;SIDETIC LETTER N17;Lo;0;R;;;;;N;;;;; +10951;SIDETIC LETTER N18;Lo;0;R;;;;;N;;;;; +10952;SIDETIC LETTER N19;Lo;0;R;;;;;N;;;;; +10953;SIDETIC LETTER N20;Lo;0;R;;;;;N;;;;; +10954;SIDETIC LETTER N21;Lo;0;R;;;;;N;;;;; +10955;SIDETIC LETTER N22;Lo;0;R;;;;;N;;;;; +10956;SIDETIC LETTER N23;Lo;0;R;;;;;N;;;;; +10957;SIDETIC LETTER N24;Lo;0;R;;;;;N;;;;; +10958;SIDETIC LETTER N25;Lo;0;R;;;;;N;;;;; +10959;SIDETIC LETTER N26;Lo;0;R;;;;;N;;;;; 10980;MEROITIC HIEROGLYPHIC LETTER A;Lo;0;R;;;;;N;;;;; 10981;MEROITIC HIEROGLYPHIC LETTER E;Lo;0;R;;;;;N;;;;; 10982;MEROITIC HIEROGLYPHIC LETTER I;Lo;0;R;;;;;N;;;;; @@ -19541,6 +19629,20 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EC2;ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EC3;ARABIC LETTER TAH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EC4;ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC5;ARABIC SMALL YEH BARREE WITH TWO DOTS BELOW;Lm;0;AL;;;;;N;;;;; +10EC6;ARABIC LETTER THIN NOON;Lo;0;AL;;;;;N;;;;; +10EC7;ARABIC LETTER YEH WITH FOUR DOTS BELOW;Lo;0;AL;;;;;N;;;;; +10ED0;ARABIC BIBLICAL END OF VERSE;Po;0;ON;;;;;N;;;;; +10ED1;ARABIC LIGATURE ALAYHAA AS-SALAATU WAS-SALAAM;So;0;ON;;;;;N;;;;; +10ED2;ARABIC LIGATURE ALAYHIM AS-SALAATU WAS-SALAAM;So;0;ON;;;;;N;;;;; +10ED3;ARABIC LIGATURE ALAYHIMAA AS-SALAATU WAS-SALAAM;So;0;ON;;;;;N;;;;; +10ED4;ARABIC LIGATURE QADDASA ALLAAHU SIRRAH;So;0;ON;;;;;N;;;;; +10ED5;ARABIC LIGATURE QUDDISA SIRRUHUM;So;0;ON;;;;;N;;;;; +10ED6;ARABIC LIGATURE QUDDISA SIRRUHUMAA;So;0;ON;;;;;N;;;;; +10ED7;ARABIC LIGATURE QUDDISAT ASRAARUHUM;So;0;ON;;;;;N;;;;; +10ED8;ARABIC LIGATURE NAWWARA ALLAAHU MARQADAH;So;0;ON;;;;;N;;;;; +10EFA;ARABIC DOUBLE VERTICAL BAR BELOW;Mn;220;NSM;;;;;N;;;;; +10EFB;ARABIC SMALL LOW NOON;Mn;220;NSM;;;;;N;;;;; 10EFC;ARABIC COMBINING ALEF OVERLAY;Mn;0;NSM;;;;;N;;;;; 10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; 10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; @@ -21521,6 +21623,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; 11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; 11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; +11B60;SHARADA VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; +11B61;SHARADA VOWEL SIGN OOE;Mc;0;L;;;;;N;;;;; +11B62;SHARADA VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +11B63;SHARADA VOWEL SIGN UUE;Mn;0;NSM;;;;;N;;;;; +11B64;SHARADA VOWEL SIGN SHORT E;Mn;0;NSM;;;;;N;;;;; +11B65;SHARADA VOWEL SIGN SHORT O;Mc;0;L;;;;;N;;;;; +11B66;SHARADA VOWEL SIGN CANDRA E;Mn;0;NSM;;;;;N;;;;; +11B67;SHARADA VOWEL SIGN CANDRA O;Mc;0;L;;;;;N;;;;; 11BC0;SUNUWAR LETTER DEVI;Lo;0;L;;;;;N;;;;; 11BC1;SUNUWAR LETTER TASLA;Lo;0;L;;;;;N;;;;; 11BC2;SUNUWAR LETTER EKO;Lo;0;L;;;;;N;;;;; @@ -21868,6 +21978,60 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11DA7;GUNJALA GONDI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 11DA8;GUNJALA GONDI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 11DA9;GUNJALA GONDI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11DB0;TOLONG SIKI LETTER I;Lo;0;L;;;;;N;;;;; +11DB1;TOLONG SIKI LETTER E;Lo;0;L;;;;;N;;;;; +11DB2;TOLONG SIKI LETTER U;Lo;0;L;;;;;N;;;;; +11DB3;TOLONG SIKI LETTER O;Lo;0;L;;;;;N;;;;; +11DB4;TOLONG SIKI LETTER A;Lo;0;L;;;;;N;;;;; +11DB5;TOLONG SIKI LETTER AA;Lo;0;L;;;;;N;;;;; +11DB6;TOLONG SIKI LETTER P;Lo;0;L;;;;;N;;;;; +11DB7;TOLONG SIKI LETTER PH;Lo;0;L;;;;;N;;;;; +11DB8;TOLONG SIKI LETTER B;Lo;0;L;;;;;N;;;;; +11DB9;TOLONG SIKI LETTER BH;Lo;0;L;;;;;N;;;;; +11DBA;TOLONG SIKI LETTER M;Lo;0;L;;;;;N;;;;; +11DBB;TOLONG SIKI LETTER T;Lo;0;L;;;;;N;;;;; +11DBC;TOLONG SIKI LETTER TH;Lo;0;L;;;;;N;;;;; +11DBD;TOLONG SIKI LETTER D;Lo;0;L;;;;;N;;;;; +11DBE;TOLONG SIKI LETTER DH;Lo;0;L;;;;;N;;;;; +11DBF;TOLONG SIKI LETTER N;Lo;0;L;;;;;N;;;;; +11DC0;TOLONG SIKI LETTER TT;Lo;0;L;;;;;N;;;;; +11DC1;TOLONG SIKI LETTER TTH;Lo;0;L;;;;;N;;;;; +11DC2;TOLONG SIKI LETTER DD;Lo;0;L;;;;;N;;;;; +11DC3;TOLONG SIKI LETTER DDH;Lo;0;L;;;;;N;;;;; +11DC4;TOLONG SIKI LETTER NN;Lo;0;L;;;;;N;;;;; +11DC5;TOLONG SIKI LETTER C;Lo;0;L;;;;;N;;;;; +11DC6;TOLONG SIKI LETTER CH;Lo;0;L;;;;;N;;;;; +11DC7;TOLONG SIKI LETTER J;Lo;0;L;;;;;N;;;;; +11DC8;TOLONG SIKI LETTER JH;Lo;0;L;;;;;N;;;;; +11DC9;TOLONG SIKI LETTER NY;Lo;0;L;;;;;N;;;;; +11DCA;TOLONG SIKI LETTER K;Lo;0;L;;;;;N;;;;; +11DCB;TOLONG SIKI LETTER KH;Lo;0;L;;;;;N;;;;; +11DCC;TOLONG SIKI LETTER G;Lo;0;L;;;;;N;;;;; +11DCD;TOLONG SIKI LETTER GH;Lo;0;L;;;;;N;;;;; +11DCE;TOLONG SIKI LETTER NG;Lo;0;L;;;;;N;;;;; +11DCF;TOLONG SIKI LETTER Y;Lo;0;L;;;;;N;;;;; +11DD0;TOLONG SIKI LETTER R;Lo;0;L;;;;;N;;;;; +11DD1;TOLONG SIKI LETTER L;Lo;0;L;;;;;N;;;;; +11DD2;TOLONG SIKI LETTER V;Lo;0;L;;;;;N;;;;; +11DD3;TOLONG SIKI LETTER NNY;Lo;0;L;;;;;N;;;;; +11DD4;TOLONG SIKI LETTER S;Lo;0;L;;;;;N;;;;; +11DD5;TOLONG SIKI LETTER H;Lo;0;L;;;;;N;;;;; +11DD6;TOLONG SIKI LETTER X;Lo;0;L;;;;;N;;;;; +11DD7;TOLONG SIKI LETTER RR;Lo;0;L;;;;;N;;;;; +11DD8;TOLONG SIKI LETTER RRH;Lo;0;L;;;;;N;;;;; +11DD9;TOLONG SIKI SIGN SELA;Lm;0;L;;;;;N;;;;; +11DDA;TOLONG SIKI SIGN HECAKA;Lo;0;L;;;;;N;;;;; +11DDB;TOLONG SIKI UNGGA;Lo;0;L;;;;;N;;;;; +11DE0;TOLONG SIKI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11DE1;TOLONG SIKI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11DE2;TOLONG SIKI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11DE3;TOLONG SIKI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11DE4;TOLONG SIKI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11DE5;TOLONG SIKI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11DE6;TOLONG SIKI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11DE7;TOLONG SIKI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11DE8;TOLONG SIKI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11DE9;TOLONG SIKI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11EE0;MAKASAR LETTER KA;Lo;0;L;;;;;N;;;;; 11EE1;MAKASAR LETTER GA;Lo;0;L;;;;;N;;;;; 11EE2;MAKASAR LETTER NGA;Lo;0;L;;;;;N;;;;; @@ -22088,8 +22252,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12035;CUNEIFORM SIGN ARAD TIMES KUR;Lo;0;L;;;;;N;;;;; 12036;CUNEIFORM SIGN ARKAB;Lo;0;L;;;;;N;;;;; 12037;CUNEIFORM SIGN ASAL2;Lo;0;L;;;;;N;;;;; -12038;CUNEIFORM SIGN ASH;Lo;0;L;;;;;N;;;;; -12039;CUNEIFORM SIGN ASH ZIDA TENU;Lo;0;L;;;;;N;;;;; +12038;CUNEIFORM SIGN ASH;Lo;0;L;;;;1;N;;;;; +12039;CUNEIFORM SIGN ASH ZIDA TENU;Lo;0;L;;;;1;N;;;;; 1203A;CUNEIFORM SIGN ASH KABA TENU;Lo;0;L;;;;;N;;;;; 1203B;CUNEIFORM SIGN ASH OVER ASH TUG2 OVER TUG2 TUG2 OVER TUG2 PAP;Lo;0;L;;;;;N;;;;; 1203C;CUNEIFORM SIGN ASH OVER ASH OVER ASH;Lo;0;L;;;;;N;;;;; @@ -22153,7 +22317,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12076;CUNEIFORM SIGN DIM2;Lo;0;L;;;;;N;;;;; 12077;CUNEIFORM SIGN DIN;Lo;0;L;;;;;N;;;;; 12078;CUNEIFORM SIGN DIN KASKAL U GUNU DISH;Lo;0;L;;;;;N;;;;; -12079;CUNEIFORM SIGN DISH;Lo;0;L;;;;;N;;;;; +12079;CUNEIFORM SIGN DISH;Lo;0;L;;;;1;N;;;;; 1207A;CUNEIFORM SIGN DU;Lo;0;L;;;;;N;;;;; 1207B;CUNEIFORM SIGN DU OVER DU;Lo;0;L;;;;;N;;;;; 1207C;CUNEIFORM SIGN DU GUNU;Lo;0;L;;;;;N;;;;; @@ -22582,12 +22746,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12223;CUNEIFORM SIGN MA2;Lo;0;L;;;;;N;;;;; 12224;CUNEIFORM SIGN MAH;Lo;0;L;;;;;N;;;;; 12225;CUNEIFORM SIGN MAR;Lo;0;L;;;;;N;;;;; -12226;CUNEIFORM SIGN MASH;Lo;0;L;;;;;N;;;;; +12226;CUNEIFORM SIGN MASH;Lo;0;L;;;;1/2;N;;;;; 12227;CUNEIFORM SIGN MASH2;Lo;0;L;;;;;N;;;;; 12228;CUNEIFORM SIGN ME;Lo;0;L;;;;;N;;;;; 12229;CUNEIFORM SIGN MES;Lo;0;L;;;;;N;;;;; 1222A;CUNEIFORM SIGN MI;Lo;0;L;;;;;N;;;;; -1222B;CUNEIFORM SIGN MIN;Lo;0;L;;;;;N;;;;; +1222B;CUNEIFORM SIGN MIN;Lo;0;L;;;;2;N;;;;; 1222C;CUNEIFORM SIGN MU;Lo;0;L;;;;;N;;;;; 1222D;CUNEIFORM SIGN MU OVER MU;Lo;0;L;;;;;N;;;;; 1222E;CUNEIFORM SIGN MUG;Lo;0;L;;;;;N;;;;; @@ -22811,9 +22975,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12308;CUNEIFORM SIGN TUM;Lo;0;L;;;;;N;;;;; 12309;CUNEIFORM SIGN TUR;Lo;0;L;;;;;N;;;;; 1230A;CUNEIFORM SIGN TUR OVER TUR ZA OVER ZA;Lo;0;L;;;;;N;;;;; -1230B;CUNEIFORM SIGN U;Lo;0;L;;;;;N;;;;; +1230B;CUNEIFORM SIGN U;Lo;0;L;;;;1;N;;;;; 1230C;CUNEIFORM SIGN U GUD;Lo;0;L;;;;;N;;;;; -1230D;CUNEIFORM SIGN U U U;Lo;0;L;;;;;N;;;;; +1230D;CUNEIFORM SIGN U U U;Lo;0;L;;;;3;N;;;;; 1230E;CUNEIFORM SIGN U OVER U PA OVER PA GAR OVER GAR;Lo;0;L;;;;;N;;;;; 1230F;CUNEIFORM SIGN U OVER U SUR OVER SUR;Lo;0;L;;;;;N;;;;; 12310;CUNEIFORM SIGN U OVER U U REVERSED OVER U REVERSED;Lo;0;L;;;;;N;;;;; @@ -22953,7 +23117,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12396;CUNEIFORM SIGN SAG TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; 12397;CUNEIFORM SIGN TI2;Lo;0;L;;;;;N;;;;; 12398;CUNEIFORM SIGN UM TIMES ME;Lo;0;L;;;;;N;;;;; -12399;CUNEIFORM SIGN U U;Lo;0;L;;;;;N;;;;; +12399;CUNEIFORM SIGN U U;Lo;0;L;;;;2;N;;;;; 12400;CUNEIFORM NUMERIC SIGN TWO ASH;Nl;0;L;;;;2;N;;;;; 12401;CUNEIFORM NUMERIC SIGN THREE ASH;Nl;0;L;;;;3;N;;;;; 12402;CUNEIFORM NUMERIC SIGN FOUR ASH;Nl;0;L;;;;4;N;;;;; @@ -30124,6 +30288,56 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16E98;MEDEFAIDRIN FULL STOP;Po;0;L;;;;;N;;;;; 16E99;MEDEFAIDRIN SYMBOL AIVA;Po;0;L;;;;;N;;;;; 16E9A;MEDEFAIDRIN EXCLAMATION OH;Po;0;L;;;;;N;;;;; +16EA0;BERIA ERFE CAPITAL LETTER ARKAB;Lu;0;L;;;;;N;;;;16EBB; +16EA1;BERIA ERFE CAPITAL LETTER BASIGNA;Lu;0;L;;;;;N;;;;16EBC; +16EA2;BERIA ERFE CAPITAL LETTER DARBAI;Lu;0;L;;;;;N;;;;16EBD; +16EA3;BERIA ERFE CAPITAL LETTER EH;Lu;0;L;;;;;N;;;;16EBE; +16EA4;BERIA ERFE CAPITAL LETTER FITKO;Lu;0;L;;;;;N;;;;16EBF; +16EA5;BERIA ERFE CAPITAL LETTER GOWAY;Lu;0;L;;;;;N;;;;16EC0; +16EA6;BERIA ERFE CAPITAL LETTER HIRDEABO;Lu;0;L;;;;;N;;;;16EC1; +16EA7;BERIA ERFE CAPITAL LETTER I;Lu;0;L;;;;;N;;;;16EC2; +16EA8;BERIA ERFE CAPITAL LETTER DJAI;Lu;0;L;;;;;N;;;;16EC3; +16EA9;BERIA ERFE CAPITAL LETTER KOBO;Lu;0;L;;;;;N;;;;16EC4; +16EAA;BERIA ERFE CAPITAL LETTER LAKKO;Lu;0;L;;;;;N;;;;16EC5; +16EAB;BERIA ERFE CAPITAL LETTER MERI;Lu;0;L;;;;;N;;;;16EC6; +16EAC;BERIA ERFE CAPITAL LETTER NINI;Lu;0;L;;;;;N;;;;16EC7; +16EAD;BERIA ERFE CAPITAL LETTER GNA;Lu;0;L;;;;;N;;;;16EC8; +16EAE;BERIA ERFE CAPITAL LETTER NGAY;Lu;0;L;;;;;N;;;;16EC9; +16EAF;BERIA ERFE CAPITAL LETTER OI;Lu;0;L;;;;;N;;;;16ECA; +16EB0;BERIA ERFE CAPITAL LETTER PI;Lu;0;L;;;;;N;;;;16ECB; +16EB1;BERIA ERFE CAPITAL LETTER ERIGO;Lu;0;L;;;;;N;;;;16ECC; +16EB2;BERIA ERFE CAPITAL LETTER ERIGO TAMURA;Lu;0;L;;;;;N;;;;16ECD; +16EB3;BERIA ERFE CAPITAL LETTER SERI;Lu;0;L;;;;;N;;;;16ECE; +16EB4;BERIA ERFE CAPITAL LETTER SHEP;Lu;0;L;;;;;N;;;;16ECF; +16EB5;BERIA ERFE CAPITAL LETTER TATASOUE;Lu;0;L;;;;;N;;;;16ED0; +16EB6;BERIA ERFE CAPITAL LETTER UI;Lu;0;L;;;;;N;;;;16ED1; +16EB7;BERIA ERFE CAPITAL LETTER WASSE;Lu;0;L;;;;;N;;;;16ED2; +16EB8;BERIA ERFE CAPITAL LETTER AY;Lu;0;L;;;;;N;;;;16ED3; +16EBB;BERIA ERFE SMALL LETTER ARKAB;Ll;0;L;;;;;N;;;16EA0;;16EA0 +16EBC;BERIA ERFE SMALL LETTER BASIGNA;Ll;0;L;;;;;N;;;16EA1;;16EA1 +16EBD;BERIA ERFE SMALL LETTER DARBAI;Ll;0;L;;;;;N;;;16EA2;;16EA2 +16EBE;BERIA ERFE SMALL LETTER EH;Ll;0;L;;;;;N;;;16EA3;;16EA3 +16EBF;BERIA ERFE SMALL LETTER FITKO;Ll;0;L;;;;;N;;;16EA4;;16EA4 +16EC0;BERIA ERFE SMALL LETTER GOWAY;Ll;0;L;;;;;N;;;16EA5;;16EA5 +16EC1;BERIA ERFE SMALL LETTER HIRDEABO;Ll;0;L;;;;;N;;;16EA6;;16EA6 +16EC2;BERIA ERFE SMALL LETTER I;Ll;0;L;;;;;N;;;16EA7;;16EA7 +16EC3;BERIA ERFE SMALL LETTER DJAI;Ll;0;L;;;;;N;;;16EA8;;16EA8 +16EC4;BERIA ERFE SMALL LETTER KOBO;Ll;0;L;;;;;N;;;16EA9;;16EA9 +16EC5;BERIA ERFE SMALL LETTER LAKKO;Ll;0;L;;;;;N;;;16EAA;;16EAA +16EC6;BERIA ERFE SMALL LETTER MERI;Ll;0;L;;;;;N;;;16EAB;;16EAB +16EC7;BERIA ERFE SMALL LETTER NINI;Ll;0;L;;;;;N;;;16EAC;;16EAC +16EC8;BERIA ERFE SMALL LETTER GNA;Ll;0;L;;;;;N;;;16EAD;;16EAD +16EC9;BERIA ERFE SMALL LETTER NGAY;Ll;0;L;;;;;N;;;16EAE;;16EAE +16ECA;BERIA ERFE SMALL LETTER OI;Ll;0;L;;;;;N;;;16EAF;;16EAF +16ECB;BERIA ERFE SMALL LETTER PI;Ll;0;L;;;;;N;;;16EB0;;16EB0 +16ECC;BERIA ERFE SMALL LETTER ERIGO;Ll;0;L;;;;;N;;;16EB1;;16EB1 +16ECD;BERIA ERFE SMALL LETTER ERIGO TAMURA;Ll;0;L;;;;;N;;;16EB2;;16EB2 +16ECE;BERIA ERFE SMALL LETTER SERI;Ll;0;L;;;;;N;;;16EB3;;16EB3 +16ECF;BERIA ERFE SMALL LETTER SHEP;Ll;0;L;;;;;N;;;16EB4;;16EB4 +16ED0;BERIA ERFE SMALL LETTER TATASOUE;Ll;0;L;;;;;N;;;16EB5;;16EB5 +16ED1;BERIA ERFE SMALL LETTER UI;Ll;0;L;;;;;N;;;16EB6;;16EB6 +16ED2;BERIA ERFE SMALL LETTER WASSE;Ll;0;L;;;;;N;;;16EB7;;16EB7 +16ED3;BERIA ERFE SMALL LETTER AY;Ll;0;L;;;;;N;;;16EB8;;16EB8 16F00;MIAO LETTER PA;Lo;0;L;;;;;N;;;;; 16F01;MIAO LETTER BA;Lo;0;L;;;;;N;;;;; 16F02;MIAO LETTER YI PA;Lo;0;L;;;;;N;;;;; @@ -30280,8 +30494,13 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16FE4;KHITAN SMALL SCRIPT FILLER;Mn;0;NSM;;;;;N;;;;; 16FF0;VIETNAMESE ALTERNATE READING MARK CA;Mc;6;L;;;;;N;;;;; 16FF1;VIETNAMESE ALTERNATE READING MARK NHAY;Mc;6;L;;;;;N;;;;; +16FF2;CHINESE SMALL SIMPLIFIED ER;Lm;0;L;;;;;N;;;;; +16FF3;CHINESE SMALL TRADITIONAL ER;Lm;0;L;;;;;N;;;;; +16FF4;YANGQIN SIGN SLOW ONE BEAT;Nl;0;L;;;;1;N;;;;; +16FF5;YANGQIN SIGN SLOW THREE HALF BEATS;Nl;0;L;;;;3/2;N;;;;; +16FF6;YANGQIN SIGN SLOW TWO BEATS;Nl;0;L;;;;2;N;;;;; 17000;;Lo;0;L;;;;;N;;;;; -187F7;;Lo;0;L;;;;;N;;;;; +187FF;;Lo;0;L;;;;;N;;;;; 18800;TANGUT COMPONENT-001;Lo;0;L;;;;;N;;;;; 18801;TANGUT COMPONENT-002;Lo;0;L;;;;;N;;;;; 18802;TANGUT COMPONENT-003;Lo;0;L;;;;;N;;;;; @@ -31522,7 +31741,122 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 18CD5;KHITAN SMALL SCRIPT CHARACTER-18CD5;Lo;0;L;;;;;N;;;;; 18CFF;KHITAN SMALL SCRIPT CHARACTER-18CFF;Lo;0;L;;;;;N;;;;; 18D00;;Lo;0;L;;;;;N;;;;; -18D08;;Lo;0;L;;;;;N;;;;; +18D1E;;Lo;0;L;;;;;N;;;;; +18D80;TANGUT COMPONENT-769;Lo;0;L;;;;;N;;;;; +18D81;TANGUT COMPONENT-770;Lo;0;L;;;;;N;;;;; +18D82;TANGUT COMPONENT-771;Lo;0;L;;;;;N;;;;; +18D83;TANGUT COMPONENT-772;Lo;0;L;;;;;N;;;;; +18D84;TANGUT COMPONENT-773;Lo;0;L;;;;;N;;;;; +18D85;TANGUT COMPONENT-774;Lo;0;L;;;;;N;;;;; +18D86;TANGUT COMPONENT-775;Lo;0;L;;;;;N;;;;; +18D87;TANGUT COMPONENT-776;Lo;0;L;;;;;N;;;;; +18D88;TANGUT COMPONENT-777;Lo;0;L;;;;;N;;;;; +18D89;TANGUT COMPONENT-778;Lo;0;L;;;;;N;;;;; +18D8A;TANGUT COMPONENT-779;Lo;0;L;;;;;N;;;;; +18D8B;TANGUT COMPONENT-780;Lo;0;L;;;;;N;;;;; +18D8C;TANGUT COMPONENT-781;Lo;0;L;;;;;N;;;;; +18D8D;TANGUT COMPONENT-782;Lo;0;L;;;;;N;;;;; +18D8E;TANGUT COMPONENT-783;Lo;0;L;;;;;N;;;;; +18D8F;TANGUT COMPONENT-784;Lo;0;L;;;;;N;;;;; +18D90;TANGUT COMPONENT-785;Lo;0;L;;;;;N;;;;; +18D91;TANGUT COMPONENT-786;Lo;0;L;;;;;N;;;;; +18D92;TANGUT COMPONENT-787;Lo;0;L;;;;;N;;;;; +18D93;TANGUT COMPONENT-788;Lo;0;L;;;;;N;;;;; +18D94;TANGUT COMPONENT-789;Lo;0;L;;;;;N;;;;; +18D95;TANGUT COMPONENT-790;Lo;0;L;;;;;N;;;;; +18D96;TANGUT COMPONENT-791;Lo;0;L;;;;;N;;;;; +18D97;TANGUT COMPONENT-792;Lo;0;L;;;;;N;;;;; +18D98;TANGUT COMPONENT-793;Lo;0;L;;;;;N;;;;; +18D99;TANGUT COMPONENT-794;Lo;0;L;;;;;N;;;;; +18D9A;TANGUT COMPONENT-795;Lo;0;L;;;;;N;;;;; +18D9B;TANGUT COMPONENT-796;Lo;0;L;;;;;N;;;;; +18D9C;TANGUT COMPONENT-797;Lo;0;L;;;;;N;;;;; +18D9D;TANGUT COMPONENT-798;Lo;0;L;;;;;N;;;;; +18D9E;TANGUT COMPONENT-799;Lo;0;L;;;;;N;;;;; +18D9F;TANGUT COMPONENT-800;Lo;0;L;;;;;N;;;;; +18DA0;TANGUT COMPONENT-801;Lo;0;L;;;;;N;;;;; +18DA1;TANGUT COMPONENT-802;Lo;0;L;;;;;N;;;;; +18DA2;TANGUT COMPONENT-803;Lo;0;L;;;;;N;;;;; +18DA3;TANGUT COMPONENT-804;Lo;0;L;;;;;N;;;;; +18DA4;TANGUT COMPONENT-805;Lo;0;L;;;;;N;;;;; +18DA5;TANGUT COMPONENT-806;Lo;0;L;;;;;N;;;;; +18DA6;TANGUT COMPONENT-807;Lo;0;L;;;;;N;;;;; +18DA7;TANGUT COMPONENT-808;Lo;0;L;;;;;N;;;;; +18DA8;TANGUT COMPONENT-809;Lo;0;L;;;;;N;;;;; +18DA9;TANGUT COMPONENT-810;Lo;0;L;;;;;N;;;;; +18DAA;TANGUT COMPONENT-811;Lo;0;L;;;;;N;;;;; +18DAB;TANGUT COMPONENT-812;Lo;0;L;;;;;N;;;;; +18DAC;TANGUT COMPONENT-813;Lo;0;L;;;;;N;;;;; +18DAD;TANGUT COMPONENT-814;Lo;0;L;;;;;N;;;;; +18DAE;TANGUT COMPONENT-815;Lo;0;L;;;;;N;;;;; +18DAF;TANGUT COMPONENT-816;Lo;0;L;;;;;N;;;;; +18DB0;TANGUT COMPONENT-817;Lo;0;L;;;;;N;;;;; +18DB1;TANGUT COMPONENT-818;Lo;0;L;;;;;N;;;;; +18DB2;TANGUT COMPONENT-819;Lo;0;L;;;;;N;;;;; +18DB3;TANGUT COMPONENT-820;Lo;0;L;;;;;N;;;;; +18DB4;TANGUT COMPONENT-821;Lo;0;L;;;;;N;;;;; +18DB5;TANGUT COMPONENT-822;Lo;0;L;;;;;N;;;;; +18DB6;TANGUT COMPONENT-823;Lo;0;L;;;;;N;;;;; +18DB7;TANGUT COMPONENT-824;Lo;0;L;;;;;N;;;;; +18DB8;TANGUT COMPONENT-825;Lo;0;L;;;;;N;;;;; +18DB9;TANGUT COMPONENT-826;Lo;0;L;;;;;N;;;;; +18DBA;TANGUT COMPONENT-827;Lo;0;L;;;;;N;;;;; +18DBB;TANGUT COMPONENT-828;Lo;0;L;;;;;N;;;;; +18DBC;TANGUT COMPONENT-829;Lo;0;L;;;;;N;;;;; +18DBD;TANGUT COMPONENT-830;Lo;0;L;;;;;N;;;;; +18DBE;TANGUT COMPONENT-831;Lo;0;L;;;;;N;;;;; +18DBF;TANGUT COMPONENT-832;Lo;0;L;;;;;N;;;;; +18DC0;TANGUT COMPONENT-833;Lo;0;L;;;;;N;;;;; +18DC1;TANGUT COMPONENT-834;Lo;0;L;;;;;N;;;;; +18DC2;TANGUT COMPONENT-835;Lo;0;L;;;;;N;;;;; +18DC3;TANGUT COMPONENT-836;Lo;0;L;;;;;N;;;;; +18DC4;TANGUT COMPONENT-837;Lo;0;L;;;;;N;;;;; +18DC5;TANGUT COMPONENT-838;Lo;0;L;;;;;N;;;;; +18DC6;TANGUT COMPONENT-839;Lo;0;L;;;;;N;;;;; +18DC7;TANGUT COMPONENT-840;Lo;0;L;;;;;N;;;;; +18DC8;TANGUT COMPONENT-841;Lo;0;L;;;;;N;;;;; +18DC9;TANGUT COMPONENT-842;Lo;0;L;;;;;N;;;;; +18DCA;TANGUT COMPONENT-843;Lo;0;L;;;;;N;;;;; +18DCB;TANGUT COMPONENT-844;Lo;0;L;;;;;N;;;;; +18DCC;TANGUT COMPONENT-845;Lo;0;L;;;;;N;;;;; +18DCD;TANGUT COMPONENT-846;Lo;0;L;;;;;N;;;;; +18DCE;TANGUT COMPONENT-847;Lo;0;L;;;;;N;;;;; +18DCF;TANGUT COMPONENT-848;Lo;0;L;;;;;N;;;;; +18DD0;TANGUT COMPONENT-849;Lo;0;L;;;;;N;;;;; +18DD1;TANGUT COMPONENT-850;Lo;0;L;;;;;N;;;;; +18DD2;TANGUT COMPONENT-851;Lo;0;L;;;;;N;;;;; +18DD3;TANGUT COMPONENT-852;Lo;0;L;;;;;N;;;;; +18DD4;TANGUT COMPONENT-853;Lo;0;L;;;;;N;;;;; +18DD5;TANGUT COMPONENT-854;Lo;0;L;;;;;N;;;;; +18DD6;TANGUT COMPONENT-855;Lo;0;L;;;;;N;;;;; +18DD7;TANGUT COMPONENT-856;Lo;0;L;;;;;N;;;;; +18DD8;TANGUT COMPONENT-857;Lo;0;L;;;;;N;;;;; +18DD9;TANGUT COMPONENT-858;Lo;0;L;;;;;N;;;;; +18DDA;TANGUT COMPONENT-859;Lo;0;L;;;;;N;;;;; +18DDB;TANGUT COMPONENT-860;Lo;0;L;;;;;N;;;;; +18DDC;TANGUT COMPONENT-861;Lo;0;L;;;;;N;;;;; +18DDD;TANGUT COMPONENT-862;Lo;0;L;;;;;N;;;;; +18DDE;TANGUT COMPONENT-863;Lo;0;L;;;;;N;;;;; +18DDF;TANGUT COMPONENT-864;Lo;0;L;;;;;N;;;;; +18DE0;TANGUT COMPONENT-865;Lo;0;L;;;;;N;;;;; +18DE1;TANGUT COMPONENT-866;Lo;0;L;;;;;N;;;;; +18DE2;TANGUT COMPONENT-867;Lo;0;L;;;;;N;;;;; +18DE3;TANGUT COMPONENT-868;Lo;0;L;;;;;N;;;;; +18DE4;TANGUT COMPONENT-869;Lo;0;L;;;;;N;;;;; +18DE5;TANGUT COMPONENT-870;Lo;0;L;;;;;N;;;;; +18DE6;TANGUT COMPONENT-871;Lo;0;L;;;;;N;;;;; +18DE7;TANGUT COMPONENT-872;Lo;0;L;;;;;N;;;;; +18DE8;TANGUT COMPONENT-873;Lo;0;L;;;;;N;;;;; +18DE9;TANGUT COMPONENT-874;Lo;0;L;;;;;N;;;;; +18DEA;TANGUT COMPONENT-875;Lo;0;L;;;;;N;;;;; +18DEB;TANGUT COMPONENT-876;Lo;0;L;;;;;N;;;;; +18DEC;TANGUT COMPONENT-877;Lo;0;L;;;;;N;;;;; +18DED;TANGUT COMPONENT-878;Lo;0;L;;;;;N;;;;; +18DEE;TANGUT COMPONENT-879;Lo;0;L;;;;;N;;;;; +18DEF;TANGUT COMPONENT-880;Lo;0;L;;;;;N;;;;; +18DF0;TANGUT COMPONENT-881;Lo;0;L;;;;;N;;;;; +18DF1;TANGUT COMPONENT-882;Lo;0;L;;;;;N;;;;; +18DF2;TANGUT COMPONENT-883;Lo;0;L;;;;;N;;;;; 1AFF0;KATAKANA LETTER MINNAN TONE-2;Lm;0;L;;;;;N;;;;; 1AFF1;KATAKANA LETTER MINNAN TONE-3;Lm;0;L;;;;;N;;;;; 1AFF2;KATAKANA LETTER MINNAN TONE-4;Lm;0;L;;;;;N;;;;; @@ -32629,6 +32963,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1CCF7;OUTLINED DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; 1CCF8;OUTLINED DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; 1CCF9;OUTLINED DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1CCFA;SNAKE SYMBOL;So;0;ON;;;;;N;;;;; +1CCFB;FLYING SAUCER SYMBOL;So;0;ON;;;;;N;;;;; +1CCFC;NOSE SYMBOL;So;0;ON;;;;;N;;;;; 1CD00;BLOCK OCTANT-3;So;0;ON;;;;;N;;;;; 1CD01;BLOCK OCTANT-23;So;0;ON;;;;;N;;;;; 1CD02;BLOCK OCTANT-123;So;0;ON;;;;;N;;;;; @@ -33065,6 +33402,46 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1CEB1;KEYHOLE;So;0;ON;;;;;N;;;;; 1CEB2;OLD PERSONAL COMPUTER WITH MONITOR IN PORTRAIT ORIENTATION;So;0;ON;;;;;N;;;;; 1CEB3;BLACK RIGHT TRIANGLE CARET;So;0;ON;;;;;N;;;;; +1CEBA;FRAGILE SYMBOL;So;0;ON;;;;;N;;;;; +1CEBB;OFFICE BUILDING SYMBOL;So;0;ON;;;;;N;;;;; +1CEBC;TREE SYMBOL;So;0;ON;;;;;N;;;;; +1CEBD;APPLE SYMBOL;So;0;ON;;;;;N;;;;; +1CEBE;CHERRY SYMBOL;So;0;ON;;;;;N;;;;; +1CEBF;STRAWBERRY SYMBOL;So;0;ON;;;;;N;;;;; +1CEC0;HEBE;So;0;ON;;;;;N;;;;; +1CEC1;IRIS;So;0;ON;;;;;N;;;;; +1CEC2;FLORA;So;0;ON;;;;;N;;;;; +1CEC3;METIS;So;0;ON;;;;;N;;;;; +1CEC4;PARTHENOPE;So;0;ON;;;;;N;;;;; +1CEC5;VICTORIA;So;0;ON;;;;;N;;;;; +1CEC6;EGERIA;So;0;ON;;;;;N;;;;; +1CEC7;IRENE;So;0;ON;;;;;N;;;;; +1CEC8;EUNOMIA;So;0;ON;;;;;N;;;;; +1CEC9;PSYCHE;So;0;ON;;;;;N;;;;; +1CECA;THETIS;So;0;ON;;;;;N;;;;; +1CECB;MELPOMENE;So;0;ON;;;;;N;;;;; +1CECC;FORTUNA;So;0;ON;;;;;N;;;;; +1CECD;ASTRONOMICAL SYMBOL FOR ASTEROID PROSERPINA;So;0;ON;;;;;N;;;;; +1CECE;BELLONA;So;0;ON;;;;;N;;;;; +1CECF;AMPHITRITE;So;0;ON;;;;;N;;;;; +1CED0;LEUKOTHEA;So;0;ON;;;;;N;;;;; +1CEE0;GEOMANTIC FIGURE POPULUS;So;0;ON;;;;;N;;;;; +1CEE1;GEOMANTIC FIGURE TRISTITIA;So;0;ON;;;;;N;;;;; +1CEE2;GEOMANTIC FIGURE ALBUS;So;0;ON;;;;;N;;;;; +1CEE3;GEOMANTIC FIGURE FORTUNA MAJOR;So;0;ON;;;;;N;;;;; +1CEE4;GEOMANTIC FIGURE RUBEUS;So;0;ON;;;;;N;;;;; +1CEE5;GEOMANTIC FIGURE ACQUISITIO;So;0;ON;;;;;N;;;;; +1CEE6;GEOMANTIC FIGURE CONJUNCTIO;So;0;ON;;;;;N;;;;; +1CEE7;GEOMANTIC FIGURE CAPUT DRACONIS;So;0;ON;;;;;N;;;;; +1CEE8;GEOMANTIC FIGURE LAETITIA;So;0;ON;;;;;N;;;;; +1CEE9;GEOMANTIC FIGURE CARCER;So;0;ON;;;;;N;;;;; +1CEEA;GEOMANTIC FIGURE AMISSIO;So;0;ON;;;;;N;;;;; +1CEEB;GEOMANTIC FIGURE PUELLA;So;0;ON;;;;;N;;;;; +1CEEC;GEOMANTIC FIGURE FORTUNA MINOR;So;0;ON;;;;;N;;;;; +1CEED;GEOMANTIC FIGURE PUER;So;0;ON;;;;;N;;;;; +1CEEE;GEOMANTIC FIGURE CAUDA DRACONIS;So;0;ON;;;;;N;;;;; +1CEEF;GEOMANTIC FIGURE VIA;So;0;ON;;;;;N;;;;; +1CEF0;MEDIUM SMALL WHITE CIRCLE WITH HORIZONTAL BAR;Sm;0;ON;;;;;N;;;;; 1CF00;ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT;Mn;0;NSM;;;;;N;;;;; 1CF01;ZNAMENNY COMBINING MARK NIZKO S KRYZHEM ON LEFT;Mn;0;NSM;;;;;N;;;;; 1CF02;ZNAMENNY COMBINING MARK TSATA ON LEFT;Mn;0;NSM;;;;;N;;;;; @@ -36004,6 +36381,61 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E5F9;OL ONAL DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E5FA;OL ONAL DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E5FF;OL ONAL ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +1E6C0;TAI YO LETTER LOW KO;Lo;0;L;;;;;N;;;;; +1E6C1;TAI YO LETTER HIGH KO;Lo;0;L;;;;;N;;;;; +1E6C2;TAI YO LETTER LOW KHO;Lo;0;L;;;;;N;;;;; +1E6C3;TAI YO LETTER HIGH KHO;Lo;0;L;;;;;N;;;;; +1E6C4;TAI YO LETTER GO;Lo;0;L;;;;;N;;;;; +1E6C5;TAI YO LETTER NGO;Lo;0;L;;;;;N;;;;; +1E6C6;TAI YO LETTER CO;Lo;0;L;;;;;N;;;;; +1E6C7;TAI YO LETTER LOW XO;Lo;0;L;;;;;N;;;;; +1E6C8;TAI YO LETTER HIGH XO;Lo;0;L;;;;;N;;;;; +1E6C9;TAI YO LETTER LOW NYO;Lo;0;L;;;;;N;;;;; +1E6CA;TAI YO LETTER HIGH NYO;Lo;0;L;;;;;N;;;;; +1E6CB;TAI YO LETTER DO;Lo;0;L;;;;;N;;;;; +1E6CC;TAI YO LETTER LOW TO;Lo;0;L;;;;;N;;;;; +1E6CD;TAI YO LETTER HIGH TO;Lo;0;L;;;;;N;;;;; +1E6CE;TAI YO LETTER THO;Lo;0;L;;;;;N;;;;; +1E6CF;TAI YO LETTER NO;Lo;0;L;;;;;N;;;;; +1E6D0;TAI YO LETTER BO;Lo;0;L;;;;;N;;;;; +1E6D1;TAI YO LETTER LOW PO;Lo;0;L;;;;;N;;;;; +1E6D2;TAI YO LETTER HIGH PO;Lo;0;L;;;;;N;;;;; +1E6D3;TAI YO LETTER PHO;Lo;0;L;;;;;N;;;;; +1E6D4;TAI YO LETTER LOW FO;Lo;0;L;;;;;N;;;;; +1E6D5;TAI YO LETTER HIGH FO;Lo;0;L;;;;;N;;;;; +1E6D6;TAI YO LETTER MO;Lo;0;L;;;;;N;;;;; +1E6D7;TAI YO LETTER YO;Lo;0;L;;;;;N;;;;; +1E6D8;TAI YO LETTER LO;Lo;0;L;;;;;N;;;;; +1E6D9;TAI YO LETTER VO;Lo;0;L;;;;;N;;;;; +1E6DA;TAI YO LETTER LOW HO;Lo;0;L;;;;;N;;;;; +1E6DB;TAI YO LETTER HIGH HO;Lo;0;L;;;;;N;;;;; +1E6DC;TAI YO LETTER QO;Lo;0;L;;;;;N;;;;; +1E6DD;TAI YO LETTER LOW KVO;Lo;0;L;;;;;N;;;;; +1E6DE;TAI YO LETTER HIGH KVO;Lo;0;L;;;;;N;;;;; +1E6E0;TAI YO LETTER AA;Lo;0;L;;;;;N;;;;; +1E6E1;TAI YO LETTER I;Lo;0;L;;;;;N;;;;; +1E6E2;TAI YO LETTER UE;Lo;0;L;;;;;N;;;;; +1E6E3;TAI YO SIGN UE;Mn;230;NSM;;;;;N;;;;; +1E6E4;TAI YO LETTER U;Lo;0;L;;;;;N;;;;; +1E6E5;TAI YO LETTER AE;Lo;0;L;;;;;N;;;;; +1E6E6;TAI YO SIGN AU;Mn;230;NSM;;;;;N;;;;; +1E6E7;TAI YO LETTER O;Lo;0;L;;;;;N;;;;; +1E6E8;TAI YO LETTER E;Lo;0;L;;;;;N;;;;; +1E6E9;TAI YO LETTER IA;Lo;0;L;;;;;N;;;;; +1E6EA;TAI YO LETTER UEA;Lo;0;L;;;;;N;;;;; +1E6EB;TAI YO LETTER UA;Lo;0;L;;;;;N;;;;; +1E6EC;TAI YO LETTER OO;Lo;0;L;;;;;N;;;;; +1E6ED;TAI YO LETTER AUE;Lo;0;L;;;;;N;;;;; +1E6EE;TAI YO SIGN AY;Mn;230;NSM;;;;;N;;;;; +1E6EF;TAI YO SIGN ANG;Mn;230;NSM;;;;;N;;;;; +1E6F0;TAI YO LETTER AN;Lo;0;L;;;;;N;;;;; +1E6F1;TAI YO LETTER AM;Lo;0;L;;;;;N;;;;; +1E6F2;TAI YO LETTER AK;Lo;0;L;;;;;N;;;;; +1E6F3;TAI YO LETTER AT;Lo;0;L;;;;;N;;;;; +1E6F4;TAI YO LETTER AP;Lo;0;L;;;;;N;;;;; +1E6F5;TAI YO SIGN OM;Mn;230;NSM;;;;;N;;;;; +1E6FE;TAI YO SYMBOL MUEANG;Lo;0;L;;;;;N;;;;; +1E6FF;TAI YO XAM LAI;Lm;0;L;;;;;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -38079,6 +38511,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6D8;LANDSLIDE;So;0;ON;;;;;N;;;;; 1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; @@ -38228,6 +38661,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; 1F775;OCCULTATION;So;0;ON;;;;;N;;;;; 1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F777;VESTA FORM TWO;So;0;ON;;;;;N;;;;; +1F778;ASTRAEA FORM TWO;So;0;ON;;;;;N;;;;; +1F779;HYGIEA FORM TWO;So;0;ON;;;;;N;;;;; +1F77A;PARTHENOPE FORM TWO;So;0;ON;;;;;N;;;;; 1F77B;HAUMEA;So;0;ON;;;;;N;;;;; 1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; 1F77D;GONGGONG;So;0;ON;;;;;N;;;;; @@ -38498,6 +38935,15 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F8BB;SOUTH WEST ARROW FROM BAR;So;0;ON;;;;;N;;;;; 1F8C0;LEFTWARDS ARROW FROM DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; 1F8C1;RIGHTWARDS ARROW FROM DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8D0;LONG RIGHTWARDS ARROW OVER LONG LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +1F8D1;LONG RIGHTWARDS HARPOON OVER LONG LEFTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D2;LONG RIGHTWARDS HARPOON ABOVE SHORT LEFTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D3;SHORT RIGHTWARDS HARPOON ABOVE LONG LEFTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D4;LONG LEFTWARDS HARPOON ABOVE SHORT RIGHTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D5;SHORT LEFTWARDS HARPOON ABOVE LONG RIGHTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D6;LONG RIGHTWARDS ARROW THROUGH X;Sm;0;ON;;;;;N;;;;; +1F8D7;LONG RIGHTWARDS ARROW WITH DOUBLE SLASH;Sm;0;ON;;;;;N;;;;; +1F8D8;LONG LEFT RIGHT ARROW WITH DEPENDENT LOBE;Sm;0;ON;;;;;N;;;;; 1F900;CIRCLED CROSS FORMEE WITH FOUR DOTS;So;0;ON;;;;;N;;;;; 1F901;CIRCLED CROSS FORMEE WITH TWO DOTS;So;0;ON;;;;;N;;;;; 1F902;CIRCLED CROSS FORMEE;So;0;ON;;;;;N;;;;; @@ -38838,6 +39284,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA51;BLACK CHESS KNIGHT-QUEEN;So;0;ON;;;;;N;;;;; 1FA52;BLACK CHESS KNIGHT-ROOK;So;0;ON;;;;;N;;;;; 1FA53;BLACK CHESS KNIGHT-BISHOP;So;0;ON;;;;;N;;;;; +1FA54;WHITE CHESS FERZ;So;0;ON;;;;;N;;;;; +1FA55;WHITE CHESS ALFIL;So;0;ON;;;;;N;;;;; +1FA56;BLACK CHESS FERZ;So;0;ON;;;;;N;;;;; +1FA57;BLACK CHESS ALFIL;So;0;ON;;;;;N;;;;; 1FA60;XIANGQI RED GENERAL;So;0;ON;;;;;N;;;;; 1FA61;XIANGQI RED MANDARIN;So;0;ON;;;;;N;;;;; 1FA62;XIANGQI RED ELEPHANT;So;0;ON;;;;;N;;;;; @@ -38875,6 +39325,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA87;MARACAS;So;0;ON;;;;;N;;;;; 1FA88;FLUTE;So;0;ON;;;;;N;;;;; 1FA89;HARP;So;0;ON;;;;;N;;;;; +1FA8A;TROMBONE;So;0;ON;;;;;N;;;;; +1FA8E;TREASURE CHEST;So;0;ON;;;;;N;;;;; 1FA8F;SHOVEL;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; @@ -38931,6 +39383,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; 1FAC6;FINGERPRINT;So;0;ON;;;;;N;;;;; +1FAC8;HAIRY CREATURE;So;0;ON;;;;;N;;;;; +1FACD;ORCA;So;0;ON;;;;;N;;;;; 1FACE;MOOSE;So;0;ON;;;;;N;;;;; 1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; @@ -38957,6 +39411,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; 1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; 1FAE9;FACE WITH BAGS UNDER EYES;So;0;ON;;;;;N;;;;; +1FAEA;DISTORTED FACE;So;0;ON;;;;;N;;;;; +1FAEF;FIGHT CLOUD;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -39215,14 +39671,15 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FBF7;SEGMENTED DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; 1FBF8;SEGMENTED DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; 1FBF9;SEGMENTED DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1FBFA;ALARM BELL SYMBOL;So;0;ON;;;;;N;;;;; 20000;;Lo;0;L;;;;;N;;;;; 2A6DF;;Lo;0;L;;;;;N;;;;; 2A700;;Lo;0;L;;;;;N;;;;; -2B739;;Lo;0;L;;;;;N;;;;; +2B73F;;Lo;0;L;;;;;N;;;;; 2B740;;Lo;0;L;;;;;N;;;;; 2B81D;;Lo;0;L;;;;;N;;;;; 2B820;;Lo;0;L;;;;;N;;;;; -2CEA1;;Lo;0;L;;;;;N;;;;; +2CEAD;;Lo;0;L;;;;;N;;;;; 2CEB0;;Lo;0;L;;;;;N;;;;; 2EBE0;;Lo;0;L;;;;;N;;;;; 2EBF0;;Lo;0;L;;;;;N;;;;; @@ -39773,6 +40230,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 3134A;;Lo;0;L;;;;;N;;;;; 31350;;Lo;0;L;;;;;N;;;;; 323AF;;Lo;0;L;;;;;N;;;;; +323B0;;Lo;0;L;;;;;N;;;;; +33479;;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; diff --git a/src/libre/class.h b/src/libre/class.h index 919571982..79460a9da 100644 --- a/src/libre/class.h +++ b/src/libre/class.h @@ -29,6 +29,7 @@ extern const struct class utf8_Bamum; extern const struct class utf8_Bassa_Vah; extern const struct class utf8_Batak; extern const struct class utf8_Bengali; +extern const struct class utf8_Beria_Erfe; extern const struct class utf8_Bhaiksuki; extern const struct class utf8_Bopomofo; extern const struct class utf8_Brahmi; @@ -153,6 +154,7 @@ extern const struct class utf8_Saurashtra; extern const struct class utf8_Sharada; extern const struct class utf8_Shavian; extern const struct class utf8_Siddham; +extern const struct class utf8_Sidetic; extern const struct class utf8_SignWriting; extern const struct class utf8_Sinhala; extern const struct class utf8_Sogdian; @@ -167,6 +169,7 @@ extern const struct class utf8_Tagbanwa; extern const struct class utf8_Tai_Le; extern const struct class utf8_Tai_Tham; extern const struct class utf8_Tai_Viet; +extern const struct class utf8_Tai_Yo; extern const struct class utf8_Takri; extern const struct class utf8_Tamil; extern const struct class utf8_Tangsa; @@ -178,6 +181,7 @@ extern const struct class utf8_Tibetan; extern const struct class utf8_Tifinagh; extern const struct class utf8_Tirhuta; extern const struct class utf8_Todhri; +extern const struct class utf8_Tolong_Siki; extern const struct class utf8_Toto; extern const struct class utf8_Tulu_Tigalari; extern const struct class utf8_Ugaritic; diff --git a/src/libre/class/utf8_Arabic.c b/src/libre/class/utf8_Arabic.c index 904ab392e..592e06d7d 100644 --- a/src/libre/class/utf8_Arabic.c +++ b/src/libre/class/utf8_Arabic.c @@ -13,21 +13,18 @@ static const struct range ranges[] = { { 0x0671UL, 0x06DCUL }, { 0x06DEUL, 0x06FFUL }, { 0x0750UL, 0x077FUL }, - { 0x0870UL, 0x088EUL }, - { 0x0890UL, 0x0891UL }, + { 0x0870UL, 0x0891UL }, { 0x0897UL, 0x08E1UL }, { 0x08E3UL, 0x08FFUL }, - { 0xFB50UL, 0xFBC2UL }, - { 0xFBD3UL, 0xFD3DUL }, - { 0xFD40UL, 0xFD8FUL }, - { 0xFD92UL, 0xFDC7UL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFB50UL, 0xFD3DUL }, + { 0xFD40UL, 0xFDCFUL }, { 0xFDF0UL, 0xFDFFUL }, { 0xFE70UL, 0xFE74UL }, { 0xFE76UL, 0xFEFCUL }, { 0x10E60UL, 0x10E7EUL }, - { 0x10EC2UL, 0x10EC4UL }, - { 0x10EFCUL, 0x10EFFUL }, + { 0x10EC2UL, 0x10EC7UL }, + { 0x10ED0UL, 0x10ED8UL }, + { 0x10EFAUL, 0x10EFFUL }, { 0x1EE00UL, 0x1EE03UL }, { 0x1EE05UL, 0x1EE1FUL }, { 0x1EE21UL, 0x1EE22UL }, diff --git a/src/libre/class/utf8_Beria_Erfe.c b/src/libre/class/utf8_Beria_Erfe.c new file mode 100644 index 000000000..3bf7b4db7 --- /dev/null +++ b/src/libre/class/utf8_Beria_Erfe.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x16EA0UL, 0x16EB8UL }, + { 0x16EBBUL, 0x16ED3UL } +}; + +const struct class utf8_Beria_Erfe = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Common.c b/src/libre/class/utf8_Common.c index 44e6f3ff2..26c3f54e8 100644 --- a/src/libre/class/utf8_Common.c +++ b/src/libre/class/utf8_Common.c @@ -43,7 +43,7 @@ static const struct range ranges[] = { { 0x2066UL, 0x2070UL }, { 0x2074UL, 0x207EUL }, { 0x2080UL, 0x208EUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0x2100UL, 0x2125UL }, { 0x2127UL, 0x2129UL }, { 0x212CUL, 0x2131UL }, @@ -54,8 +54,7 @@ static const struct range ranges[] = { { 0x2440UL, 0x244AUL }, { 0x2460UL, 0x27FFUL }, { 0x2900UL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2BFFUL }, + { 0x2B76UL, 0x2BFFUL }, { 0x2E00UL, 0x2E5DUL }, { 0x2FF0UL, 0x3004UL }, { 0x3006UL, 0x3006UL }, @@ -101,8 +100,10 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x102E1UL, 0x102FBUL }, { 0x1BCA0UL, 0x1BCA3UL }, - { 0x1CC00UL, 0x1CCF9UL }, + { 0x1CC00UL, 0x1CCFCUL }, { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEF0UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -151,11 +152,10 @@ static const struct range ranges[] = { { 0x1F240UL, 0x1F248UL }, { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, - { 0x1F300UL, 0x1F6D7UL }, + { 0x1F300UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -165,16 +165,18 @@ static const struct range ranges[] = { { 0x1F890UL, 0x1F8ADUL }, { 0x1F8B0UL, 0x1F8BBUL }, { 0x1F8C0UL, 0x1F8C1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8D0UL, 0x1F8D8UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA89UL }, - { 0x1FA8FUL, 0x1FAC6UL }, - { 0x1FACEUL, 0x1FADCUL }, - { 0x1FADFUL, 0x1FAE9UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBF9UL }, + { 0x1FB94UL, 0x1FBFAUL }, { 0xE0001UL, 0xE0001UL }, { 0xE0020UL, 0xE007FUL } }; diff --git a/src/libre/class/utf8_Han.c b/src/libre/class/utf8_Han.c index ce3bbce61..0fdf663e0 100644 --- a/src/libre/class/utf8_Han.c +++ b/src/libre/class/utf8_Han.c @@ -29,16 +29,15 @@ static const struct range ranges[] = { { 0xF900UL, 0xFA6DUL }, { 0xFA70UL, 0xFAD9UL }, { 0x16FE2UL, 0x16FE3UL }, - { 0x16FF0UL, 0x16FF1UL }, + { 0x16FF0UL, 0x16FF6UL }, { 0x20000UL, 0x2A6DFUL }, - { 0x2A700UL, 0x2B739UL }, - { 0x2B740UL, 0x2B81DUL }, - { 0x2B820UL, 0x2CEA1UL }, + { 0x2A700UL, 0x2B81DUL }, + { 0x2B820UL, 0x2CEADUL }, { 0x2CEB0UL, 0x2EBE0UL }, { 0x2EBF0UL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x3134AUL }, - { 0x31350UL, 0x323AFUL }, + { 0x31350UL, 0x33479UL }, { 0x1720UL, 0x1734UL }, { 0x10D00UL, 0x10D27UL }, { 0x10D30UL, 0x10D39UL } diff --git a/src/libre/class/utf8_Inherited.c b/src/libre/class/utf8_Inherited.c index 8e5afc76c..46ee6739a 100644 --- a/src/libre/class/utf8_Inherited.c +++ b/src/libre/class/utf8_Inherited.c @@ -8,7 +8,8 @@ static const struct range ranges[] = { { 0x064BUL, 0x0655UL }, { 0x0670UL, 0x0670UL }, { 0x0951UL, 0x0954UL }, - { 0x1AB0UL, 0x1ACEUL }, + { 0x1AB0UL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1CD0UL, 0x1CD2UL }, { 0x1CD4UL, 0x1CE0UL }, { 0x1CE2UL, 0x1CE8UL }, diff --git a/src/libre/class/utf8_Kannada.c b/src/libre/class/utf8_Kannada.c index c2f9ae6e6..a4f8ebd40 100644 --- a/src/libre/class/utf8_Kannada.c +++ b/src/libre/class/utf8_Kannada.c @@ -12,7 +12,7 @@ static const struct range ranges[] = { { 0x0CC6UL, 0x0CC8UL }, { 0x0CCAUL, 0x0CCDUL }, { 0x0CD5UL, 0x0CD6UL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE3UL }, { 0x0CE6UL, 0x0CEFUL }, { 0x0CF1UL, 0x0CF3UL } diff --git a/src/libre/class/utf8_L.c b/src/libre/class/utf8_L.c index b2c396fcc..e37048bc1 100644 --- a/src/libre/class/utf8_L.c +++ b/src/libre/class/utf8_L.c @@ -53,7 +53,7 @@ static const struct range ranges[] = { { 0x0840UL, 0x0858UL }, { 0x0860UL, 0x086AUL }, { 0x0870UL, 0x0887UL }, - { 0x0889UL, 0x088EUL }, + { 0x0889UL, 0x088FUL }, { 0x08A0UL, 0x08C9UL }, { 0x0904UL, 0x0939UL }, { 0x093DUL, 0x093DUL }, @@ -119,7 +119,7 @@ static const struct range ranges[] = { { 0x0C2AUL, 0x0C39UL }, { 0x0C3DUL, 0x0C3DUL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C61UL }, { 0x0C80UL, 0x0C80UL }, { 0x0C85UL, 0x0C8CUL }, @@ -128,7 +128,7 @@ static const struct range ranges[] = { { 0x0CAAUL, 0x0CB3UL }, { 0x0CB5UL, 0x0CB9UL }, { 0x0CBDUL, 0x0CBDUL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE1UL }, { 0x0CF1UL, 0x0CF2UL }, { 0x0D04UL, 0x0D0CUL }, @@ -314,11 +314,8 @@ static const struct range ranges[] = { { 0xA6A0UL, 0xA6E5UL }, { 0xA717UL, 0xA71FUL }, { 0xA722UL, 0xA788UL }, - { 0xA78BUL, 0xA7CDUL }, - { 0xA7D0UL, 0xA7D1UL }, - { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7DCUL }, - { 0xA7F2UL, 0xA801UL }, + { 0xA78BUL, 0xA7DCUL }, + { 0xA7F1UL, 0xA801UL }, { 0xA803UL, 0xA805UL }, { 0xA807UL, 0xA80AUL }, { 0xA80CUL, 0xA822UL }, @@ -434,6 +431,7 @@ static const struct range ranges[] = { { 0x108F4UL, 0x108F5UL }, { 0x10900UL, 0x10915UL }, { 0x10920UL, 0x10939UL }, + { 0x10940UL, 0x10959UL }, { 0x10980UL, 0x109B7UL }, { 0x109BEUL, 0x109BFUL }, { 0x10A00UL, 0x10A00UL }, @@ -456,7 +454,7 @@ static const struct range ranges[] = { { 0x10D6FUL, 0x10D85UL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EB0UL, 0x10EB1UL }, - { 0x10EC2UL, 0x10EC4UL }, + { 0x10EC2UL, 0x10EC7UL }, { 0x10F00UL, 0x10F1CUL }, { 0x10F27UL, 0x10F27UL }, { 0x10F30UL, 0x10F45UL }, @@ -549,6 +547,7 @@ static const struct range ranges[] = { { 0x11D67UL, 0x11D68UL }, { 0x11D6AUL, 0x11D89UL }, { 0x11D98UL, 0x11D98UL }, + { 0x11DB0UL, 0x11DDBUL }, { 0x11EE0UL, 0x11EF2UL }, { 0x11F02UL, 0x11F02UL }, { 0x11F04UL, 0x11F10UL }, @@ -572,16 +571,19 @@ static const struct range ranges[] = { { 0x16B7DUL, 0x16B8FUL }, { 0x16D40UL, 0x16D6CUL }, { 0x16E40UL, 0x16E7FUL }, + { 0x16EA0UL, 0x16EB8UL }, + { 0x16EBBUL, 0x16ED3UL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F50UL, 0x16F50UL }, { 0x16F93UL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE1UL }, { 0x16FE3UL, 0x16FE3UL }, + { 0x16FF2UL, 0x16FF3UL }, { 0x17000UL, 0x17000UL }, - { 0x187F7UL, 0x187F7UL }, - { 0x18800UL, 0x18CD5UL }, + { 0x187FFUL, 0x18CD5UL }, { 0x18CFFUL, 0x18D00UL }, - { 0x18D08UL, 0x18D08UL }, + { 0x18D1EUL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, { 0x1AFFDUL, 0x1AFFEUL }, @@ -636,6 +638,12 @@ static const struct range ranges[] = { { 0x1E4D0UL, 0x1E4EBUL }, { 0x1E5D0UL, 0x1E5EDUL }, { 0x1E5F0UL, 0x1E5F0UL }, + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6E2UL }, + { 0x1E6E4UL, 0x1E6E5UL }, + { 0x1E6E7UL, 0x1E6EDUL }, + { 0x1E6F0UL, 0x1E6F4UL }, + { 0x1E6FEUL, 0x1E6FFUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -679,11 +687,10 @@ static const struct range ranges[] = { { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, - { 0x2B739UL, 0x2B739UL }, - { 0x2B740UL, 0x2B740UL }, + { 0x2B73FUL, 0x2B740UL }, { 0x2B81DUL, 0x2B81DUL }, { 0x2B820UL, 0x2B820UL }, - { 0x2CEA1UL, 0x2CEA1UL }, + { 0x2CEADUL, 0x2CEADUL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, { 0x2EBF0UL, 0x2EBF0UL }, @@ -692,7 +699,8 @@ static const struct range ranges[] = { { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, { 0x31350UL, 0x31350UL }, - { 0x323AFUL, 0x323AFUL } + { 0x323AFUL, 0x323B0UL }, + { 0x33479UL, 0x33479UL } }; const struct class utf8_L = { diff --git a/src/libre/class/utf8_Latin.c b/src/libre/class/utf8_Latin.c index 9f59bd35e..362ab61a6 100644 --- a/src/libre/class/utf8_Latin.c +++ b/src/libre/class/utf8_Latin.c @@ -26,11 +26,8 @@ static const struct range ranges[] = { { 0x2160UL, 0x2188UL }, { 0x2C60UL, 0x2C7FUL }, { 0xA722UL, 0xA787UL }, - { 0xA78BUL, 0xA7CDUL }, - { 0xA7D0UL, 0xA7D1UL }, - { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7DCUL }, - { 0xA7F2UL, 0xA7FFUL }, + { 0xA78BUL, 0xA7DCUL }, + { 0xA7F1UL, 0xA7FFUL }, { 0xAB30UL, 0xAB5AUL }, { 0xAB5CUL, 0xAB64UL }, { 0xAB66UL, 0xAB69UL }, diff --git a/src/libre/class/utf8_Ll.c b/src/libre/class/utf8_Ll.c index 889a1da5e..ad884ead6 100644 --- a/src/libre/class/utf8_Ll.c +++ b/src/libre/class/utf8_Ll.c @@ -148,7 +148,7 @@ static const struct range ranges[] = { { 0x024BUL, 0x024BUL }, { 0x024DUL, 0x024DUL }, { 0x024FUL, 0x0293UL }, - { 0x0295UL, 0x02AFUL }, + { 0x0296UL, 0x02AFUL }, { 0x0371UL, 0x0371UL }, { 0x0373UL, 0x0373UL }, { 0x0377UL, 0x0377UL }, @@ -609,6 +609,7 @@ static const struct range ranges[] = { { 0xA7C8UL, 0xA7C8UL }, { 0xA7CAUL, 0xA7CAUL }, { 0xA7CDUL, 0xA7CDUL }, + { 0xA7CFUL, 0xA7CFUL }, { 0xA7D1UL, 0xA7D1UL }, { 0xA7D3UL, 0xA7D3UL }, { 0xA7D5UL, 0xA7D5UL }, @@ -633,6 +634,7 @@ static const struct range ranges[] = { { 0x10D70UL, 0x10D85UL }, { 0x118C0UL, 0x118DFUL }, { 0x16E60UL, 0x16E7FUL }, + { 0x16EBBUL, 0x16ED3UL }, { 0x1D41AUL, 0x1D433UL }, { 0x1D44EUL, 0x1D454UL }, { 0x1D456UL, 0x1D467UL }, diff --git a/src/libre/class/utf8_Lm.c b/src/libre/class/utf8_Lm.c index 884125481..3052e9072 100644 --- a/src/libre/class/utf8_Lm.c +++ b/src/libre/class/utf8_Lm.c @@ -49,7 +49,7 @@ static const struct range ranges[] = { { 0xA717UL, 0xA71FUL }, { 0xA770UL, 0xA770UL }, { 0xA788UL, 0xA788UL }, - { 0xA7F2UL, 0xA7F4UL }, + { 0xA7F1UL, 0xA7F4UL }, { 0xA7F8UL, 0xA7F9UL }, { 0xA9CFUL, 0xA9CFUL }, { 0xA9E6UL, 0xA9E6UL }, @@ -65,18 +65,22 @@ static const struct range ranges[] = { { 0x107B2UL, 0x107BAUL }, { 0x10D4EUL, 0x10D4EUL }, { 0x10D6FUL, 0x10D6FUL }, + { 0x10EC5UL, 0x10EC5UL }, + { 0x11DD9UL, 0x11DD9UL }, { 0x16B40UL, 0x16B43UL }, { 0x16D40UL, 0x16D42UL }, { 0x16D6BUL, 0x16D6CUL }, { 0x16F93UL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE1UL }, { 0x16FE3UL, 0x16FE3UL }, + { 0x16FF2UL, 0x16FF3UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, { 0x1AFFDUL, 0x1AFFEUL }, { 0x1E030UL, 0x1E06DUL }, { 0x1E137UL, 0x1E13DUL }, { 0x1E4EBUL, 0x1E4EBUL }, + { 0x1E6FFUL, 0x1E6FFUL }, { 0x1E94BUL, 0x1E94BUL } }; diff --git a/src/libre/class/utf8_Lo.c b/src/libre/class/utf8_Lo.c index 0ba2e7a9b..859b511dc 100644 --- a/src/libre/class/utf8_Lo.c +++ b/src/libre/class/utf8_Lo.c @@ -7,7 +7,7 @@ static const struct range ranges[] = { { 0x00BAUL, 0x00BAUL }, { 0x01BBUL, 0x01BBUL }, { 0x01C0UL, 0x01C3UL }, - { 0x0294UL, 0x0294UL }, + { 0x0294UL, 0x0295UL }, { 0x05D0UL, 0x05EAUL }, { 0x05EFUL, 0x05F2UL }, { 0x0620UL, 0x063FUL }, @@ -27,7 +27,7 @@ static const struct range ranges[] = { { 0x0840UL, 0x0858UL }, { 0x0860UL, 0x086AUL }, { 0x0870UL, 0x0887UL }, - { 0x0889UL, 0x088EUL }, + { 0x0889UL, 0x088FUL }, { 0x08A0UL, 0x08C8UL }, { 0x0904UL, 0x0939UL }, { 0x093DUL, 0x093DUL }, @@ -93,7 +93,7 @@ static const struct range ranges[] = { { 0x0C2AUL, 0x0C39UL }, { 0x0C3DUL, 0x0C3DUL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C61UL }, { 0x0C80UL, 0x0C80UL }, { 0x0C85UL, 0x0C8CUL }, @@ -102,7 +102,7 @@ static const struct range ranges[] = { { 0x0CAAUL, 0x0CB3UL }, { 0x0CB5UL, 0x0CB9UL }, { 0x0CBDUL, 0x0CBDUL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE1UL }, { 0x0CF1UL, 0x0CF2UL }, { 0x0D04UL, 0x0D0CUL }, @@ -331,6 +331,7 @@ static const struct range ranges[] = { { 0x108F4UL, 0x108F5UL }, { 0x10900UL, 0x10915UL }, { 0x10920UL, 0x10939UL }, + { 0x10940UL, 0x10959UL }, { 0x10980UL, 0x109B7UL }, { 0x109BEUL, 0x109BFUL }, { 0x10A00UL, 0x10A00UL }, @@ -352,6 +353,7 @@ static const struct range ranges[] = { { 0x10E80UL, 0x10EA9UL }, { 0x10EB0UL, 0x10EB1UL }, { 0x10EC2UL, 0x10EC4UL }, + { 0x10EC6UL, 0x10EC7UL }, { 0x10F00UL, 0x10F1CUL }, { 0x10F27UL, 0x10F27UL }, { 0x10F30UL, 0x10F45UL }, @@ -443,6 +445,8 @@ static const struct range ranges[] = { { 0x11D67UL, 0x11D68UL }, { 0x11D6AUL, 0x11D89UL }, { 0x11D98UL, 0x11D98UL }, + { 0x11DB0UL, 0x11DD8UL }, + { 0x11DDAUL, 0x11DDBUL }, { 0x11EE0UL, 0x11EF2UL }, { 0x11F02UL, 0x11F02UL }, { 0x11F04UL, 0x11F10UL }, @@ -467,10 +471,10 @@ static const struct range ranges[] = { { 0x16F00UL, 0x16F4AUL }, { 0x16F50UL, 0x16F50UL }, { 0x17000UL, 0x17000UL }, - { 0x187F7UL, 0x187F7UL }, - { 0x18800UL, 0x18CD5UL }, + { 0x187FFUL, 0x18CD5UL }, { 0x18CFFUL, 0x18D00UL }, - { 0x18D08UL, 0x18D08UL }, + { 0x18D1EUL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL }, { 0x1B000UL, 0x1B122UL }, { 0x1B132UL, 0x1B132UL }, { 0x1B150UL, 0x1B152UL }, @@ -489,6 +493,12 @@ static const struct range ranges[] = { { 0x1E4D0UL, 0x1E4EAUL }, { 0x1E5D0UL, 0x1E5EDUL }, { 0x1E5F0UL, 0x1E5F0UL }, + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6E2UL }, + { 0x1E6E4UL, 0x1E6E5UL }, + { 0x1E6E7UL, 0x1E6EDUL }, + { 0x1E6F0UL, 0x1E6F4UL }, + { 0x1E6FEUL, 0x1E6FEUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -530,11 +540,10 @@ static const struct range ranges[] = { { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, - { 0x2B739UL, 0x2B739UL }, - { 0x2B740UL, 0x2B740UL }, + { 0x2B73FUL, 0x2B740UL }, { 0x2B81DUL, 0x2B81DUL }, { 0x2B820UL, 0x2B820UL }, - { 0x2CEA1UL, 0x2CEA1UL }, + { 0x2CEADUL, 0x2CEADUL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, { 0x2EBF0UL, 0x2EBF0UL }, @@ -543,7 +552,8 @@ static const struct range ranges[] = { { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, { 0x31350UL, 0x31350UL }, - { 0x323AFUL, 0x323AFUL } + { 0x323AFUL, 0x323B0UL }, + { 0x33479UL, 0x33479UL } }; const struct class utf8_Lo = { diff --git a/src/libre/class/utf8_Lu.c b/src/libre/class/utf8_Lu.c index 150038e7e..e1f752b18 100644 --- a/src/libre/class/utf8_Lu.c +++ b/src/libre/class/utf8_Lu.c @@ -605,7 +605,10 @@ static const struct range ranges[] = { { 0xA7C4UL, 0xA7C7UL }, { 0xA7C9UL, 0xA7C9UL }, { 0xA7CBUL, 0xA7CCUL }, + { 0xA7CEUL, 0xA7CEUL }, { 0xA7D0UL, 0xA7D0UL }, + { 0xA7D2UL, 0xA7D2UL }, + { 0xA7D4UL, 0xA7D4UL }, { 0xA7D6UL, 0xA7D6UL }, { 0xA7D8UL, 0xA7D8UL }, { 0xA7DAUL, 0xA7DAUL }, @@ -622,6 +625,7 @@ static const struct range ranges[] = { { 0x10D50UL, 0x10D65UL }, { 0x118A0UL, 0x118BFUL }, { 0x16E40UL, 0x16E5FUL }, + { 0x16EA0UL, 0x16EB8UL }, { 0x1D400UL, 0x1D419UL }, { 0x1D434UL, 0x1D44DUL }, { 0x1D468UL, 0x1D481UL }, diff --git a/src/libre/class/utf8_M.c b/src/libre/class/utf8_M.c index f0b3e3af6..ce4a80f2b 100644 --- a/src/libre/class/utf8_M.c +++ b/src/libre/class/utf8_M.c @@ -139,7 +139,8 @@ static const struct range ranges[] = { { 0x1A55UL, 0x1A5EUL }, { 0x1A60UL, 0x1A7CUL }, { 0x1A7FUL, 0x1A7FUL }, - { 0x1AB0UL, 0x1ACEUL }, + { 0x1AB0UL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1B00UL, 0x1B04UL }, { 0x1B34UL, 0x1B44UL }, { 0x1B6BUL, 0x1B73UL }, @@ -205,7 +206,7 @@ static const struct range ranges[] = { { 0x10D24UL, 0x10D27UL }, { 0x10D69UL, 0x10D6DUL }, { 0x10EABUL, 0x10EACUL }, - { 0x10EFCUL, 0x10EFFUL }, + { 0x10EFAUL, 0x10EFFUL }, { 0x10F46UL, 0x10F50UL }, { 0x10F82UL, 0x10F85UL }, { 0x11000UL, 0x11002UL }, @@ -267,6 +268,7 @@ static const struct range ranges[] = { { 0x11A47UL, 0x11A47UL }, { 0x11A51UL, 0x11A5BUL }, { 0x11A8AUL, 0x11A99UL }, + { 0x11B60UL, 0x11B67UL }, { 0x11C2FUL, 0x11C36UL }, { 0x11C38UL, 0x11C3FUL }, { 0x11C92UL, 0x11CA7UL }, @@ -321,6 +323,10 @@ static const struct range ranges[] = { { 0x1E2ECUL, 0x1E2EFUL }, { 0x1E4ECUL, 0x1E4EFUL }, { 0x1E5EEUL, 0x1E5EFUL }, + { 0x1E6E3UL, 0x1E6E3UL }, + { 0x1E6E6UL, 0x1E6E6UL }, + { 0x1E6EEUL, 0x1E6EFUL }, + { 0x1E6F5UL, 0x1E6F5UL }, { 0x1E8D0UL, 0x1E8D6UL }, { 0x1E944UL, 0x1E94AUL }, { 0xE0100UL, 0xE01EFUL } diff --git a/src/libre/class/utf8_Mc.c b/src/libre/class/utf8_Mc.c index 77917668c..aa391dcb9 100644 --- a/src/libre/class/utf8_Mc.c +++ b/src/libre/class/utf8_Mc.c @@ -175,6 +175,9 @@ static const struct range ranges[] = { { 0x11A39UL, 0x11A39UL }, { 0x11A57UL, 0x11A58UL }, { 0x11A97UL, 0x11A97UL }, + { 0x11B61UL, 0x11B61UL }, + { 0x11B65UL, 0x11B65UL }, + { 0x11B67UL, 0x11B67UL }, { 0x11C2FUL, 0x11C2FUL }, { 0x11C3EUL, 0x11C3EUL }, { 0x11CA9UL, 0x11CA9UL }, diff --git a/src/libre/class/utf8_Mn.c b/src/libre/class/utf8_Mn.c index c204fa592..4ad1f3ddd 100644 --- a/src/libre/class/utf8_Mn.c +++ b/src/libre/class/utf8_Mn.c @@ -145,7 +145,8 @@ static const struct range ranges[] = { { 0x1A73UL, 0x1A7CUL }, { 0x1A7FUL, 0x1A7FUL }, { 0x1AB0UL, 0x1ABDUL }, - { 0x1ABFUL, 0x1ACEUL }, + { 0x1ABFUL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1B00UL, 0x1B03UL }, { 0x1B34UL, 0x1B34UL }, { 0x1B36UL, 0x1B3AUL }, @@ -227,7 +228,7 @@ static const struct range ranges[] = { { 0x10D24UL, 0x10D27UL }, { 0x10D69UL, 0x10D6DUL }, { 0x10EABUL, 0x10EACUL }, - { 0x10EFCUL, 0x10EFFUL }, + { 0x10EFAUL, 0x10EFFUL }, { 0x10F46UL, 0x10F50UL }, { 0x10F82UL, 0x10F85UL }, { 0x11001UL, 0x11001UL }, @@ -302,6 +303,9 @@ static const struct range ranges[] = { { 0x11A59UL, 0x11A5BUL }, { 0x11A8AUL, 0x11A96UL }, { 0x11A98UL, 0x11A99UL }, + { 0x11B60UL, 0x11B60UL }, + { 0x11B62UL, 0x11B64UL }, + { 0x11B66UL, 0x11B66UL }, { 0x11C30UL, 0x11C36UL }, { 0x11C38UL, 0x11C3DUL }, { 0x11C3FUL, 0x11C3FUL }, @@ -357,6 +361,10 @@ static const struct range ranges[] = { { 0x1E2ECUL, 0x1E2EFUL }, { 0x1E4ECUL, 0x1E4EFUL }, { 0x1E5EEUL, 0x1E5EFUL }, + { 0x1E6E3UL, 0x1E6E3UL }, + { 0x1E6E6UL, 0x1E6E6UL }, + { 0x1E6EEUL, 0x1E6EFUL }, + { 0x1E6F5UL, 0x1E6F5UL }, { 0x1E8D0UL, 0x1E8D6UL }, { 0x1E944UL, 0x1E94AUL }, { 0xE0100UL, 0xE01EFUL } diff --git a/src/libre/class/utf8_N.c b/src/libre/class/utf8_N.c index a4b7fdebc..fa7dcc037 100644 --- a/src/libre/class/utf8_N.c +++ b/src/libre/class/utf8_N.c @@ -119,6 +119,7 @@ static const struct range ranges[] = { { 0x11C50UL, 0x11C6CUL }, { 0x11D50UL, 0x11D59UL }, { 0x11DA0UL, 0x11DA9UL }, + { 0x11DE0UL, 0x11DE9UL }, { 0x11F50UL, 0x11F59UL }, { 0x11FC0UL, 0x11FD4UL }, { 0x12400UL, 0x1246EUL }, @@ -129,6 +130,7 @@ static const struct range ranges[] = { { 0x16B5BUL, 0x16B61UL }, { 0x16D70UL, 0x16D79UL }, { 0x16E80UL, 0x16E96UL }, + { 0x16FF4UL, 0x16FF6UL }, { 0x1CCF0UL, 0x1CCF9UL }, { 0x1D2C0UL, 0x1D2D3UL }, { 0x1D2E0UL, 0x1D2F3UL }, diff --git a/src/libre/class/utf8_Nd.c b/src/libre/class/utf8_Nd.c index 7506f2d2e..19822a94b 100644 --- a/src/libre/class/utf8_Nd.c +++ b/src/libre/class/utf8_Nd.c @@ -60,6 +60,7 @@ static const struct range ranges[] = { { 0x11C50UL, 0x11C59UL }, { 0x11D50UL, 0x11D59UL }, { 0x11DA0UL, 0x11DA9UL }, + { 0x11DE0UL, 0x11DE9UL }, { 0x11F50UL, 0x11F59UL }, { 0x16130UL, 0x16139UL }, { 0x16A60UL, 0x16A69UL }, diff --git a/src/libre/class/utf8_Nl.c b/src/libre/class/utf8_Nl.c index fdf83eb7f..71097a550 100644 --- a/src/libre/class/utf8_Nl.c +++ b/src/libre/class/utf8_Nl.c @@ -14,7 +14,8 @@ static const struct range ranges[] = { { 0x10341UL, 0x10341UL }, { 0x1034AUL, 0x1034AUL }, { 0x103D1UL, 0x103D5UL }, - { 0x12400UL, 0x1246EUL } + { 0x12400UL, 0x1246EUL }, + { 0x16FF4UL, 0x16FF6UL } }; const struct class utf8_Nl = { diff --git a/src/libre/class/utf8_P.c b/src/libre/class/utf8_P.c index 9c506bd13..664a1d28c 100644 --- a/src/libre/class/utf8_P.c +++ b/src/libre/class/utf8_P.c @@ -151,6 +151,7 @@ static const struct range ranges[] = { { 0x10B99UL, 0x10B9CUL }, { 0x10D6EUL, 0x10D6EUL }, { 0x10EADUL, 0x10EADUL }, + { 0x10ED0UL, 0x10ED0UL }, { 0x10F55UL, 0x10F59UL }, { 0x10F86UL, 0x10F89UL }, { 0x11047UL, 0x1104DUL }, diff --git a/src/libre/class/utf8_Po.c b/src/libre/class/utf8_Po.c index ef95b8586..4b20947c0 100644 --- a/src/libre/class/utf8_Po.c +++ b/src/libre/class/utf8_Po.c @@ -146,6 +146,7 @@ static const struct range ranges[] = { { 0x10AF0UL, 0x10AF6UL }, { 0x10B39UL, 0x10B3FUL }, { 0x10B99UL, 0x10B9CUL }, + { 0x10ED0UL, 0x10ED0UL }, { 0x10F55UL, 0x10F59UL }, { 0x10F86UL, 0x10F89UL }, { 0x11047UL, 0x1104DUL }, diff --git a/src/libre/class/utf8_S.c b/src/libre/class/utf8_S.c index 0b69e43fe..9b66a152c 100644 --- a/src/libre/class/utf8_S.c +++ b/src/libre/class/utf8_S.c @@ -75,7 +75,7 @@ static const struct range ranges[] = { { 0x2052UL, 0x2052UL }, { 0x207AUL, 0x207CUL }, { 0x208AUL, 0x208CUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0x2100UL, 0x2101UL }, { 0x2103UL, 0x2106UL }, { 0x2108UL, 0x2109UL }, @@ -103,8 +103,7 @@ static const struct range ranges[] = { { 0x2999UL, 0x29D7UL }, { 0x29DCUL, 0x29FBUL }, { 0x29FEUL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2BFFUL }, + { 0x2B76UL, 0x2BFFUL }, { 0x2CE5UL, 0x2CEAUL }, { 0x2E50UL, 0x2E51UL }, { 0x2E80UL, 0x2E99UL }, @@ -138,9 +137,10 @@ static const struct range ranges[] = { { 0xAB5BUL, 0xAB5BUL }, { 0xAB6AUL, 0xAB6BUL }, { 0xFB29UL, 0xFB29UL }, - { 0xFBB2UL, 0xFBC2UL }, + { 0xFBB2UL, 0xFBD2UL }, { 0xFD40UL, 0xFD4FUL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFD90UL, 0xFD91UL }, + { 0xFDC8UL, 0xFDCFUL }, { 0xFDFCUL, 0xFDFFUL }, { 0xFE62UL, 0xFE62UL }, { 0xFE64UL, 0xFE66UL }, @@ -164,13 +164,17 @@ static const struct range ranges[] = { { 0x10877UL, 0x10878UL }, { 0x10AC8UL, 0x10AC8UL }, { 0x10D8EUL, 0x10D8FUL }, + { 0x10ED1UL, 0x10ED8UL }, { 0x1173FUL, 0x1173FUL }, { 0x11FD5UL, 0x11FF1UL }, { 0x16B3CUL, 0x16B3FUL }, { 0x16B45UL, 0x16B45UL }, { 0x1BC9CUL, 0x1BC9CUL }, { 0x1CC00UL, 0x1CCEFUL }, + { 0x1CCFAUL, 0x1CCFCUL }, { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEF0UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -215,11 +219,10 @@ static const struct range ranges[] = { { 0x1F240UL, 0x1F248UL }, { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, - { 0x1F300UL, 0x1F6D7UL }, + { 0x1F300UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -229,16 +232,19 @@ static const struct range ranges[] = { { 0x1F890UL, 0x1F8ADUL }, { 0x1F8B0UL, 0x1F8BBUL }, { 0x1F8C0UL, 0x1F8C1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8D0UL, 0x1F8D8UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA89UL }, - { 0x1FA8FUL, 0x1FAC6UL }, - { 0x1FACEUL, 0x1FADCUL }, - { 0x1FADFUL, 0x1FAE9UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBEFUL } + { 0x1FB94UL, 0x1FBEFUL }, + { 0x1FBFAUL, 0x1FBFAUL } }; const struct class utf8_S = { diff --git a/src/libre/class/utf8_Sc.c b/src/libre/class/utf8_Sc.c index 0effc2b86..3bba263b8 100644 --- a/src/libre/class/utf8_Sc.c +++ b/src/libre/class/utf8_Sc.c @@ -14,7 +14,7 @@ static const struct range ranges[] = { { 0x0BF9UL, 0x0BF9UL }, { 0x0E3FUL, 0x0E3FUL }, { 0x17DBUL, 0x17DBUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0xA838UL, 0xA838UL }, { 0xFDFCUL, 0xFDFCUL }, { 0xFE69UL, 0xFE69UL }, diff --git a/src/libre/class/utf8_Sharada.c b/src/libre/class/utf8_Sharada.c index e51b2edff..dbfa21d3f 100644 --- a/src/libre/class/utf8_Sharada.c +++ b/src/libre/class/utf8_Sharada.c @@ -3,7 +3,8 @@ #include "class.h" static const struct range ranges[] = { - { 0x11180UL, 0x111DFUL } + { 0x11180UL, 0x111DFUL }, + { 0x11B60UL, 0x11B67UL } }; const struct class utf8_Sharada = { diff --git a/src/libre/class/utf8_Sidetic.c b/src/libre/class/utf8_Sidetic.c new file mode 100644 index 000000000..ed53b543d --- /dev/null +++ b/src/libre/class/utf8_Sidetic.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x10940UL, 0x10959UL } +}; + +const struct class utf8_Sidetic = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Sm.c b/src/libre/class/utf8_Sm.c index 9c39b8003..e157e5f8c 100644 --- a/src/libre/class/utf8_Sm.c +++ b/src/libre/class/utf8_Sm.c @@ -57,6 +57,7 @@ static const struct range ranges[] = { { 0xFFE2UL, 0xFFE2UL }, { 0xFFE9UL, 0xFFECUL }, { 0x10D8EUL, 0x10D8FUL }, + { 0x1CEF0UL, 0x1CEF0UL }, { 0x1D6C1UL, 0x1D6C1UL }, { 0x1D6DBUL, 0x1D6DBUL }, { 0x1D6FBUL, 0x1D6FBUL }, @@ -67,7 +68,8 @@ static const struct range ranges[] = { { 0x1D789UL, 0x1D789UL }, { 0x1D7A9UL, 0x1D7A9UL }, { 0x1D7C3UL, 0x1D7C3UL }, - { 0x1EEF0UL, 0x1EEF1UL } + { 0x1EEF0UL, 0x1EEF1UL }, + { 0x1F8D0UL, 0x1F8D8UL } }; const struct class utf8_Sm = { diff --git a/src/libre/class/utf8_So.c b/src/libre/class/utf8_So.c index d7e608ca4..93ca7e36b 100644 --- a/src/libre/class/utf8_So.c +++ b/src/libre/class/utf8_So.c @@ -82,8 +82,7 @@ static const struct range ranges[] = { { 0x2B00UL, 0x2B2FUL }, { 0x2B45UL, 0x2B46UL }, { 0x2B4DUL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2BFFUL }, + { 0x2B76UL, 0x2BFFUL }, { 0x2CE5UL, 0x2CEAUL }, { 0x2E50UL, 0x2E51UL }, { 0x2E80UL, 0x2E99UL }, @@ -111,8 +110,10 @@ static const struct range ranges[] = { { 0xA836UL, 0xA837UL }, { 0xA839UL, 0xA839UL }, { 0xAA77UL, 0xAA79UL }, + { 0xFBC3UL, 0xFBD2UL }, { 0xFD40UL, 0xFD4FUL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFD90UL, 0xFD91UL }, + { 0xFDC8UL, 0xFDCFUL }, { 0xFDFDUL, 0xFDFFUL }, { 0xFFE4UL, 0xFFE4UL }, { 0xFFE8UL, 0xFFE8UL }, @@ -126,6 +127,7 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x10877UL, 0x10878UL }, { 0x10AC8UL, 0x10AC8UL }, + { 0x10ED1UL, 0x10ED8UL }, { 0x1173FUL, 0x1173FUL }, { 0x11FD5UL, 0x11FDCUL }, { 0x11FE1UL, 0x11FF1UL }, @@ -133,7 +135,10 @@ static const struct range ranges[] = { { 0x16B45UL, 0x16B45UL }, { 0x1BC9CUL, 0x1BC9CUL }, { 0x1CC00UL, 0x1CCEFUL }, + { 0x1CCFAUL, 0x1CCFCUL }, { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEEFUL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -166,11 +171,10 @@ static const struct range ranges[] = { { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, { 0x1F300UL, 0x1F3FAUL }, - { 0x1F400UL, 0x1F6D7UL }, + { 0x1F400UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -180,16 +184,18 @@ static const struct range ranges[] = { { 0x1F890UL, 0x1F8ADUL }, { 0x1F8B0UL, 0x1F8BBUL }, { 0x1F8C0UL, 0x1F8C1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA89UL }, - { 0x1FA8FUL, 0x1FAC6UL }, - { 0x1FACEUL, 0x1FADCUL }, - { 0x1FADFUL, 0x1FAE9UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBEFUL } + { 0x1FB94UL, 0x1FBEFUL }, + { 0x1FBFAUL, 0x1FBFAUL } }; const struct class utf8_So = { diff --git a/src/libre/class/utf8_Tai_Yo.c b/src/libre/class/utf8_Tai_Yo.c new file mode 100644 index 000000000..643d73fe2 --- /dev/null +++ b/src/libre/class/utf8_Tai_Yo.c @@ -0,0 +1,15 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6F5UL }, + { 0x1E6FEUL, 0x1E6FFUL } +}; + +const struct class utf8_Tai_Yo = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Tangut.c b/src/libre/class/utf8_Tangut.c index cbd82c907..6bf5e5687 100644 --- a/src/libre/class/utf8_Tangut.c +++ b/src/libre/class/utf8_Tangut.c @@ -4,9 +4,9 @@ static const struct range ranges[] = { { 0x16FE0UL, 0x16FE0UL }, - { 0x17000UL, 0x187F7UL }, - { 0x18800UL, 0x18AFFUL }, - { 0x18D00UL, 0x18D08UL } + { 0x17000UL, 0x18AFFUL }, + { 0x18D00UL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL } }; const struct class utf8_Tangut = { diff --git a/src/libre/class/utf8_Telugu.c b/src/libre/class/utf8_Telugu.c index a2c65ae49..1b80f5e1f 100644 --- a/src/libre/class/utf8_Telugu.c +++ b/src/libre/class/utf8_Telugu.c @@ -12,7 +12,7 @@ static const struct range ranges[] = { { 0x0C4AUL, 0x0C4DUL }, { 0x0C55UL, 0x0C56UL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C63UL }, { 0x0C66UL, 0x0C6FUL }, { 0x0C77UL, 0x0C7FUL } diff --git a/src/libre/class/utf8_Tolong_Siki.c b/src/libre/class/utf8_Tolong_Siki.c new file mode 100644 index 000000000..e670a957b --- /dev/null +++ b/src/libre/class/utf8_Tolong_Siki.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x11DB0UL, 0x11DDBUL }, + { 0x11DE0UL, 0x11DE9UL } +}; + +const struct class utf8_Tolong_Siki = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_assigned.c b/src/libre/class/utf8_assigned.c index d38345570..32b689a26 100644 --- a/src/libre/class/utf8_assigned.c +++ b/src/libre/class/utf8_assigned.c @@ -24,8 +24,7 @@ static const struct range ranges[] = { { 0x0840UL, 0x085BUL }, { 0x085EUL, 0x085EUL }, { 0x0860UL, 0x086AUL }, - { 0x0870UL, 0x088EUL }, - { 0x0890UL, 0x0891UL }, + { 0x0870UL, 0x0891UL }, { 0x0897UL, 0x0983UL }, { 0x0985UL, 0x098CUL }, { 0x098FUL, 0x0990UL }, @@ -109,7 +108,7 @@ static const struct range ranges[] = { { 0x0C4AUL, 0x0C4DUL }, { 0x0C55UL, 0x0C56UL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C63UL }, { 0x0C66UL, 0x0C6FUL }, { 0x0C77UL, 0x0C8CUL }, @@ -121,7 +120,7 @@ static const struct range ranges[] = { { 0x0CC6UL, 0x0CC8UL }, { 0x0CCAUL, 0x0CCDUL }, { 0x0CD5UL, 0x0CD6UL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE3UL }, { 0x0CE6UL, 0x0CEFUL }, { 0x0CF1UL, 0x0CF3UL }, @@ -216,7 +215,8 @@ static const struct range ranges[] = { { 0x1A7FUL, 0x1A89UL }, { 0x1A90UL, 0x1A99UL }, { 0x1AA0UL, 0x1AADUL }, - { 0x1AB0UL, 0x1ACEUL }, + { 0x1AB0UL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1B00UL, 0x1B4CUL }, { 0x1B4EUL, 0x1BF3UL }, { 0x1BFCUL, 0x1C37UL }, @@ -245,14 +245,13 @@ static const struct range ranges[] = { { 0x2066UL, 0x2071UL }, { 0x2074UL, 0x208EUL }, { 0x2090UL, 0x209CUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0x20D0UL, 0x20F0UL }, { 0x2100UL, 0x218BUL }, { 0x2190UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x2460UL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2CF3UL }, + { 0x2B76UL, 0x2CF3UL }, { 0x2CF9UL, 0x2D25UL }, { 0x2D27UL, 0x2D27UL }, { 0x2D2DUL, 0x2D2DUL }, @@ -284,11 +283,8 @@ static const struct range ranges[] = { { 0xA490UL, 0xA4C6UL }, { 0xA4D0UL, 0xA62BUL }, { 0xA640UL, 0xA6F7UL }, - { 0xA700UL, 0xA7CDUL }, - { 0xA7D0UL, 0xA7D1UL }, - { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7DCUL }, - { 0xA7F2UL, 0xA82CUL }, + { 0xA700UL, 0xA7DCUL }, + { 0xA7F1UL, 0xA82CUL }, { 0xA830UL, 0xA839UL }, { 0xA840UL, 0xA877UL }, { 0xA880UL, 0xA8C5UL }, @@ -328,10 +324,7 @@ static const struct range ranges[] = { { 0xFB3EUL, 0xFB3EUL }, { 0xFB40UL, 0xFB41UL }, { 0xFB43UL, 0xFB44UL }, - { 0xFB46UL, 0xFBC2UL }, - { 0xFBD3UL, 0xFD8FUL }, - { 0xFD92UL, 0xFDC7UL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFB46UL, 0xFDCFUL }, { 0xFDF0UL, 0xFE19UL }, { 0xFE20UL, 0xFE52UL }, { 0xFE54UL, 0xFE66UL }, @@ -402,7 +395,7 @@ static const struct range ranges[] = { { 0x108F4UL, 0x108F5UL }, { 0x108FBUL, 0x1091BUL }, { 0x1091FUL, 0x10939UL }, - { 0x1093FUL, 0x1093FUL }, + { 0x1093FUL, 0x10959UL }, { 0x10980UL, 0x109B7UL }, { 0x109BCUL, 0x109CFUL }, { 0x109D2UL, 0x10A03UL }, @@ -434,8 +427,9 @@ static const struct range ranges[] = { { 0x10E80UL, 0x10EA9UL }, { 0x10EABUL, 0x10EADUL }, { 0x10EB0UL, 0x10EB1UL }, - { 0x10EC2UL, 0x10EC4UL }, - { 0x10EFCUL, 0x10F27UL }, + { 0x10EC2UL, 0x10EC7UL }, + { 0x10ED0UL, 0x10ED8UL }, + { 0x10EFAUL, 0x10F27UL }, { 0x10F30UL, 0x10F59UL }, { 0x10F70UL, 0x10F89UL }, { 0x10FB0UL, 0x10FCBUL }, @@ -518,6 +512,7 @@ static const struct range ranges[] = { { 0x11A50UL, 0x11AA2UL }, { 0x11AB0UL, 0x11AF8UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11B60UL, 0x11B67UL }, { 0x11BC0UL, 0x11BE1UL }, { 0x11BF0UL, 0x11BF9UL }, { 0x11C00UL, 0x11C08UL }, @@ -540,6 +535,8 @@ static const struct range ranges[] = { { 0x11D90UL, 0x11D91UL }, { 0x11D93UL, 0x11D98UL }, { 0x11DA0UL, 0x11DA9UL }, + { 0x11DB0UL, 0x11DDBUL }, + { 0x11DE0UL, 0x11DE9UL }, { 0x11EE0UL, 0x11EF8UL }, { 0x11F00UL, 0x11F10UL }, { 0x11F12UL, 0x11F3AUL }, @@ -569,16 +566,18 @@ static const struct range ranges[] = { { 0x16B7DUL, 0x16B8FUL }, { 0x16D40UL, 0x16D79UL }, { 0x16E40UL, 0x16E9AUL }, + { 0x16EA0UL, 0x16EB8UL }, + { 0x16EBBUL, 0x16ED3UL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F4FUL, 0x16F87UL }, { 0x16F8FUL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE4UL }, - { 0x16FF0UL, 0x16FF1UL }, + { 0x16FF0UL, 0x16FF6UL }, { 0x17000UL, 0x17000UL }, - { 0x187F7UL, 0x187F7UL }, - { 0x18800UL, 0x18CD5UL }, + { 0x187FFUL, 0x18CD5UL }, { 0x18CFFUL, 0x18D00UL }, - { 0x18D08UL, 0x18D08UL }, + { 0x18D1EUL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, { 0x1AFFDUL, 0x1AFFEUL }, @@ -593,8 +592,10 @@ static const struct range ranges[] = { { 0x1BC80UL, 0x1BC88UL }, { 0x1BC90UL, 0x1BC99UL }, { 0x1BC9CUL, 0x1BCA3UL }, - { 0x1CC00UL, 0x1CCF9UL }, + { 0x1CC00UL, 0x1CCFCUL }, { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEF0UL }, { 0x1CF00UL, 0x1CF2DUL }, { 0x1CF30UL, 0x1CF46UL }, { 0x1CF50UL, 0x1CFC3UL }, @@ -648,6 +649,9 @@ static const struct range ranges[] = { { 0x1E4D0UL, 0x1E4F9UL }, { 0x1E5D0UL, 0x1E5FAUL }, { 0x1E5FFUL, 0x1E5FFUL }, + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6F5UL }, + { 0x1E6FEUL, 0x1E6FFUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -705,11 +709,10 @@ static const struct range ranges[] = { { 0x1F240UL, 0x1F248UL }, { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, - { 0x1F300UL, 0x1F6D7UL }, + { 0x1F300UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -719,24 +722,25 @@ static const struct range ranges[] = { { 0x1F890UL, 0x1F8ADUL }, { 0x1F8B0UL, 0x1F8BBUL }, { 0x1F8C0UL, 0x1F8C1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8D0UL, 0x1F8D8UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA89UL }, - { 0x1FA8FUL, 0x1FAC6UL }, - { 0x1FACEUL, 0x1FADCUL }, - { 0x1FADFUL, 0x1FAE9UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBF9UL }, + { 0x1FB94UL, 0x1FBFAUL }, { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, - { 0x2B739UL, 0x2B739UL }, - { 0x2B740UL, 0x2B740UL }, + { 0x2B73FUL, 0x2B740UL }, { 0x2B81DUL, 0x2B81DUL }, { 0x2B820UL, 0x2B820UL }, - { 0x2CEA1UL, 0x2CEA1UL }, + { 0x2CEADUL, 0x2CEADUL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, { 0x2EBF0UL, 0x2EBF0UL }, @@ -745,7 +749,8 @@ static const struct range ranges[] = { { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, { 0x31350UL, 0x31350UL }, - { 0x323AFUL, 0x323AFUL }, + { 0x323AFUL, 0x323B0UL }, + { 0x33479UL, 0x33479UL }, { 0xE0001UL, 0xE0001UL }, { 0xE0020UL, 0xE007FUL }, { 0xE0100UL, 0xE01EFUL }, diff --git a/src/libre/class_name.c b/src/libre/class_name.c index d96a1d610..e4cb7cc0f 100644 --- a/src/libre/class_name.c +++ b/src/libre/class_name.c @@ -50,6 +50,7 @@ static struct { { &utf8_Bassa_Vah, "Bassa Vah" }, { &utf8_Batak, "Batak" }, { &utf8_Bengali, "Bengali" }, + { &utf8_Beria_Erfe, "Beria Erfe" }, { &utf8_Bhaiksuki, "Bhaiksuki" }, { &utf8_Bopomofo, "Bopomofo" }, { &utf8_Brahmi, "Brahmi" }, @@ -174,6 +175,7 @@ static struct { { &utf8_Sharada, "Sharada" }, { &utf8_Shavian, "Shavian" }, { &utf8_Siddham, "Siddham" }, + { &utf8_Sidetic, "Sidetic" }, { &utf8_SignWriting, "SignWriting" }, { &utf8_Sinhala, "Sinhala" }, { &utf8_Sogdian, "Sogdian" }, @@ -188,6 +190,7 @@ static struct { { &utf8_Tai_Le, "Tai Le" }, { &utf8_Tai_Tham, "Tai Tham" }, { &utf8_Tai_Viet, "Tai Viet" }, + { &utf8_Tai_Yo, "Tai Yo" }, { &utf8_Takri, "Takri" }, { &utf8_Tamil, "Tamil" }, { &utf8_Tangsa, "Tangsa" }, @@ -199,6 +202,7 @@ static struct { { &utf8_Tifinagh, "Tifinagh" }, { &utf8_Tirhuta, "Tirhuta" }, { &utf8_Todhri, "Todhri" }, + { &utf8_Tolong_Siki, "Tolong Siki" }, { &utf8_Toto, "Toto" }, { &utf8_Tulu_Tigalari, "Tulu Tigalari" }, { &utf8_Ugaritic, "Ugaritic" }, From 4155d56a3cf65ffd026aa6d22cae510147652001 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 27 Aug 2024 14:08:39 -0400 Subject: [PATCH 311/373] experimental: Add eager outputs, similar to endids but eagerly matched. When combining several unanchored regexes it becomes VERY expensive to handle combinations of matches via the end state -- essentially, the whole reachable DFA gets separate matching and non-matching copies for each pattern, leading to a DFA whose size is proportional to the number of *possible combinations* of matches. With eager outputs, we can set a flag for matching as we reach the end of the original pattern (before looping back and possibly also matching other patterns), which keeps the state count from blowing up in fsm_determinise. To see how much difference this makes, the test tests/eager_output/run7 combines 26 different patterns. It should finish very quickly (~50 msec, just now). Try running it with `env FORCE_ENDIDS=N` for N increasing from 4 to 26. Around 10-11 it will start taking several seconds, and memory usage will roughly double with each step. This PR adds `fsm_union_repeated_pattern_group`, a variant of `fsm_union_array` that combines a set of DFAs into a single NFA, but correctly handles a mix of anchored and unanchored ends without the state count blowing up. It currently needs flags passed in for each fsm indicating whether the start and/or end are anchored, and there is a hacky special case that removes self-edges from states with eager outputs and instead connects them to a single overall unanchored end loop. I haven't yet figured out how to handle this properly in the general case, but it works for this specific use case, provided all the DFAs are combined at once. (Combining multiple DFAs each produced by determinising fsm_union_repeated_pattern_group's result probably won't work correctly.) I have tried detecting and ignoring those edges in fsm_determinise, after epsilon removal, but so far either it still causes the graph size to blow up or subtly breaks something else. This is still experimental, and the code generation for `-lc` here is quite hacky -- it expects the caller to define a `FSM_SET_EAGER_OUTPUT` macro, since the code generation interface doesn't define where the match info will go yet. A later PR will add a new code generation mode with better support for eager outputs, and I plan to eventually integrate this better with rx, AMBIG_MULTIPLE, and so on. (This squashes down a couple false starts.) --- Makefile | 1 + fuzz/target.c | 524 +++++++++++++++++- include/fsm/bool.h | 10 + include/fsm/fsm.h | 46 ++ include/fsm/print.h | 3 + src/libfsm/Makefile | 1 + src/libfsm/clone.c | 38 ++ src/libfsm/consolidate.c | 46 ++ src/libfsm/determinise.c | 131 ++++- src/libfsm/determinise_internal.h | 10 +- src/libfsm/eager_output.c | 403 ++++++++++++++ src/libfsm/eager_output.h | 46 ++ src/libfsm/epsilons.c | 147 ++++- src/libfsm/exec.c | 55 +- src/libfsm/fsm.c | 11 + src/libfsm/internal.h | 5 + src/libfsm/libfsm.syms | 11 + src/libfsm/merge.c | 42 ++ src/libfsm/minimise.c | 79 ++- src/libfsm/print/c.c | 13 + src/libfsm/print/ir.c | 35 ++ src/libfsm/print/ir.h | 5 + src/libfsm/state.c | 8 + src/libfsm/union.c | 233 ++++++++ tests/eager_output/Makefile | 22 + tests/eager_output/eager_output1.c | 12 + tests/eager_output/eager_output2.c | 17 + tests/eager_output/eager_output3.c | 16 + tests/eager_output/eager_output4.c | 13 + tests/eager_output/eager_output5.c | 14 + tests/eager_output/eager_output6.c | 34 ++ tests/eager_output/eager_output7.c | 103 ++++ tests/eager_output/eager_output_at_start.c | 12 + tests/eager_output/eager_output_fr1.c | 13 + tests/eager_output/eager_output_fr2.c | 13 + tests/eager_output/eager_output_fr3.c | 13 + .../eager_output_mixed_anchored_unanchored.c | 46 ++ tests/eager_output/utils.c | 278 ++++++++++ tests/eager_output/utils.h | 64 +++ 39 files changed, 2537 insertions(+), 36 deletions(-) create mode 100644 src/libfsm/eager_output.c create mode 100644 src/libfsm/eager_output.h create mode 100644 tests/eager_output/Makefile create mode 100644 tests/eager_output/eager_output1.c create mode 100644 tests/eager_output/eager_output2.c create mode 100644 tests/eager_output/eager_output3.c create mode 100644 tests/eager_output/eager_output4.c create mode 100644 tests/eager_output/eager_output5.c create mode 100644 tests/eager_output/eager_output6.c create mode 100644 tests/eager_output/eager_output7.c create mode 100644 tests/eager_output/eager_output_at_start.c create mode 100644 tests/eager_output/eager_output_fr1.c create mode 100644 tests/eager_output/eager_output_fr2.c create mode 100644 tests/eager_output/eager_output_fr3.c create mode 100644 tests/eager_output/eager_output_mixed_anchored_unanchored.c create mode 100644 tests/eager_output/utils.c create mode 100644 tests/eager_output/utils.h diff --git a/Makefile b/Makefile index b9e196d7b..ad16b1ec7 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,7 @@ SUBDIR += tests/equals SUBDIR += tests/subtract SUBDIR += tests/detect_required SUBDIR += tests/determinise +SUBDIR += tests/eager_output SUBDIR += tests/endids SUBDIR += tests/epsilons SUBDIR += tests/fsm diff --git a/fuzz/target.c b/fuzz/target.c index 543891bb9..d56a9bf82 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -26,10 +26,21 @@ /* 10 seconds */ #define TIMEOUT_USEC (10ULL * 1000 * 1000) +static bool verbosity_checked = false; +static bool verbose = false; + +#define LOG(...) \ + do { \ + if (verbose) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while (0) \ + enum run_mode { MODE_DEFAULT, MODE_SHUFFLE_MINIMISE, MODE_ALL_PRINT_FUNCTIONS, + MODE_EAGER_OUTPUT, }; @@ -344,6 +355,508 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const return EXIT_SUCCESS; } +#define MAX_PATTERNS 4 +struct eager_output_cb_info { + size_t used; + fsm_output_id_t ids[MAX_PATTERNS]; +}; + +static void +reset_eager_output_info(struct eager_output_cb_info *info) +{ + info->used = 0; +} + +struct feo_env { + bool ok; + size_t pattern_count; + size_t fsm_count; + size_t max_match_count; + size_t max_steps; + + char *patterns[MAX_PATTERNS]; + struct fsm *fsms[MAX_PATTERNS]; + struct fsm *combined; + + /* which pattern is being used for generation, (size_t)-1 for combined */ + size_t current_pattern; + + struct eager_output_cb_info outputs; + struct eager_output_cb_info outputs_combined; +}; + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque) +{ + struct eager_output_cb_info *info = (struct eager_output_cb_info *)opaque; + + for (size_t i = 0; i < info->used; i++) { + if (info->ids[i] == id) { + return; /* already present */ + } + } + + assert(info->used < MAX_PATTERNS); + info->ids[info->used++] = id; +} + +static enum fsm_generate_matches_cb_res +gen_combined_check_individual_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +static enum fsm_generate_matches_cb_res +gen_individual_check_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +#define DEF_MAX_STEPS 100000 +#define DEF_MAX_MATCH_COUNT 1000 + +/* This isn't part of the public interface, per se. */ +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +static int +fuzz_eager_output(const uint8_t *data, size_t size) +{ + struct feo_env env = { + .ok = true, + .pattern_count = 0, + .max_steps = DEF_MAX_STEPS, + .max_match_count = DEF_MAX_MATCH_COUNT, + }; + + { + const char *steps = getenv("STEPS"); + const char *matches = getenv("MATCHES"); + if (steps != NULL) { + env.max_steps = strtoul(steps, NULL, 10); + assert(env.max_steps > 0); + } + if (matches != NULL) { + env.max_match_count = strtoul(matches, NULL, 10); + assert(env.max_match_count > 0); + } + } + + int ret = 0; + + size_t max_pattern_length = 0; + + /* chop data into a series of patterns */ + { + size_t prev = 0; + size_t offset = 0; + + /* Patterns with lots of '.' can take a while to determinise. + * That slows down fuzzer coverage, but isn't interesting here. */ + size_t dots = 0; + + while (offset < size && env.pattern_count < MAX_PATTERNS) { +#define MAX_DOTS 4 + if (data[offset] == '.') { dots++; } + + if (data[offset] == '\0' || data[offset] == '\n' || offset == size - 1) { + size_t len = offset - prev; + + if (dots > MAX_DOTS) { + /* ignored */ + prev = offset; + } else if (len > 0) { + char *pattern = malloc(len + 1); + assert(pattern != NULL); + + memcpy(pattern, &data[prev], len); + if (len > 0 && pattern[len] == '\n') { + len--; /* drop trailing newline */ + } + pattern[len] = '\0'; + bool keep = true; + + if (len > 0) { + for (size_t i = 0; i < len - 1; i++) { + if (pattern[i] == '\\' && pattern[i + 1] == 'x') { + /* ignore unhandled parser errors from "\x", see #386 */ + keep = false; + } + } + } + + if (keep) { + env.patterns[env.pattern_count++] = pattern; + + if (len > max_pattern_length) { + max_pattern_length = len; + } + } else { + free(pattern); + } + prev = offset; + dots = 0; + } + } + + offset++; + } + } + + struct re_anchoring_info anchorage[MAX_PATTERNS] = {0}; + + /* for each pattern, attempt to compile to a DFA */ + for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { + const char *p = env.patterns[p_i]; + + if (!re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL, &anchorage[p_i])) { + continue; /* unsupported regex */ + } + + p = env.patterns[p_i]; + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); + + LOG("%s: pattern %zd: '%s' => %p\n", __func__, p_i, env.patterns[p_i], (void *)fsm); + + if (fsm == NULL) { + continue; /* invalid regex */ + } + + const fsm_output_id_t endid = (fsm_output_id_t)p_i; + ret = fsm_seteageroutputonends(fsm, endid); + assert(ret == 1); + + if (verbose) { + fprintf(stderr, "==== pattern %zd, pre det\n", p_i); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(fsm); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(fsm, i)); + } + } + + ret = fsm_determinise(fsm); + assert(ret == 1); + + ret = fsm_minimise(fsm); + assert(ret == 1); + + fsm_state_t start; + if (!fsm_getstart(fsm, &start)) { + fsm_free(fsm); + continue; + } + + if (verbose) { + fprintf(stderr, "==== pattern %zd, post det\n", p_i); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(fsm); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(fsm, i)); + } + } + + fsm_eager_output_set_cb(fsm, append_eager_output_cb, &env.outputs); + env.fsms[env.fsm_count++] = fsm; + } + + /* don't bother checking combined behavior unless there's multiple DFAs */ + if (env.fsm_count < 2) { goto cleanup; } + + /* copy and combine fsms into one DFA */ + { + size_t used = 0; + struct fsm_union_entry entries[MAX_PATTERNS] = {0}; + + for (size_t i = 0; i < env.fsm_count; i++) { + /* there can be gaps, fsms[] lines up with patterns[] */ + if (env.fsms[i] == NULL) { continue; } + + fsm_state_t start; + if (!fsm_getstart(env.fsms[i], &start)) { + assert(!"hit"); + } + + struct fsm *cp = fsm_clone(env.fsms[i]); + assert(cp != NULL); + + if (verbose) { + fprintf(stderr, "==== cp %zd\n", i); + fsm_dump(stderr, cp); + fsm_eager_output_dump(stderr, cp); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(cp); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(cp, i)); + } + } + + entries[used].fsm = cp; + entries[used].anchored_start = anchorage[i].start; + entries[used].anchored_end = anchorage[i].end; + used++; + } + + if (used == 0) { + goto cleanup; /* nothing to do */ + } + + /* consumes entries[] */ + struct fsm *fsm = fsm_union_repeated_pattern_group(used, entries, NULL); + assert(fsm != NULL); + + if (verbose) { + fprintf(stderr, "==== combined (pre-det)\n"); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + if (!fsm_determinise(fsm)) { + assert(!"failed to determinise"); + } + + if (!fsm_minimise(fsm)) { + assert(!"failed to minimise"); + } + + LOG("%s: combined state_count %d\n", __func__, fsm_countstates(fsm)); + env.combined = fsm; + /* fsm_eager_output_set_cb(fsm, append_eager_output_cb, &env.outputs_combined); */ + + if (verbose) { + fprintf(stderr, "==== combined\n"); + fsm_dump(stderr, env.combined); + fsm_eager_output_dump(stderr, env.combined); + fprintf(stderr, "====\n"); + } + + } + + /* Use fsm_generate_matches to check for matches that got lost + * and false positives introduced while combining the DFAs. + * Use the combined DFA to generate matches, check that the + * match behavior agrees with the individual DFA copies. */ + env.current_pattern = (size_t)-1; + if (!fsm_generate_matches(env.combined, max_pattern_length, gen_combined_check_individual_cb, &env)) { + goto cleanup; + } + + if (!env.ok) { goto cleanup; } + + /* Likewise, use every individual DFA to generate matches and */ + /* check behavior against the combined DFA. */ + for (size_t i = 0; i < env.pattern_count; i++) { + env.current_pattern = i; + if (!fsm_generate_matches(env.combined, max_pattern_length, gen_individual_check_combined_cb, &env)) { + goto cleanup; + } + } + + ret = env.ok ? EXIT_SUCCESS : EXIT_FAILURE; +cleanup: + for (size_t i = 0; i < MAX_PATTERNS; i++) { + if (env.patterns[i] != NULL) { + free(env.patterns[i]); + env.patterns[i] = NULL; + } + if (env.fsms[i] != NULL) { + fsm_free(env.fsms[i]); + } + } + if (env.combined != NULL) { + fsm_free(env.combined); + } + + return ret; +} + +static int +cmp_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static bool +match_input_get_eager_outputs(struct fsm *fsm, const char *input, size_t input_length, + struct eager_output_cb_info *dst) +{ + (void)input_length; + fsm_state_t end; + + reset_eager_output_info(dst); + + fsm_eager_output_set_cb(fsm, append_eager_output_cb, dst); + const int ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + if (ret == 0) { + return false; /* no match */ + } else { + assert(ret == 1); /* match */ + } + + /* sort the IDs, to make comparison cheaper */ + qsort(dst->ids, dst->used, sizeof(dst->ids[0]), cmp_output_id); + return true; /* match */ +} + +/* For a given matching input generated by the combined DFA, check that + * only the expected individual source DFAs match. */ +static enum fsm_generate_matches_cb_res +gen_combined_check_individual_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + (void)fsm; + (void)depth; + (void)end_state; + + struct feo_env *env = opaque; + assert(env->current_pattern == (size_t)-1); + + if (match_count > env->max_match_count) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + + /* execute, to set eager outputs */ + if (!match_input_get_eager_outputs(env->combined, input, input_length, &env->outputs_combined)) { + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + size_t individual_outputs_used = 0; + fsm_output_id_t individual_outputs[MAX_PATTERNS]; + + for (size_t i = 0; i < env->pattern_count; i++) { + struct fsm *fsm = env->fsms[i]; + if (fsm == NULL) { continue; } + + if (!match_input_get_eager_outputs(fsm, input, input_length, &env->outputs)) { + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (env->outputs.used > 0) { + assert(env->outputs.used == 1); + individual_outputs[individual_outputs_used++] = env->outputs.ids[0]; + } + } + + bool match = true; + if (env->outputs_combined.used != individual_outputs_used) { + match = false; + } + + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + const fsm_output_id_t cur = env->outputs_combined.ids[cmb_i]; + assert(env->fsms[cmb_i] != NULL); + bool found = false; + for (size_t i = 0; i < individual_outputs_used; i++) { + if (individual_outputs[i] == cur) { + found = true; + break; + } + } + if (!found) { + match = false; + break; + } + } + + if (!match) { + fprintf(stderr, "%s: combined <-> individual mismatch for input '%s'(%zd)!\n", __func__, input, input_length); + + fprintf(stderr, "-- combined: %zu IDs:", env->outputs_combined.used); + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + fprintf(stderr, " %d", env->outputs_combined.ids[cmb_i]); + } + fprintf(stderr, "\n"); + fprintf(stderr, "-- individiual: %zu IDs:", individual_outputs_used); + for (size_t i = 0; i < individual_outputs_used; i++) { + fprintf(stderr, " %d", individual_outputs[i]); + } + fprintf(stderr, "\n"); + goto fail; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; + +fail: + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; +} + +/* For a given matching input generated by one of the source DFAs, check that + * the combined DFA also matches, and that the only other source DFAs that match + * are ones that should according to the combined DFA. */ +static enum fsm_generate_matches_cb_res +gen_individual_check_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + (void)fsm; + (void)depth; + (void)end_state; + + struct feo_env *env = opaque; + assert(env->current_pattern < env->pattern_count); + if (match_count > env->max_match_count) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + + struct fsm *cur_fsm = env->fsms[env->current_pattern]; + if (cur_fsm == NULL) { return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; } + + /* execute, to set eager outputs */ + if (!match_input_get_eager_outputs(cur_fsm, input, input_length, &env->outputs)) { + goto fail; + } + if (!match_input_get_eager_outputs(env->combined, input, input_length, &env->outputs_combined)) { + goto fail; + } + + assert(env->outputs.used == 1); + + bool found = false; + for (size_t i = 0; i < env->outputs_combined.used; i++) { + if (env->outputs_combined.ids[i] == env->outputs.ids[0]) { + found = true; + break; + } + } + + if (!found) { + fprintf(stderr, "%s: combined <-> individual mismatch for input '%s'(%zd)!\n", __func__, input, input_length); + + fprintf(stderr, "-- combined: %zu IDs:", env->outputs_combined.used); + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + fprintf(stderr, " %d", env->outputs_combined.ids[cmb_i]); + } + fprintf(stderr, "\n"); + fprintf(stderr, "-- pattern %zd: %zu IDs:", env->current_pattern, env->outputs.used); + for (size_t i = 0; i < env->outputs.used; i++) { + fprintf(stderr, " %d", env->outputs.ids[i]); + } + fprintf(stderr, "\n"); + goto fail; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; + +fail: + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; +} +#undef MAX_PATTERNS + #define MAX_FUZZER_DATA (64 * 1024) static uint8_t data_buf[MAX_FUZZER_DATA + 1]; @@ -358,6 +871,7 @@ get_run_mode(void) switch (mode[0]) { case 'm': return MODE_SHUFFLE_MINIMISE; case 'p': return MODE_ALL_PRINT_FUNCTIONS; + case 'E': return MODE_EAGER_OUTPUT; case 'd': default: return MODE_DEFAULT; @@ -373,6 +887,11 @@ harness_fuzzer_target(const uint8_t *data, size_t size) return EXIT_SUCCESS; } + if (!verbosity_checked) { + verbosity_checked = true; + verbose = getenv("VERBOSE") != NULL; + } + /* Ensure that input is '\0'-terminated. */ if (size > MAX_FUZZER_DATA) { size = MAX_FUZZER_DATA; @@ -392,6 +911,9 @@ harness_fuzzer_target(const uint8_t *data, size_t size) case MODE_SHUFFLE_MINIMISE: return shuffle_minimise(pattern); + case MODE_EAGER_OUTPUT: + return fuzz_eager_output(data, size); + case MODE_ALL_PRINT_FUNCTIONS: { if (dev_null == NULL) { @@ -403,7 +925,7 @@ harness_fuzzer_target(const uint8_t *data, size_t size) const bool det = b0 & 0x1; const bool min = b0 & 0x2; const enum fsm_io io_mode = (b0 >> 2) % 3; - + const char *shifted_pattern = (const char *)&data_buf[1]; int res = fuzz_all_print_functions(dev_null, shifted_pattern, det, min, io_mode); return res; diff --git a/include/fsm/bool.h b/include/fsm/bool.h index d92518297..4d9f1889a 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -52,6 +52,16 @@ struct fsm * fsm_union_array(size_t fsm_count, struct fsm **fsms, struct fsm_combined_base_pair *bases); +struct fsm_union_entry { + struct fsm *fsm; + bool anchored_start; + bool anchored_end; +}; + +struct fsm * +fsm_union_repeated_pattern_group(size_t entry_count, + struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases); + struct fsm * fsm_intersect(struct fsm *a, struct fsm *b); diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 1dd710d0e..b862cb041 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -7,6 +7,7 @@ #ifndef FSM_H #define FSM_H +#include #include struct fsm; @@ -27,6 +28,9 @@ typedef unsigned int fsm_state_t; * original FSM(s) matched when executing a combined FSM. */ typedef unsigned int fsm_end_id_t; +/* Eager output ID. */ +typedef unsigned int fsm_output_id_t; + #define FSM_END_ID_MAX UINT_MAX /* @@ -266,6 +270,39 @@ fsm_mapendids(struct fsm * fsm, fsm_endid_remap_fun remap, void *opaque); void fsm_increndids(struct fsm * fsm, int delta); +/* Associate an eagerly matched numeric ID with the end states in an fsm. + * + * This is similar to fsm_setendid, but has different performance + * trade-offs. In particular, it can become extremely expensive to + * combine multiple DFAs with endids on their end states when they + * representing regexes with unanchored ends, because the FSM has to + * explicitly represent all the possible combinations of matches by + * copying the entire path to every reachable end state. Eager endids + * are associated with the edge leaving the main pattern match. + * + * Returns 1 on success, 0 on error. + * */ +int +fsm_seteagerendid(struct fsm *fsm, fsm_end_id_t id); + +/* Set an eager output ID to emit every time the state is entered. + * This turns the automata into a Moore machine. */ +int +fsm_seteageroutput(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id); + +/* Set an eager output ID on all current end states. */ +int +fsm_seteageroutputonends(struct fsm *fsm, fsm_output_id_t id); + +/* HACK */ +typedef void +fsm_eager_output_cb(fsm_output_id_t id, void *opaque); +void +fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque); + +void +fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque); + /* * Find the state (if there is just one), or add epsilon edges from all states, * for which the given predicate is true. @@ -451,6 +488,15 @@ fsm_shortest(const struct fsm *fsm, fsm_state_t start, fsm_state_t goal, unsigned (*cost)(fsm_state_t from, fsm_state_t to, char c)); +/* HACK */ +typedef void +fsm_eager_endid_cb(fsm_end_id_t id, void *opaque); +void +fsm_eager_endid_set_cb(struct fsm *fsm, fsm_eager_endid_cb *cb, void *opaque); + +void +fsm_eager_endid_get_cb(const struct fsm *fsm, fsm_eager_endid_cb **cb, void **opaque); + /* * Execute an FSM reading input from the user-specified callback fsm_getc(). * fsm_getc() is passed the opaque pointer given, and is expected to return diff --git a/include/fsm/print.h b/include/fsm/print.h index c9ec7ec0a..a8aa4db54 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -45,6 +45,9 @@ enum fsm_print_lang { struct fsm_state_metadata { const fsm_end_id_t *end_ids; size_t end_id_count; + + const fsm_output_id_t *eager_output_ids; + size_t eager_output_count; }; /* diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 5e2ed57e3..c7782f0ff 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -8,6 +8,7 @@ SRC += src/libfsm/consolidate.c SRC += src/libfsm/clone.c SRC += src/libfsm/closure.c SRC += src/libfsm/detect_required.c +SRC += src/libfsm/eager_output.c SRC += src/libfsm/edge.c SRC += src/libfsm/empty.c SRC += src/libfsm/end.c diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 9fd236a4d..2161599ae 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -19,6 +19,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define LOG_CLONE_ENDIDS 0 @@ -28,6 +29,9 @@ copy_capture_actions(struct fsm *dst, const struct fsm *src); static int copy_end_ids(struct fsm *dst, const struct fsm *src); +static int +copy_eager_output_ids(struct fsm *dst, const struct fsm *src); + struct fsm * fsm_clone(const struct fsm *fsm) { @@ -80,6 +84,12 @@ fsm_clone(const struct fsm *fsm) fsm_free(new); return NULL; } + + /* does not copy callback */ + if (!copy_eager_output_ids(new, fsm)) { + fsm_free(new); + return NULL; + } } return new; @@ -159,3 +169,31 @@ copy_end_ids(struct fsm *dst, const struct fsm *src) return env.ok; } + +struct copy_eager_output_ids_env { + bool ok; + struct fsm *dst; +}; + +static int +copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct copy_eager_output_ids_env *env = opaque; + if (!fsm_seteageroutput(env->dst, state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +copy_eager_output_ids(struct fsm *dst, const struct fsm *src) +{ + struct copy_eager_output_ids_env env; + env.dst = dst; + env.ok = true; + + fsm_eager_output_iter_all(src, copy_eager_output_ids_cb, &env); + return env.ok; +} diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 236a4f6f5..b7a8905b2 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -25,6 +25,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define LOG_MAPPING 0 #define LOG_CONSOLIDATE_CAPTURES 0 @@ -53,6 +54,10 @@ static int consolidate_end_ids(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count); +static int +consolidate_eager_output_ids(struct fsm *dst, const struct fsm *src, + const fsm_state_t *mapping, size_t mapping_count); + static fsm_state_t mapping_cb(fsm_state_t id, const void *opaque) { @@ -154,6 +159,10 @@ fsm_consolidate(const struct fsm *src, } } + if (!consolidate_eager_output_ids(dst, src, mapping, mapping_count)) { + goto cleanup; + } + f_free(src->alloc, seen); return dst; @@ -270,3 +279,40 @@ consolidate_end_ids(struct fsm *dst, const struct fsm *src, return ret; } + +struct consolidate_eager_output_ids_env { + bool ok; + struct fsm *dst; + const fsm_state_t *mapping; + size_t mapping_count; +}; + +static int +consolidate_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct consolidate_eager_output_ids_env *env = opaque; + assert(state < env->mapping_count); + const fsm_state_t dst_state = env->mapping[state]; + + if (!fsm_seteageroutput(env->dst, dst_state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +consolidate_eager_output_ids(struct fsm *dst, const struct fsm *src, + const fsm_state_t *mapping, size_t mapping_count) +{ + struct consolidate_eager_output_ids_env env = { + .ok = true, + .dst = dst, + .mapping = mapping, + .mapping_count = mapping_count, + }; + fsm_eager_output_iter_all(src, consolidate_eager_output_ids_cb, &env); + return env.ok; +} + diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index d5ba396a4..9833fd878 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -6,6 +6,9 @@ #include "determinise_internal.h" +#include +#include + static void dump_labels(FILE *f, const uint64_t labels[4]) { @@ -266,6 +269,10 @@ fsm_determinise_with_config(struct fsm *nfa, goto cleanup; } + if (!remap_eager_outputs(&map, issp, dfa, nfa)) { + goto cleanup; + } + fsm_move(nfa, dfa); } @@ -363,6 +370,22 @@ add_reverse_mapping(const struct fsm_alloc *alloc, return 1; } +static void +free_reverse_mappings(const struct fsm_alloc *alloc, size_t map_count, struct reverse_mapping *rmaps) +{ + if (rmaps == NULL) { return; } + + for (size_t map_i = 0; map_i < map_count; map_i++) { + struct reverse_mapping *rmap = &rmaps[map_i]; + for (size_t i = 0; i < rmap->count; i++) { + f_free(alloc, rmap[i].list); + rmap->count = 0; + rmap[i].list = NULL; + } + } + f_free(alloc, rmaps); +} + static int det_copy_capture_actions_cb(fsm_state_t state, enum capture_action_type type, unsigned capture_id, fsm_state_t to, @@ -434,7 +457,7 @@ hash_iss(interned_state_set_id iss) } static struct mapping * -map_first(struct map *map, struct map_iter *iter) +map_first(const struct map *map, struct map_iter *iter) { iter->m = map; iter->i = 0; @@ -672,22 +695,14 @@ stack_pop(struct mappingstack *stack) return item; } -static int -remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, - struct fsm *dst_dfa, struct fsm *src_nfa) +static struct reverse_mapping * +build_reverse_mappings(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa) { + struct reverse_mapping *reverse_mappings = NULL; struct map_iter it; struct state_iter si; struct mapping *m; - struct reverse_mapping *reverse_mappings; - fsm_state_t state; - const size_t capture_count = fsm_countcaptures(src_nfa); - size_t i, j; - int res = 0; - - if (capture_count == 0) { - return 1; - } /* This is not 1 to 1 -- if state X is now represented by multiple * states Y in the DFA, and state X has action(s) when transitioning @@ -698,9 +713,7 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, * checking reachability from every X, but the actual path * handling later will also check reachability. */ reverse_mappings = f_calloc(dst_dfa->alloc, src_nfa->statecount, sizeof(reverse_mappings[0])); - if (reverse_mappings == NULL) { - return 0; - } + if (reverse_mappings == NULL) { goto cleanup; } /* build reverse mappings table: for every NFA state X, if X is part * of the new DFA state Y, then add Y to a list for X */ @@ -710,6 +723,7 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, assert(m->dfastate < dst_dfa->statecount); ss = interned_state_set_get_state_set(issp, iss_id); + fsm_state_t state; for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { if (!add_reverse_mapping(dst_dfa->alloc, reverse_mappings, @@ -719,33 +733,47 @@ remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, } } -#if LOG_DETERMINISE_CAPTURES +#if LOG_BUILD_REVERSE_MAPPING fprintf(stderr, "#### reverse mapping for %zu states\n", src_nfa->statecount); - for (i = 0; i < src_nfa->statecount; i++) { + for (size_t i = 0; i < src_nfa->statecount; i++) { struct reverse_mapping *rm = &reverse_mappings[i]; fprintf(stderr, "%lu:", i); - for (j = 0; j < rm->count; j++) { + for (size_t j = 0; j < rm->count; j++) { fprintf(stderr, " %u", rm->list[j]); } fprintf(stderr, "\n"); } -#else - (void)j; #endif + return reverse_mappings; + +cleanup: + free_reverse_mappings(dst_dfa->alloc, src_nfa->statecount, reverse_mappings); + return NULL; +} + +static int +remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, struct fsm *src_nfa) +{ + const size_t capture_count = fsm_countcaptures(src_nfa); + int res = 0; + + if (capture_count == 0) { + return 1; + } + + struct reverse_mapping *reverse_mappings = build_reverse_mappings(map, issp, dst_dfa, src_nfa); + if (reverse_mappings == NULL) { goto cleanup; } + if (!det_copy_capture_actions(reverse_mappings, dst_dfa, src_nfa)) { goto cleanup; } res = 1; -cleanup: - for (i = 0; i < src_nfa->statecount; i++) { - if (reverse_mappings[i].list != NULL) { - f_free(dst_dfa->alloc, reverse_mappings[i].list); - } - } - f_free(dst_dfa->alloc, reverse_mappings); +cleanup: + free_reverse_mappings(dst_dfa->alloc, src_nfa->statecount, reverse_mappings); return res; } @@ -2559,3 +2587,50 @@ analyze_closures__grow_outputs(struct analyze_closures_env *env) env->output_ceil = nceil; return 1; } + +struct remap_eager_output_env { + bool ok; + struct fsm *dst; + fsm_state_t dst_state; +}; + +static int +remap_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct remap_eager_output_env *env = opaque; + if (!fsm_seteageroutput(env->dst, env->dst_state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +remap_eager_outputs(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa) +{ + /* For each DFA state, get the set of NFA states corresponding to it from the + * map and issp, then copy every eager output ID over. */ + struct map_iter iter; + for (struct mapping *b = map_first(map, &iter); b != NULL; b = map_next(&iter)) { + struct state_set *ss = interned_state_set_get_state_set(issp, b->iss); + assert(ss != NULL); + + struct state_iter it; + fsm_state_t s; + state_set_reset(ss, &it); + while (state_set_next(&it, &s)) { + struct remap_eager_output_env env = { + .ok = true, + .dst = dst_dfa, + .dst_state = b->dfastate, + }; + fsm_eager_output_iter_state(src_nfa, s, remap_eager_output_cb, &env); + if (!env.ok) { return 0; } + } + } + + return 1; +} diff --git a/src/libfsm/determinise_internal.h b/src/libfsm/determinise_internal.h index cfd4ea663..2e925d28c 100644 --- a/src/libfsm/determinise_internal.h +++ b/src/libfsm/determinise_internal.h @@ -23,6 +23,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #include @@ -35,6 +36,7 @@ #define LOG_AC 0 #define LOG_GROUPING 0 #define LOG_ANALYSIS_STATS 0 +#define LOG_BUILD_REVERSE_MAPPING 0 #if LOG_DETERMINISE_CAPTURES || LOG_INPUT #include @@ -72,7 +74,7 @@ struct map { }; struct map_iter { - struct map *m; + const struct map *m; size_t i; }; @@ -304,7 +306,7 @@ static void map_free(struct map *map); static struct mapping * -map_first(struct map *map, struct map_iter *iter); +map_first(const struct map *map, struct map_iter *iter); static struct mapping * map_next(struct map_iter *iter); @@ -325,6 +327,10 @@ static int remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, struct fsm *dst_dfa, struct fsm *src_nfa); +static int +remap_eager_outputs(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa); + static struct mappingstack * stack_init(const struct fsm_alloc *alloc); diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c new file mode 100644 index 000000000..e37a8a4bf --- /dev/null +++ b/src/libfsm/eager_output.c @@ -0,0 +1,403 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include + +#include "internal.h" + +#include +#include + +#include +#include +#include + +#include "eager_output.h" + +#define LOG_LEVEL 0 + +/* must be a power of 2 */ +#define DEF_BUCKET_COUNT 4 +#define DEF_ENTRY_CEIL 2 + +struct eager_output_info { + fsm_eager_output_cb *cb; + void *opaque; + + struct eager_output_htab { + size_t bucket_count; + size_t buckets_used; + /* empty if entry is NULL, otherwise keyed by state */ + struct eager_output_bucket { + fsm_state_t state; + struct eager_output_entry { + unsigned used; + unsigned ceil; + fsm_end_id_t ids[]; + } *entry; + } *buckets; + } htab; +}; + +void +fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque) +{ +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_set_cb %p\n", (void *)fsm); +#endif + assert(fsm != NULL); + assert(fsm->eager_output_info != NULL); + fsm->eager_output_info->cb = cb; + fsm->eager_output_info->opaque = opaque; +} + +void +fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque) +{ + *cb = fsm->eager_output_info->cb; + *opaque = fsm->eager_output_info->opaque; +} + +int +fsm_eager_output_init(struct fsm *fsm) +{ + struct eager_output_info *ei = f_calloc(fsm->alloc, 1, sizeof(*ei)); + + if (ei == NULL) { return 0; } + + struct eager_output_bucket *buckets = f_calloc(fsm->alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + f_free(fsm->alloc, ei); + return 0; + } + +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_init %p\n", (void *)fsm); +#endif + + ei->htab.buckets = buckets; + ei->htab.bucket_count = DEF_BUCKET_COUNT; + + fsm->eager_output_info = ei; + return 1; +} + +void +fsm_eager_output_free(struct fsm *fsm) +{ + if (fsm == NULL || fsm->eager_output_info == NULL) { return; } + + for (size_t i = 0; i < fsm->eager_output_info->htab.bucket_count; i++) { + struct eager_output_bucket *b = &fsm->eager_output_info->htab.buckets[i]; + if (b->entry == NULL) { continue; } + f_free(fsm->alloc, b->entry); + } + f_free(fsm->alloc, fsm->eager_output_info->htab.buckets); + + f_free(fsm->alloc, fsm->eager_output_info); +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_free %p\n", (void *)fsm); +#endif + fsm->eager_output_info = NULL; +} + +int +fsm_seteageroutputonends(struct fsm *fsm, fsm_output_id_t id) +{ + assert(fsm != NULL); + const size_t count = fsm_countstates(fsm); + for (size_t i = 0; i < count; i++) { + if (fsm_isend(fsm, i)) { + if (!fsm_seteageroutput(fsm, i, id)) { return 0; } + } + } + return 1; +} + +static bool +grow_htab(const struct fsm_alloc *alloc, struct eager_output_htab *htab) +{ + const size_t nbucket_count = 2*htab->bucket_count; + assert(nbucket_count != 0); + + struct eager_output_bucket *nbuckets = f_calloc(alloc, nbucket_count, + sizeof(nbuckets[0])); + if (nbuckets == NULL) { return false; } + + const uint64_t nmask = nbucket_count - 1; + assert((nmask & nbucket_count) == 0); /* power of 2 */ + + for (size_t ob_i = 0; ob_i < htab->bucket_count; ob_i++) { + struct eager_output_bucket *ob = &htab->buckets[ob_i]; + if (ob->entry == NULL) { continue; } + + const uint64_t hash = hash_id(ob->state); + for (size_t probes = 0; probes < nbucket_count; probes++) { + const size_t nb_i = (hash + probes) & nmask; + struct eager_output_bucket *nb = &nbuckets[nb_i]; + if (nb->entry == NULL) { + nb->state = ob->state; + nb->entry = ob->entry; + break; + } else { + assert(nb->state != ob->state); + } + } + } + + f_free(alloc, htab->buckets); + htab->bucket_count = nbucket_count; + htab->buckets = nbuckets; + return true; +} + +int +fsm_seteageroutput(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) +{ + assert(fsm != NULL); + + struct eager_output_info *info = fsm->eager_output_info; + assert(info->htab.bucket_count > 0); + + if (info->htab.buckets_used >= info->htab.bucket_count/2) { + if (!grow_htab(fsm->alloc, &info->htab)) { return 0; } + } + + const uint64_t hash = hash_id(state); + const uint64_t mask = info->htab.bucket_count - 1; + assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ + + /* fprintf(stderr, "%s: bucket_count %zd\n", __func__, info->htab.bucket_count); */ + for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { + const size_t b_i = (hash + probes) & mask; + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + /* fprintf(stderr, "%s: state %d -> b_i %zd, state %d, entry %p\n", */ + /* __func__, state, b_i, b->state, (void *)b->entry); */ + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty */ + /* add */ + const size_t alloc_sz = sizeof(*e) + + DEF_ENTRY_CEIL * sizeof(e->ids[0]); + e = f_calloc(fsm->alloc, 1, alloc_sz); + if (e == NULL) { + return 0; + } + e->ceil = DEF_ENTRY_CEIL; + b->state = state; + b->entry = e; + info->htab.buckets_used++; + /* fprintf(stderr, "%s: buckets_used %zd\n", __func__, info->htab.buckets_used); */ + /* fprintf(stderr, "%s: saved new entry in bucket %zd\n", __func__, b_i); */ + } else if (b->state != state) { /* collision */ + continue; + } + + if (e->used == e->ceil) { + const size_t nceil = 2 * e->ceil; + const size_t nsize = sizeof(*e) + + nceil * sizeof(e->ids[0]); + struct eager_output_entry *nentry = f_realloc(fsm->alloc, e, nsize); + if (nentry == NULL) { return 0; } + nentry->ceil = nceil; + b->entry = nentry; + e = b->entry; + } + + /* ignore duplicates */ + for (size_t i = 0; i < e->used; i++) { + if (e->ids[i] == id) { return 1; } + } + + e->ids[e->used++] = id; + /* fprintf(stderr, "%s: e->ids_used %u\n", __func__, e->used); */ + fsm->states[state].has_eager_outputs = 1; + return 1; + } + + return 1; +} + +bool +fsm_eager_output_has_eager_output(const struct fsm *fsm) +{ + assert(fsm->eager_output_info != NULL); + const struct eager_output_htab *htab = &fsm->eager_output_info->htab; + + for (size_t b_i = 0; b_i < htab->bucket_count; b_i++) { + struct eager_output_bucket *b = &htab->buckets[b_i]; + if (b->entry == NULL) { continue; } + if (b->entry->used > 0) { return 1; } + } + return 0; +} + +bool +fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state) +{ + assert(state < fsm->statecount); + return fsm->states[state].has_eager_outputs; +} + +void +fsm_eager_output_iter_state(const struct fsm *fsm, + fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque) +{ + assert(fsm != NULL); + assert(cb != NULL); + + const uint64_t hash = hash_id(state); + + struct eager_output_info *info = fsm->eager_output_info; + const uint64_t mask = info->htab.bucket_count - 1; + assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ + + for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { + const size_t b_i = (hash + probes) & mask; + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + /* fprintf(stderr, "%s: state %d -> b_i %zd, state %d, entry %p\n", */ + /* __func__, state, b_i, b->state, (void *)b->entry); */ + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty */ + return; + } else if (b->state != state) { /* collision */ + continue; + } + + assert(e->used == 0 || fsm->states[state].has_eager_outputs); + + for (size_t i = 0; i < e->used; i++) { + if (!cb(state, e->ids[i], opaque)) { return; } + } + } +} + +void +fsm_eager_output_iter_all(const struct fsm *fsm, + fsm_eager_output_iter_cb *cb, void *opaque) +{ + assert(fsm != NULL); + assert(cb != NULL); + assert(fsm->eager_output_info != NULL); + + struct eager_output_info *info = fsm->eager_output_info; + + /* fprintf(stderr, "%s: bucket_count %zd\n", __func__, info->htab.bucket_count); */ + for (size_t b_i = 0; b_i < info->htab.bucket_count; b_i++) { + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + struct eager_output_entry *e = b->entry; + /* fprintf(stderr, "%s: b_i %zd, state %d, entry %p\n", */ + /* __func__, b_i, b->state, (void *)b->entry); */ + if (e == NULL) { /* empty */ + continue; + } + assert(e->used == 0 || fsm->states[b->state].has_eager_outputs); + + for (size_t i = 0; i < e->used; i++) { + if (!cb(b->state, e->ids[i], opaque)) { return; } + } + } +} + +struct dump_env { + FILE *f; + size_t count; +}; + +static int +dump_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) + +{ + struct dump_env *env = opaque; + fprintf(env->f, "-- %d: id %d\n", state, id); + env->count++; + return 1; +} + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm) +{ + struct dump_env env = { .f = f }; + fprintf(f, "%s:\n", __func__); + fsm_eager_output_iter_all(fsm, dump_cb, (void *)&env); + fprintf(f, "== %zu total\n", env.count); +} + +static int +inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + (void)id; + size_t *count = opaque; + (*count)++; + return 1; +} + +bool +fsm_eager_output_has_any(const struct fsm *fsm, + fsm_state_t state, size_t *count) +{ + size_t c = 0; + fsm_eager_output_iter_state(fsm, state, &inc_cb, &c); + if (count != NULL) { *count = c; } + return c > 0; +} + +int +fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count) +{ + /* Don't reallocate unless something has actually changed. */ + bool changes = false; + for (size_t i = 0; i < mapping_count; i++) { + if (mapping[i] != i) { + changes = true; + break; + } + } + + /* nothing to do */ + if (!changes) { return 1; } + + struct eager_output_info *eoi = fsm->eager_output_info; + + struct eager_output_bucket *nbuckets = f_calloc(fsm->alloc, + eoi->htab.bucket_count, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + + const uint64_t mask = eoi->htab.bucket_count - 1; + assert((eoi->htab.bucket_count & mask) == 0); + + for (size_t ob_i = 0; ob_i < eoi->htab.bucket_count; ob_i++) { + const struct eager_output_bucket *ob = &eoi->htab.buckets[ob_i]; + if (ob->entry == NULL) { continue; } + + assert(ob->state < mapping_count); + const fsm_state_t nstate = mapping[ob->state]; + if (nstate == FSM_STATE_REMAP_NO_STATE) { continue; } + + const uint64_t hash = hash_id(nstate); + + bool placed = false; + for (size_t probes = 0; probes < eoi->htab.bucket_count; probes++) { + const size_t nb_i = (hash + probes) & mask; + struct eager_output_bucket *nb = &nbuckets[nb_i]; + if (nb->entry == NULL) { + nb->state = nstate; + nb->entry = ob->entry; + placed = true; + break; + } + } + assert(placed); + } + + f_free(fsm->alloc, eoi->htab.buckets); + eoi->htab.buckets = nbuckets; + return 1; +} diff --git a/src/libfsm/eager_output.h b/src/libfsm/eager_output.h new file mode 100644 index 000000000..1b48ba4c4 --- /dev/null +++ b/src/libfsm/eager_output.h @@ -0,0 +1,46 @@ +#ifndef EAGER_OUTPUT_H +#define EAGER_OUTPUT_H + +#include +#include +#include + +struct eager_output_info; + +int +fsm_eager_output_init(struct fsm *fsm); + +void +fsm_eager_output_free(struct fsm *fsm); + +bool +fsm_eager_output_has_eager_output(const struct fsm *fsm); + +bool +fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state); + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +/* Callback for fsm_eager_output_iter_*. + * The return value indicates whether iteration should continue. + * The results may not be sorted in any particular order. */ +typedef int +fsm_eager_output_iter_cb(fsm_state_t state, fsm_output_id_t id, void *opaque); + +void +fsm_eager_output_iter_state(const struct fsm *fsm, + fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque); + +void +fsm_eager_output_iter_all(const struct fsm *fsm, + fsm_eager_output_iter_cb *cb, void *opaque); + +bool +fsm_eager_output_has_any(const struct fsm *fsm, + fsm_state_t state, size_t *count); + +int +fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); + +#endif diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 9394a2d9b..adfcdec2a 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -9,24 +9,42 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define DUMP_EPSILON_CLOSURES 0 #define DEF_PENDING_CAPTURE_ACTIONS_CEIL 2 #define LOG_RM_EPSILONS_CAPTURES 0 #define DEF_CARRY_ENDIDS_COUNT 2 +#define LOG_LEVEL 0 + +#if LOG_LEVEL > 0 +static bool log_it; +#define LOG(LVL, ...) \ + do { \ + if (log_it && LVL <= LOG_LEVEL) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while (0) +#else +#define LOG(_LVL, ...) +#endif + struct remap_env { #ifndef NDEBUG char tag; @@ -57,6 +75,49 @@ static int carry_endids(struct fsm *fsm, struct state_set *states, fsm_state_t s); +static void +mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label); + +struct eager_output_buf { +#define DEF_EAGER_OUTPUT_BUF_CEIL 8 + bool ok; + const struct fsm_alloc *alloc; + size_t ceil; + size_t used; + fsm_output_id_t *ids; +}; + +static bool +append_eager_output_id(struct eager_output_buf *buf, fsm_output_id_t id) +{ + if (buf->used == buf->ceil) { + const size_t nceil = buf->ceil == 0 ? DEF_EAGER_OUTPUT_BUF_CEIL : 2*buf->ceil; + fsm_output_id_t *nids = f_realloc(buf->alloc, buf->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + buf->ok = false; + return false; + } + buf->ids = nids; + buf->ceil = nceil; + } + + for (size_t i = 0; i < buf->used; i++) { + /* avoid duplicates */ + if (buf->ids[i] == id) { return true; } + } + + buf->ids[buf->used++] = id; + return true; +} + +static int +collect_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct eager_output_buf *buf = opaque; + return append_eager_output_id(buf, id) ? 1 : 0; +} + int fsm_remove_epsilons(struct fsm *nfa) { @@ -64,9 +125,20 @@ fsm_remove_epsilons(struct fsm *nfa) int res = 0; struct state_set **eclosures = NULL; fsm_state_t s; + struct eager_output_buf eager_output_buf = { + .ok = true, + .alloc = nfa->alloc, + }; + uint64_t *reachable_by_label = NULL; + + LOG(2, "%s: starting\n", __func__); INIT_TIMERS(); +#if LOG_LEVEL > 0 + log_it = getenv("LOG") != NULL; +#endif + assert(nfa != NULL); TIME(&pre); @@ -94,6 +166,17 @@ fsm_remove_epsilons(struct fsm *nfa) } #endif + const size_t state_words = u64bitset_words(state_count); + reachable_by_label = f_calloc(nfa->alloc, state_words, sizeof(reachable_by_label[0])); + if (reachable_by_label == NULL) { goto cleanup; } + + mark_states_reachable_by_label(nfa, reachable_by_label); + + fsm_state_t start; + if (!fsm_getstart(nfa, &start)) { + goto cleanup; /* no start state */ + } + for (s = 0; s < state_count; s++) { struct state_iter si; fsm_state_t es_id; @@ -101,6 +184,12 @@ fsm_remove_epsilons(struct fsm *nfa) struct edge_group_iter egi; struct edge_group_iter_info info; + /* If the state isn't reachable by a label and isn't the start state, + * skip processing -- it will soon become garbage. */ + if (!u64bitset_get(reachable_by_label, s) && s != start) { + continue; + } + /* Process the epsilon closure. */ state_set_reset(eclosures[s], &si); while (state_set_next(&si, &es_id)) { @@ -129,6 +218,16 @@ fsm_remove_epsilons(struct fsm *nfa) } } + /* Collect every eager output ID from any state + * in the current state's epsilon closure to the + * current state. These will be added at the end. */ + { + if (fsm_eager_output_has_any(nfa, es_id, NULL)) { + fsm_eager_output_iter_state(nfa, es_id, collect_eager_output_ids_cb, &eager_output_buf); + if (!eager_output_buf.ok) { goto cleanup; } + } + } + /* For every state in this state's transitive * epsilon closure, add all of their sets of * labeled edges. */ @@ -144,6 +243,13 @@ fsm_remove_epsilons(struct fsm *nfa) } } } + + for (size_t i = 0; i < eager_output_buf.used; i++) { + if (!fsm_seteageroutput(nfa, s, eager_output_buf.ids[i])) { + goto cleanup; + } + } + eager_output_buf.used = 0; /* clear */ } /* Remove the epsilon-edge state sets from everything. @@ -170,13 +276,53 @@ fsm_remove_epsilons(struct fsm *nfa) res = 1; cleanup: + LOG(2, "%s: finishing\n", __func__); if (eclosures != NULL) { closure_free(nfa, eclosures, state_count); } + f_free(nfa->alloc, reachable_by_label); + f_free(nfa->alloc, eager_output_buf.ids); return res; } +/* For every state, mark every state reached by a labeled edge as + * reachable. This doesn't check that the FROM state is reachable from + * the start state (trim will do that soon enough), it's just used to + * check which states will become unreachable once epsilon edges are + * removed. We don't need to add eager endids for them, because they + * will soon be disconnected from the epsilon-free NFA. */ +static void +mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label) +{ + fsm_state_t start; + if (!fsm_getstart(nfa, &start)) { + return; /* nothing reachable */ + } + u64bitset_set(reachable_by_label, start); + + const fsm_state_t state_count = fsm_countstates(nfa); + + for (size_t s_i = 0; s_i < state_count; s_i++) { + struct edge_group_iter egi; + struct edge_group_iter_info info; + + struct fsm_state *s = &nfa->states[s_i]; + + /* Clear the visited flag, it will be used to avoid cycles. */ +#if 1 + assert(s->visited == 0); /* stale */ +#endif + s->visited = 0; + + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + LOG(1, "%s: reachable: %d\n", __func__, info.to); + u64bitset_set(reachable_by_label, info.to); + } + } +} + static int remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) { @@ -425,4 +571,3 @@ carry_endids(struct fsm *fsm, struct state_set *states, return env.ok; } - diff --git a/src/libfsm/exec.c b/src/libfsm/exec.c index 9f7b21802..077494b8f 100644 --- a/src/libfsm/exec.c +++ b/src/libfsm/exec.c @@ -20,9 +20,12 @@ #include "internal.h" #include "capture.h" +#include "eager_output.h" #define LOG_EXEC 0 +#define LOG_EAGER 0 + static int transition(const struct fsm *fsm, fsm_state_t state, int c, size_t offset, struct fsm_capture *captures, @@ -43,6 +46,44 @@ transition(const struct fsm *fsm, fsm_state_t state, int c, return 1; } +struct check_eager_outputs_for_state_env { + const struct fsm *fsm; + fsm_eager_output_cb *cb; + void *opaque; +}; + +static int +match_eager_outputs_for_state_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +{ + /* HACK update the types here once it's working */ + (void)state; + struct check_eager_outputs_for_state_env *env = opaque; +#if LOG_EAGER + fprintf(stderr, "%s: state %d, id %d\n", __func__, state, id); +#endif + env->cb(id, env->opaque); + return 1; +} + +static int +match_eager_outputs_for_state(const struct fsm *fsm, fsm_state_t state) +{ + /* HACK update the types here once it's working */ + fsm_eager_output_cb *cb = NULL; + void *opaque = NULL; + fsm_eager_output_get_cb(fsm, &cb, &opaque); + if (cb == NULL) { return 1; } /* nothing to do */ + + struct check_eager_outputs_for_state_env env = { + .fsm = fsm, + .cb = cb, + .opaque = opaque, + }; + fsm_eager_output_iter_state(fsm, + state, match_eager_outputs_for_state_cb, &env); + return 1; +} + int fsm_exec(const struct fsm *fsm, int (*fsm_getc)(void *opaque), void *opaque, @@ -73,6 +114,7 @@ fsm_exec(const struct fsm *fsm, errno = EINVAL; return -1; } + const fsm_state_t start = state; for (i = 0; i < capture_count; i++) { captures[i].pos[0] = FSM_CAPTURE_NO_POS; @@ -83,6 +125,12 @@ fsm_exec(const struct fsm *fsm, fprintf(stderr, "fsm_exec: starting at %d\n", state); #endif + if (fsm->states[start].has_eager_outputs) { + if (!match_eager_outputs_for_state(fsm, start)) { + return 0; + } + } + while (c = fsm_getc(opaque), c != EOF) { if (!transition(fsm, state, c, offset, captures, &state)) { #if LOG_EXEC @@ -91,6 +139,12 @@ fsm_exec(const struct fsm *fsm, return 0; } + if (fsm->states[state].has_eager_outputs) { + if (!match_eager_outputs_for_state(fsm, state)) { + return 0; + } + } + #if LOG_EXEC fprintf(stderr, "fsm_exec: @ %zu, input '%c', new state %u\n", offset, c, state); @@ -113,4 +167,3 @@ fsm_exec(const struct fsm *fsm, *end = state; return 1; } - diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index ba2d2db26..c442c8262 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -21,6 +21,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" /* guess for default state allocation */ #define FSM_DEFAULT_STATEALLOC 128 @@ -39,6 +40,7 @@ free_contents(struct fsm *fsm) fsm_capture_free(fsm); fsm_endid_free(fsm); + fsm_eager_output_free(fsm); f_free(fsm->alloc, fsm->states); } @@ -92,6 +94,14 @@ fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc) return NULL; } + if (!fsm_eager_output_init(new)) { + f_free(new->alloc, new->states); + f_free(new->alloc, new); + fsm_capture_free(new); + fsm_endid_free(new); + return NULL; + } + return new; } @@ -133,6 +143,7 @@ fsm_move(struct fsm *dst, struct fsm *src) dst->capture_info = src->capture_info; dst->endid_info = src->endid_info; + dst->eager_output_info = src->eager_output_info; f_free(src->alloc, src); } diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index f84bbef0f..46997c82a 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -60,6 +60,10 @@ struct fsm_state { /* meaningful within one particular transformation only */ unsigned int visited:1; + + /* If 0, then this state has no need for checking + * the fsm->eager_output_info struct. */ + unsigned int has_eager_outputs:1; }; struct fsm { @@ -75,6 +79,7 @@ struct fsm { struct fsm_capture_info *capture_info; struct endid_info *endid_info; + struct eager_output_info *eager_output_info; }; struct fsm * diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 67497d00a..f645c4ceb 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -2,6 +2,7 @@ fsm_complement fsm_union fsm_union_array +fsm_union_repeated_pattern_group fsm_intersect fsm_intersect_charset @@ -72,6 +73,8 @@ fsm_removestate fsm_shuffle fsm_vacuum +fsm_new_statealloc + fsm_addedge_any fsm_addedge_epsilon fsm_addedge_literal @@ -95,6 +98,14 @@ fsm_setendid fsm_mapendids fsm_increndids +fsm_endid_dump + +fsm_seteageroutput +fsm_seteageroutputonends +# short term hack +fsm_eager_output_set_cb +fsm_eager_output_dump + fsm_countedges fsm_countstates diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index 8c972c145..ccc1568ff 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -22,6 +22,7 @@ #include "capture.h" #include "internal.h" #include "endids.h" +#include "eager_output.h" #define LOG_MERGE_ENDIDS 0 @@ -39,6 +40,9 @@ copy_capture_actions(struct fsm *dst, struct fsm *src); static int copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); +static int +copy_eager_output_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); + static struct fsm * merge(struct fsm *dst, struct fsm *src, fsm_state_t *base_dst, fsm_state_t *base_src, @@ -113,6 +117,11 @@ merge(struct fsm *dst, struct fsm *src, return NULL; } + if (!copy_eager_output_ids(dst, src, *base_src)) { + /* non-recoverable -- destructive operation */ + return NULL; + } + f_free(src->alloc, src->states); src->states = NULL; src->statealloc = 0; @@ -194,6 +203,39 @@ copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) return fsm_endid_iter_bulk(src, copy_end_ids_cb, &env); } +struct copy_eager_output_ids_env { + bool ok; + struct fsm *dst; + struct fsm *src; + fsm_state_t base_src; +}; + +static int +copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct copy_eager_output_ids_env *env = opaque; + if (!fsm_seteageroutput(env->dst, state + env->base_src, id)) { + env->ok = false; + return 0; + } + + return 1; + +} + +static int +copy_eager_output_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) +{ + struct copy_eager_output_ids_env env = { + .ok = true, + .dst = dst, + .src = src, + .base_src = base_src, + }; + fsm_eager_output_iter_all(src, copy_eager_output_ids_cb, &env); + return env.ok; +} + struct fsm * fsm_mergeab(struct fsm *a, struct fsm *b, fsm_state_t *base_b) diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index a8d53c57e..86f00b46f 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -25,6 +25,8 @@ #include "internal.h" #include "capture.h" +#include "eager_output.h" +#include "endids.h" #define LOG_MAPPINGS 0 #define LOG_STEPS 0 @@ -54,12 +56,21 @@ struct end_metadata { unsigned count; fsm_end_id_t *ids; } end; + + struct end_metadata_eager_outputs { + unsigned count; + fsm_output_id_t *ids; + } eager_outputs; }; static int collect_end_ids(const struct fsm *fsm, fsm_state_t s, struct end_metadata_end *e); +static int +collect_eager_output_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_eager_outputs *e); + int fsm_minimise(struct fsm *fsm) { @@ -122,6 +133,10 @@ fsm_minimise(struct fsm *fsm) /* Minimisation should never add states. */ assert(minimised_states <= orig_states); + for (size_t i = 0; i < fsm->statecount; i++) { + assert(mapping[i] < fsm->statecount); + } + /* Use the mapping to consolidate the current states * into a new DFA, combining states that could not be * proven distinguishable. */ @@ -693,6 +708,9 @@ same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) if (a->end.count != b->end.count) { return 0; } + if (a->eager_outputs.count != b->eager_outputs.count) { + return 0; + } /* compare -- these must be sorted */ @@ -702,6 +720,12 @@ same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) } } + for (size_t i = 0; i < a->eager_outputs.count; i++) { + if (a->eager_outputs.ids[i] != b->eager_outputs.ids[i]) { + return 0; + } + } + return 1; } @@ -750,14 +774,21 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) #endif while (s != NO_ID) { struct end_metadata *e = &end_md[s]; - if (!fsm_isend(fsm, s)) { - break; /* this EC has non-end states, skip */ + const bool is_end = fsm_isend(fsm, s); + const bool has_eager_outputs = fsm_eager_output_state_has_eager_output(fsm, s); + + if (!is_end && !has_eager_outputs) { + break; /* skip */ } if (!collect_end_ids(fsm, s, &e->end)) { goto cleanup; } + if (!collect_eager_output_ids(fsm, s, &e->eager_outputs)) { + goto cleanup; + } + s = env->jump[s]; } } @@ -789,6 +820,10 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) incremental_hash_of_ids(&hash, s_md->end.ids[eid_i]); } + for (size_t eo_i = 0; eo_i < s_md->eager_outputs.count; eo_i++) { + incremental_hash_of_ids(&hash, s_md->eager_outputs.ids[eo_i]); + } + for (size_t b_i = 0; b_i < bucket_count; b_i++) { fsm_state_t *b = &htab[(b_i + hash) & mask]; const fsm_state_t other = *b; @@ -932,6 +967,9 @@ split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) if (e->end.ids != NULL) { f_free(fsm->alloc, e->end.ids); } + if (e->eager_outputs.ids != NULL) { + f_free(fsm->alloc, e->eager_outputs.ids); + } } f_free(fsm->alloc, end_md); } @@ -959,7 +997,7 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, #if LOG_ECS fprintf(stderr, "%d:", s); - for (size_t i = 0; i < written; i++) { + for (size_t i = 0; i < e->count; i++) { fprintf(stderr, " %u", e->ids[i]); } fprintf(stderr, "\n"); @@ -968,6 +1006,41 @@ collect_end_ids(const struct fsm *fsm, fsm_state_t s, return 1; } +static int +collect_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct end_metadata_eager_outputs *e = opaque; + e->ids[e->count++] = id; + return 1; +} + +static int cmp_eager_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static int +collect_eager_output_ids(const struct fsm *fsm, fsm_state_t state, + struct end_metadata_eager_outputs *e) +{ + size_t count = 0; + if (!fsm_eager_output_has_any(fsm, state, &count)) { + return 1; /* nothing to do */ + } + + e->ids = f_malloc(fsm->alloc, count * sizeof(e->ids[0])); + if (e->ids == NULL) { return 0; } + + fsm_eager_output_iter_state(fsm, state, collect_cb, e); + + /* sort, to normalize set */ + qsort(e->ids, e->count, sizeof(e->ids[0]), cmp_eager_output_id); + return 1; +} + #if EXPENSIVE_CHECKS static void check_done_ec_offset(const struct min_env *env) diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index f55b2e748..b10cd9ace 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -228,6 +228,14 @@ print_case(FILE *f, const struct ir *ir, fsm_state_t state_id, .end_id_count = ir->states[state_id].endids.count, }; + if (cs->eager_outputs != NULL && opt->fragment) { + /* If .fragment is set and the state has eager outputs, then emit a call to a + * macro (the caller is expected to define). This is a temporary interface. */ + for (size_t i = 0; i < cs->eager_outputs->count; i++) { + fprintf(f, "\t\t\tFSM_SET_EAGER_OUTPUT(%u);\n", cs->eager_outputs->ids[i]); + } + } + switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); @@ -383,6 +391,11 @@ print_endstates(FILE *f, const struct fsm_state_metadata state_metadata = { .end_ids = ir->states[i].endids.ids, .end_id_count = ir->states[i].endids.count, + + .eager_output_count = (ir->states[i].eager_outputs == NULL + ? 0 : ir->states[i].eager_outputs->count), + .eager_output_ids = (ir->states[i].eager_outputs == NULL + ? NULL : ir->states[i].eager_outputs->ids), }; if (-1 == print_hook_accept(f, opt, hooks, diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 457716dcc..81d5890e0 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -26,6 +26,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/eager_output.h" #include "ir.h" @@ -505,6 +506,23 @@ make_example(const struct fsm *fsm, fsm_state_t s, char **example) return 0; } +static int +append_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct ir_state_eager_output *outputs = opaque; + (void)state; + outputs->ids[outputs->count++] = id; + return 1; +} + +static int +cmp_fsm_output_id_t(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + struct ir * make_ir(const struct fsm *fsm, const struct fsm_options *opt) { @@ -544,6 +562,8 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) ir->states[i].endids.ids = NULL; ir->states[i].endids.count = 0; + ir->states[i].eager_outputs = NULL; + if (fsm_isend(fsm, i)) { fsm_end_id_t *ids; size_t count; @@ -567,6 +587,20 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) ir->states[i].endids.count = count; } + size_t count; + if (fsm_eager_output_has_any(fsm, i, &count)) { + struct ir_state_eager_output *outputs = f_malloc(fsm->alloc, + sizeof(*outputs) + count * sizeof(outputs->ids[0])); + if (outputs == NULL) { + goto error; + } + outputs->count = 0; + fsm_eager_output_iter_state(fsm, i, append_eager_output_cb, outputs); + assert(outputs->count == count); + qsort(outputs->ids, outputs->count, sizeof(outputs->ids[0]), cmp_fsm_output_id_t); + ir->states[i].eager_outputs = outputs; + } + if (make_state(fsm, i, &ir->states[i]) == -1) { goto error; } @@ -630,6 +664,7 @@ free_ir(const struct fsm *fsm, struct ir *ir) for (i = 0; i < ir->n; i++) { f_free(fsm->alloc, (void *) ir->states[i].example); f_free(fsm->alloc, (void *) ir->states[i].endids.ids); + f_free(fsm->alloc, (void *) ir->states[i].eager_outputs); switch (ir->states[i].strategy) { case IR_TABLE: diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index 074097da3..b4b93a9eb 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -59,6 +59,11 @@ struct ir_state { size_t count; } endids; + struct ir_state_eager_output { + size_t count; + fsm_output_id_t ids[]; + } *eager_outputs; /* NULL -> 0 */ + unsigned int isend:1; enum ir_strategy strategy; diff --git a/src/libfsm/state.c b/src/libfsm/state.c index c845cbe46..d96c33653 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -19,6 +19,7 @@ #include "internal.h" #include "endids.h" +#include "eager_output.h" int fsm_addstate(struct fsm *fsm, fsm_state_t *state) @@ -44,6 +45,7 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) for (i = fsm->statealloc; i < n; i++) { tmp[i].has_capture_actions = 0; + tmp[i].has_eager_outputs = 0; } fsm->statealloc = n; @@ -87,6 +89,8 @@ fsm_addstate_bulk(struct fsm *fsm, size_t n) new->visited = 0; new->epsilons = NULL; new->edges = NULL; + + new->has_eager_outputs = 0; } fsm->statecount += n; @@ -259,6 +263,10 @@ fsm_compact_states(struct fsm *fsm, if (!fsm_endid_compact(fsm, mapping, orig_statecount)) { return 0; } + if (!fsm_eager_output_compact(fsm, mapping, orig_statecount)) { + return 0; + } + assert(dst == kept); assert(kept == fsm->statecount); diff --git a/src/libfsm/union.c b/src/libfsm/union.c index a3b4b230c..0b18cd30c 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -15,9 +15,14 @@ #include #include #include +#include +#include #include "internal.h" +#include +#include "eager_output.h" + #define LOG_UNION_ARRAY 0 struct fsm * @@ -151,3 +156,231 @@ fsm_union_array(size_t fsm_count, return res; } + +#define LOG_UNION_REPEATED_PATTERN_GROUP 0 + +/* Combine an array of FSMs into a single FSM in one pass, with an extra loop + * so that more than one pattern with eager outputs can match. */ +struct fsm * +fsm_union_repeated_pattern_group(size_t entry_count, + struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases) +{ + const struct fsm_alloc *alloc = entries[0].fsm->alloc; + const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; + + if (entry_count == 1) { + return entries[0].fsm; + } + + size_t est_total_states = 0; + for (size_t i = 0; i < entry_count; i++) { + assert(entries[i].fsm); + if (entries[i].fsm->alloc != alloc) { + errno = EINVAL; + return NULL; + } + const size_t count = fsm_countstates(entries[i].fsm); + est_total_states += count; + } + + est_total_states += 5; /* new start and end, new unanchored start and end loops */ + + struct fsm *res = fsm_new_statealloc(alloc, est_total_states); + if (res == NULL) { return NULL; } + + /* collected end states */ + struct ends_buf { + size_t ceil; + size_t used; + fsm_state_t *states; + } ends = { .ceil = 0 }; + + /* The new overall start state, which will have an epsilon edge to... */ + fsm_state_t global_start; + if (!fsm_addstate(res, &global_start)) { goto fail; } + + /* states linking to the starts of unanchored and anchored subgraphs, respectively. */ + fsm_state_t global_start_loop, global_start_anchored; + if (!fsm_addstate(res, &global_start_loop)) { goto fail; } + if (!fsm_addstate(res, &global_start_anchored)) { goto fail; } + + /* The unanchored end loop state, and an end state with no outgoing edges. */ + fsm_state_t global_end_loop, global_end; + if (!fsm_addstate(res, &global_end)) { goto fail; } + if (!fsm_addstate(res, &global_end_loop)) { goto fail; } + + /* link the start to the start loop and anchored start, and the start loop to itself */ + if (log) { + fprintf(stderr, "link_before: global_start %d -> global_start_loop %d and global_start_anchored %d\n", + global_start, global_start_loop, global_start_anchored); + } + if (!fsm_addedge_epsilon(res, global_start, global_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_start_anchored)) { goto fail; } + if (!fsm_addedge_any(res, global_start_loop, global_start_loop)) { goto fail; } + + /* link the end loop and end */ + if (log) { + fprintf(stderr, "link_before: global_end_loop %d -> global_end %d (and -> self)\n", global_end_loop, global_end); + } + if (!fsm_addedge_epsilon(res, global_end_loop, global_end)) { goto fail; } + if (!fsm_addedge_any(res, global_end_loop, global_end_loop)) { goto fail; } + + if (bases != NULL) { + memset(bases, 0x00, entry_count * sizeof(bases[0])); + } + + for (size_t fsm_i = 0; fsm_i < entry_count; fsm_i++) { + ends.used = 0; /* reset */ + + struct fsm *fsm = entries[fsm_i].fsm; + entries[fsm_i].fsm = NULL; /* transfer ownership */ + + const size_t state_count = fsm_countstates(fsm); + + fsm_state_t fsm_start; + if (!fsm_getstart(fsm, &fsm_start)) { + fsm_free(fsm); /* no start, just discard */ + continue; + } + + for (fsm_state_t s_i = 0; s_i < state_count; s_i++) { + if (fsm_isend(fsm, s_i)) { + if (ends.used == ends.ceil) { /* grow? */ + size_t nceil = (ends.ceil == 0 ? 4 : 2*ends.ceil); + fsm_state_t *nstates = f_realloc(alloc, + ends.states, nceil * sizeof(nstates[0])); + if (nstates == NULL) { goto fail; } + ends.ceil = nceil; + ends.states = nstates; + } + ends.states[ends.used++] = s_i; + } + } + + if (ends.used == 0) { + fsm_free(fsm); /* no ends, just discard */ + continue; + } + + /* When combining these, remove self-edges from any states on the FSMs to be + * combined that also have eager output IDs. We are about to add an epsilon edge + * from each to a shared state that won't have eager output IDs. + * + * Eager output matching should be idempotent, so carrying it to other reachable + * state is redundant, and it leads to a combinatorial explosion that blows up the + * state count while determinising the combined FSM otherwise. + * + * For example, if /aaa/, /bbb/, and /ccc/ are combined into a DFA that repeats + * the sub-patterns (like `^.*(?:(aaa)|(bbb)|(ccc))+.*$`), the self-edge at each + * eager output state would combine with every reachable state from then on, + * leading to a copy of the whole reachable subgraph colored by every + * combination of eager output IDs: aaa, bbb, ccc, aaa+bbb, aaa+ccc, + * bbb+ccc, aaa+bbb+ccc. Instead of three relatively separate subgraphs + * that set the eager output at their last state, one for each pattern, + * it leads to 8 (2**3) subgraph clusters because it encodes _each + * distinct combination_ in the DFA. This becomes incredibly expensive + * as the combined pattern count increases; it's essentially what I'm + * trying to avoid by adding eager output support in the first place. + * + * FIXME: instead of actively removing these, filter in fsm_determinise? */ + if (fsm_eager_output_has_eager_output(fsm)) { + /* for any state that has eager outputs and a self edge, + * remove the self edge before further linkage */ + for (fsm_state_t s = 0; s < fsm->statecount; s++) { + if (!fsm_eager_output_has_any(fsm, s, NULL)) { continue; } + struct edge_set *edges = fsm->states[s].edges; + struct edge_set *new = edge_set_new(); + + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (info.to != s) { + if (!edge_set_add_bulk(&new, fsm->alloc, + info.symbols, info.to)) { + goto fail; + } + } + } + edge_set_free(fsm->alloc, edges); + fsm->states[s].edges = new; + } + } + + /* call fsm_merge; we really don't care which is which */ + struct fsm_combine_info combine_info; + struct fsm *merged = fsm_merge(res, fsm, &combine_info); + if (merged == NULL) { goto fail; } + + /* update offsets if res had its state IDs shifted forward */ + global_start += combine_info.base_a; + global_start_loop += combine_info.base_a; + global_start_anchored += combine_info.base_a;; + global_end += combine_info.base_a; + global_end_loop += combine_info.base_a; + + /* also update offsets for the FSM's states */ + fsm_start += combine_info.base_b; + for (size_t i = 0; i < ends.used; i++) { + ends.states[i] += combine_info.base_b; + } + + if (bases != NULL) { + bases[fsm_i].state = combine_info.base_b; + bases[fsm_i].capture = combine_info.capture_base_b; + } + + if (log) { + fprintf(stderr, "%s: fsm[%zd].start: %d\n", __func__, fsm_i, fsm_start); + for (size_t i = 0; i < ends.used; i++) { + fprintf(stderr, "%s: fsm[%zd].ends[%zd]: %d\n", __func__, fsm_i, i, ends.states[i]); + } + } + + /* link to the FSM's start state */ + const fsm_state_t start_src = entries[fsm_i].anchored_start ? global_start_anchored : global_start_loop; + if (!fsm_addedge_epsilon(merged, start_src, fsm_start)) { goto fail; } + if (log) { + fprintf(stderr, "%s: linking %s %d to fsm[%zd]'s start %d (anchored? %d)\n", + __func__, + entries[fsm_i].anchored_start ? "global_start_anchored" : "global_start_loop", + start_src, fsm_i, fsm_start, entries[fsm_i].anchored_start); + } + + /* link from the FSM's ends */ + const fsm_state_t end_dst = entries[fsm_i].anchored_end ? global_end : global_end_loop; + for (size_t i = 0; i < ends.used; i++) { + if (log) { + fprintf(stderr, "%s: linking fsm[%zd]'s end[%zd] %d (anchored? %d) to %s %d\n", + __func__, fsm_i, i, ends.states[i], entries[fsm_i].anchored_end, + entries[fsm_i].anchored_end ? "global_end" : "global_end_loop", + end_dst); + } + if (!fsm_addedge_epsilon(merged, ends.states[i], end_dst)) { goto fail; } + } + + res = merged; + } + + /* Link from the global_end_loop to the global_start_loop, so patterns with an + * unanchored start can follow other patterns with an unanchored end. */ + if (log) { + fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", + __func__, global_start, global_start_loop, global_start_anchored, global_end_loop, global_end); + fprintf(stderr, "%s: linking global_end_loop %d to global_start_loop %d\n", + __func__, global_end_loop, global_start_loop); + fprintf(stderr, "%s: setting global_start %d and end %d\n", __func__, global_start, global_end); + } + if (!fsm_addedge_epsilon(res, global_end_loop, global_start_loop)) { goto fail; } + + /* This needs to be set after merging, because that clears the start state. */ + fsm_setstart(res, global_start); + fsm_setend(res, global_end, 1); + + f_free(alloc, ends.states); + return res; + +fail: + f_free(alloc, ends.states); + return NULL; +} diff --git a/tests/eager_output/Makefile b/tests/eager_output/Makefile new file mode 100644 index 000000000..a650bf802 --- /dev/null +++ b/tests/eager_output/Makefile @@ -0,0 +1,22 @@ +.include "../../share/mk/top.mk" + +TEST.tests/eager_output != ls -1 tests/eager_output/eager_output*.c +TEST_SRCDIR.tests/eager_output = tests/eager_output +TEST_OUTDIR.tests/eager_output = ${BUILD}/tests/eager_output + +.for n in ${TEST.tests/eager_output:T:R:C/^eager_output//} +INCDIR.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c += src/adt +.endfor + +SRC += ${TEST_SRCDIR.tests/eager_output}/utils.c + +.for n in ${TEST.tests/eager_output:T:R:C/^eager_output//} +test:: ${TEST_OUTDIR.tests/eager_output}/res${n} +SRC += ${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c +CFLAGS.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c += -UNDEBUG + +${TEST_OUTDIR.tests/eager_output}/run${n}: ${TEST_OUTDIR.tests/eager_output}/eager_output${n}.o ${TEST_OUTDIR.tests/eager_output}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c} -o ${TEST_OUTDIR.tests/eager_output}/run${n} ${TEST_OUTDIR.tests/eager_output}/eager_output${n}.o ${TEST_OUTDIR.tests/eager_output}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a +${TEST_OUTDIR.tests/eager_output}/res${n}: ${TEST_OUTDIR.tests/eager_output}/run${n} + ( ${TEST_OUTDIR.tests/eager_output}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/eager_output}/res${n} +.endfor diff --git a/tests/eager_output/eager_output1.c b/tests/eager_output/eager_output1.c new file mode 100644 index 000000000..f20ef77b7 --- /dev/null +++ b/tests/eager_output/eager_output1.c @@ -0,0 +1,12 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "abc" }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output2.c b/tests/eager_output/eager_output2.c new file mode 100644 index 000000000..cdac204e2 --- /dev/null +++ b/tests/eager_output/eager_output2.c @@ -0,0 +1,17 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab(c|d|e)" }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "abd", .expected_ids = { 1 } }, + { .input = "abe", .expected_ids = { 1 } }, + { .input = "Xabe", .expected_ids = { 1 } }, + { .input = "abeX", .expected_ids = { 1 } }, + { .input = "XabeX", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output3.c b/tests/eager_output/eager_output3.c new file mode 100644 index 000000000..c11bc58a4 --- /dev/null +++ b/tests/eager_output/eager_output3.c @@ -0,0 +1,16 @@ +#include "utils.h" + +/* test that eager endids are correctly propagated through fsm_determinise() and fsm_minimise() */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab(c|d|e)?" }, + .inputs = { + { .input = "ab", .expected_ids = { 1 } }, + { .input = "abc", .expected_ids = { 1 } }, + { .input = "abd", .expected_ids = { 1 } }, + { .input = "abe", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output4.c b/tests/eager_output/eager_output4.c new file mode 100644 index 000000000..47cd32029 --- /dev/null +++ b/tests/eager_output/eager_output4.c @@ -0,0 +1,13 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "abcde$" }, + .inputs = { + { .input = "abcde", .expected_ids = { 1 } }, + { .input = "Xabcde", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output5.c b/tests/eager_output/eager_output5.c new file mode 100644 index 000000000..4551c68b1 --- /dev/null +++ b/tests/eager_output/eager_output5.c @@ -0,0 +1,14 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "^abc$", "^ab*c$" }, + .inputs = { + { .input = "ac", .expected_ids = { 2 } }, + { .input = "abc", .expected_ids = { 1, 2 } }, + { .input = "abbc", .expected_ids = { 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output6.c b/tests/eager_output/eager_output6.c new file mode 100644 index 000000000..5431d0981 --- /dev/null +++ b/tests/eager_output/eager_output6.c @@ -0,0 +1,34 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "apple", + "banana", + "carrot", + "durian", + "eggplant", + "fig", + "grapefruit", + "hazelnut", + "iceberg lettuce", + "jicama", + }, + .inputs = { + { .input = "apple", .expected_ids = { 1 } }, + { .input = "banana", .expected_ids = { 2 } }, + { .input = "carrot", .expected_ids = { 3 } }, + { .input = "durian", .expected_ids = { 4 } }, + { .input = "eggplant", .expected_ids = { 5 } }, + { .input = "fig", .expected_ids = { 6 } }, + { .input = "grapefruit", .expected_ids = { 7 } }, + { .input = "hazelnut", .expected_ids = { 8 } }, + { .input = "iceberg lettuce", .expected_ids = { 9 } }, + { .input = "jicama", .expected_ids = { 10 } }, + { .input = "apple banana carrot", .expected_ids = { 1, 2, 3 } }, + }, + }; + + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output7.c b/tests/eager_output/eager_output7.c new file mode 100644 index 000000000..3d123878b --- /dev/null +++ b/tests/eager_output/eager_output7.c @@ -0,0 +1,103 @@ +#include "utils.h" + +int main(void) +{ + /* Run this test with env FORCE_ENDIDS=N ... to see how much more + * expensive it is to combine the first N patterns using endids, + * rather than eager_outputs. It becomes VERY slow for >= 9 or so. + * (Note that the checks probably will not pass for N < 4, because + * it will start skipping appear in the early test inputs.) */ + bool force_endids = false; + size_t force_endid_count = 0; + { + const char *str = getenv("FORCE_ENDIDS"); + if (str != NULL) { + force_endid_count = atoi(str); + if (force_endid_count == 0) { + force_endid_count = 26; + } + force_endids = true; + } + } + + struct eager_output_test test = { + .patterns = { + [0] = "apple", + [1] = "banana", + [2] = "carrot", + [3] = "durian", + [4] = "eggplant", + [5] = "fig", + [6] = "grapefruit", + [7] = "hazelnut", + [8] = "iceberg lettuce", + [9] = "jicama", + [10] = "kiwano", + [11] = "lemon", + [12] = "mango", + [13] = "nectarine", + [14] = "orange", + [15] = "plum", + [16] = "quince", + [17] = "radish", + [18] = "strawberry", + [19] = "turnip", + [20] = "ube", + [21] = "vanilla", + [22] = "watermelon", + [23] = "xigua watermelon", + [24] = "yam", + [25] = "zucchini", + }, + .inputs = { + /* Note: expected IDs are shifted by 1, it's 0-terminated. */ + { .input = "apple", .expected_ids = { 1 } }, + { .input = "banana", .expected_ids = { 2 } }, + { .input = "carrot", .expected_ids = { 3 } }, + { .input = "apple banana", .expected_ids = { 1, 2 } }, + { .input = "carrot durian apple", .expected_ids = { 1, 3, 4 } }, + { .input = "carrot fig apple", .expected_ids = { 1, 3, 6 } }, + + /* leading characters and an incomplete trailing match */ + { .input = "mumble mumble fig hazelnut banana xigua watermelo", .expected_ids = { 2, 6, 8 } }, + + /* redundant matches */ + { .input = "ube ube ube ube ube", .expected_ids = { 21 } }, + + /* everything */ + { .input = + "apple banana carrot durian eggplant fig grapefruit " + "hazelnut iceberg lettuce jicamaa kiwano lemon mango " + "nectarine orange plum quince radish strawberry " + "turnip ube vanilla watermelon xigua watermelon yam zucchini", + .expected_ids = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + }, + }, + /* everything, only spaces appearing in patterns */ + { .input = + "applebananacarrotdurianeggplantfiggrapefruit" + "hazelnuticeberg lettucejicamaakiwanolemonmango" + "nectarineorangeplumquinceradishstrawberry" + "turnipubevanillawatermelonxigua watermelonyamzucchini", + .expected_ids = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + }, + }, + }, + }; + + /* truncate patterns to the first N */ + if (force_endids) { + assert(force_endid_count > 0 && force_endid_count <= 26); + test.patterns[force_endid_count] = NULL; + + /* truncate test inputs to just the first couple, since + * later inputs use later patterns */ + test.inputs[5].input = NULL; + } + + return run_test(&test, false, force_endids); +} diff --git a/tests/eager_output/eager_output_at_start.c b/tests/eager_output/eager_output_at_start.c new file mode 100644 index 000000000..407aa4e77 --- /dev/null +++ b/tests/eager_output/eager_output_at_start.c @@ -0,0 +1,12 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_fr1.c b/tests/eager_output/eager_output_fr1.c new file mode 100644 index 000000000..e8e5f3395 --- /dev/null +++ b/tests/eager_output/eager_output_fr1.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab", "" }, + .inputs = { + { .input = "ab", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_fr2.c b/tests/eager_output/eager_output_fr2.c new file mode 100644 index 000000000..404e98644 --- /dev/null +++ b/tests/eager_output/eager_output_fr2.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "", "" }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_fr3.c b/tests/eager_output/eager_output_fr3.c new file mode 100644 index 000000000..c7e4127a6 --- /dev/null +++ b/tests/eager_output/eager_output_fr3.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "^", "" }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_mixed_anchored_unanchored.c b/tests/eager_output/eager_output_mixed_anchored_unanchored.c new file mode 100644 index 000000000..a586f9840 --- /dev/null +++ b/tests/eager_output/eager_output_mixed_anchored_unanchored.c @@ -0,0 +1,46 @@ +#include "utils.h" + +int main(void) +{ + /* fprintf(stderr, "%s: skipping for now, this doesn't pass yet.\n", __FILE__); */ + /* return EXIT_SUCCESS; */ + + struct eager_output_test test = { + .patterns = { + "^abc$", + "def", + "^ghi", + "jkl$", + "mno", + }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "def", .expected_ids = { 2 } }, + { .input = "ghi", .expected_ids = { 3 } }, + { .input = "jkl", .expected_ids = { 4 } }, + { .input = "mno", .expected_ids = { 5 } }, + + { .input = "defmno", .expected_ids = { 2, 5 } }, + { .input = " def mno ", .expected_ids = { 2, 5 } }, + + /* Matching a start-anchored pattern followed by + * unanchored ones should just work. */ + { .input = "ghi def", .expected_ids = { 2, 3 } }, + + /* An unanchored pattern before a start-anchored pattern + * should only match the unanchored pattern. */ + { .input = "def ghi", .expected_ids = { 2 } }, + + /* Matching an unanchored pattern before an + * end-anchored one is fine. */ + { .input = "mno jkl", .expected_ids = { 4, 5 } }, + + /* This should match "mno" with the "jkl" prefix + * ignored by the unanchored start, which does + * not count as a match for "jkl$". */ + { .input = "jkl mno", .expected_ids = { 5 } }, + }, + }; + + return run_test(&test, false, false); +} diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c new file mode 100644 index 000000000..4bee8d848 --- /dev/null +++ b/tests/eager_output/utils.c @@ -0,0 +1,278 @@ +#include "utils.h" + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +void +fsm_endid_dump(FILE *f, const struct fsm *fsm); + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque) +{ + struct cb_info *info = (struct cb_info *)opaque; + assert(info->used < MAX_IDS); + + for (size_t i = 0; i < info->used; i++) { + if (info->ids[i] == id) { + return; /* already present */ + } + } + + info->ids[info->used++] = id; +} + +int +cmp_output(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +struct fsm_options print_options = { + .consolidate_edges = 1, + .comments = 0, + .group_edges = 1, +}; + +void +dump(const struct fsm *fsm) +{ + fsm_print(stderr, fsm, + &print_options, NULL, FSM_PRINT_DOT); +} + +int +run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids) +{ + struct fsm_union_entry entries[MAX_PATTERNS] = {0}; + + allow_extra_outputs = false; + + size_t fsms_used = 0; + int ret = 0; + + int log = 0; + { + const char *logstr = getenv("LOG"); + if (logstr != NULL) { + if (logstr[0] == 'y') { /* make "y" or "yes" non-zero */ + logstr = "1"; + } + log = atoi(logstr); + } + } + + for (size_t i = 0; i < MAX_PATTERNS; i++) { + const char *p = test->patterns[i]; + if (test->patterns[i] == NULL) { break; } + const size_t len = strlen(p); + struct fsm_union_entry *e = &entries[fsms_used]; + + /* For sake of these patterns, they are anchored if the first/last + * character is '^' and '$', respectively. This is too simplistic + * for the general case, though. */ + if (len > 0) { + if (p[0] == '^') { e->anchored_start = true; } + if (p[len - 1] == '$') { e->anchored_end = true; } + /* fprintf(stderr, "%s: p[%zd]: '%s', start %d, end %d\n", */ + /* __func__, fsms_used, p, e->anchored_start, e->anchored_end); */ + } + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); + assert(fsm != NULL); + + /* Zero is used to terminate expected_ids, so don't use it here. */ + const fsm_output_id_t output_id = (fsm_output_id_t) (i + 1); + const fsm_end_id_t end_id = (fsm_end_id_t) (i + 1); + + /* Set either an end ID or an eager output ID, depending on + * whether the fsm is anchored at the end or not. */ + if (e->anchored_end || force_endids) { + ret = fsm_setendid(fsm, end_id); + } else { + ret = fsm_seteageroutputonends(fsm, output_id); + } + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== source DFA %zd (pre det+min)\n", i); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + ret = fsm_determinise(fsm); + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== source DFA %zd (post det)\n", i); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + ret = fsm_minimise(fsm); + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== source DFA %zd (post det+min)\n", i); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + e->fsm = fsm; + fsms_used++; + } + + /* If there's only one pattern this just returns fsms[0]. */ + struct fsm *fsm = fsm_union_repeated_pattern_group(fsms_used, entries, NULL); + assert(fsm != NULL); + + if (log) { + fprintf(stderr, "==== combined (pre det+min)\n"); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + if (log) { + fprintf(stderr, "=== determinising combined... NFA has %u states\n", fsm_countstates(fsm)); + } + ret = fsm_determinise(fsm); + assert(ret == 1); + if (log) { + fprintf(stderr, "=== determinising combined...done, DFA has %u states\n", fsm_countstates(fsm)); + } + + if (log) { + fprintf(stderr, "==== combined (post det)\n"); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + ret = fsm_minimise(fsm); + if (log) { + fprintf(stderr, "=== minimised combined...done, DFA has %u states\n", fsm_countstates(fsm)); + } + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== combined (post det+min)\n"); + if (log > 1) { dump(fsm); } + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + struct cb_info outputs = { 0 }; + fsm_eager_output_set_cb(fsm, append_eager_output_cb, &outputs); + + for (size_t i_i = 0; i_i < MAX_INPUTS; i_i++) { + outputs.used = 0; + const char *input = test->inputs[i_i].input; + if (input == NULL) { break; } + + size_t expected_id_count = 0; + for (size_t id_i = 0; id_i < MAX_ENDIDS; id_i++) { + const fsm_output_id_t id = test->inputs[i_i].expected_ids[id_i]; + if (id == 0) { break; } + expected_id_count++; + + /* must be ascending */ + if (id_i > 0) { + assert(id > test->inputs[i_i].expected_ids[id_i - 1]); + } + } + + if (log) { + fprintf(stderr, "%s: input %zd: \"%s\", expecting %zd ids:", + __func__, i_i, input, expected_id_count); + for (size_t i = 0; i < expected_id_count; i++) { + fprintf(stderr, " %d", test->inputs[i_i].expected_ids[i]); + } + } + + if (test->inputs[i_i].expect_fail) { + expected_id_count = 0; + } + + fsm_state_t end; /* only set on match */ + ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + + if (ret == 1) { +#define ENDID_BUF_SIZE 32 + fsm_end_id_t endid_buf[ENDID_BUF_SIZE] = {0}; + const size_t endid_count = fsm_endid_count(fsm, end); + /* fprintf(stderr, "%s: endid_count %zd for state %d\n", __func__, endid_count, end); */ + assert(endid_count < ENDID_BUF_SIZE); + if (!fsm_endid_get(fsm, end, /*ENDID_BUF_SIZE*/ endid_count, endid_buf)) { + assert(!"fsm_endid_get failed"); + } + + /* Copy endid outputs into outputs.ids[], since for testing + * purposes we don't care about the difference between eager + * output and endids here -- the values don't overlap. */ + assert(outputs.used + endid_count <= MAX_IDS); + for (size_t endid_i = 0; endid_i < endid_count; endid_i++) { + if (log) { + fprintf(stderr, "-- adding endid %zd: %d\n", endid_i, endid_buf[endid_i]); + } + outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + } + } + + if (ret == 0) { + /* if it didn't match, ignore the eager output IDs. this should + * eventually happen internal to fsm_exec or codegen. */ + outputs.used = 0; + } + + /* NEXT match IDs, sort outputs[] buffer first */ + qsort(outputs.ids, outputs.used, sizeof(outputs.ids[0]), cmp_output); + + if (log) { + fprintf(stderr, "-- got %zd:", outputs.used); + for (size_t i = 0; i < outputs.used; i++) { + fprintf(stderr, " %d", outputs.ids[i]); + } + fprintf(stderr, "\n"); + } + + if (expected_id_count == 0) { + assert(ret == 0 || outputs.used == 0); /* no match */ + continue; + } else { + assert(ret == 1); + } + + if (!allow_extra_outputs) { + assert(outputs.used == expected_id_count); + } else { + assert(outputs.used >= expected_id_count); + } + + size_t floor = 0; + for (size_t exp_i = 0; exp_i < outputs.used; exp_i++) { + bool found = false; + for (size_t got_i = floor; got_i < outputs.used; got_i++) { + if (outputs.ids[got_i] == test->inputs[i_i].expected_ids[exp_i]) { + floor = got_i + 1; + found = true; + break; + } + } + assert(found); + } + } + + fsm_free(fsm); + + return EXIT_SUCCESS;; +} diff --git a/tests/eager_output/utils.h b/tests/eager_output/utils.h new file mode 100644 index 000000000..672c01977 --- /dev/null +++ b/tests/eager_output/utils.h @@ -0,0 +1,64 @@ +#ifndef UTILS_H +#define UTILS_H + +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#define MAX_IDS 32 + +#include + +#include + +#define MAX_PATTERNS 150 +#define MAX_INPUTS 64 +#define MAX_ENDIDS 32 + +struct eager_output_test { + const char *patterns[MAX_PATTERNS]; + + struct { + const char *input; + bool expect_fail; + /* Terminated by 0. pattern[i] => id of i+1. Must be sorted. */ + fsm_output_id_t expected_ids[MAX_ENDIDS]; + } inputs[MAX_INPUTS]; +}; + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque); + +int +cmp_output(const void *pa, const void *pb); + +int +run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids); + +struct cb_info { + size_t used; + fsm_end_id_t ids[MAX_IDS]; +}; + +void +dump(const struct fsm *fsm); + +void +append_eager_output_cb(fsm_end_id_t id, void *opaque); + +#endif From 74907caa28f7e241a3cdf0cf81242f8a1fd4b636 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 13:32:38 -0400 Subject: [PATCH 312/373] Ensure .has_eager_outputs is zeroed on new states. (msan) --- src/libfsm/state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/state.c b/src/libfsm/state.c index d96c33653..8f1146038 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -65,6 +65,7 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) new->visited = 0; new->epsilons = NULL; new->edges = NULL; + new->has_eager_outputs = 0; } fsm->statecount++; From f2ddf1d3ab7e74bcbbe431d05f505aa19a04268d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 14:15:33 -0400 Subject: [PATCH 313/373] eager_output interface cleanup: Replace _any with _count and _get. fsm_eager_output_count and fsm_eager_output_get aligns better with the endid interface, and now fsm_eager_output_get ensures the buffer contents are sorted, --- include/fsm/fsm.h | 10 ++++++ src/libfsm/eager_output.c | 69 +++++++++++++++++++++++++++------------ src/libfsm/eager_output.h | 4 --- src/libfsm/epsilons.c | 3 +- src/libfsm/libfsm.syms | 2 ++ src/libfsm/minimise.c | 4 +-- src/libfsm/print/ir.c | 30 ++++------------- src/libfsm/union.c | 3 +- 8 files changed, 73 insertions(+), 52 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index b862cb041..f78d91d71 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -303,6 +303,16 @@ fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque); void fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque); +/* Get the number of eager output IDs associated with a state. */ +size_t +fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state); + +/* Get eager output associated with a state. It's expected that buf[] has + * sufficient space -- call fsm_eager_output_count first to get the count. + * The contents of buf will be sorted and unique. */ +void +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, fsm_output_id_t *buf); + /* * Find the state (if there is just one), or add epsilon edges from all states, * for which the given predicate is true. diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c index e37a8a4bf..e00e96cd1 100644 --- a/src/libfsm/eager_output.c +++ b/src/libfsm/eager_output.c @@ -276,6 +276,55 @@ fsm_eager_output_iter_state(const struct fsm *fsm, } } +static int +inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + (void)id; + size_t *count = opaque; + (*count)++; + return 1; +} + +/* Get the number of eager output IDs associated with a state. */ +size_t +fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state) +{ + size_t res = 0; + fsm_eager_output_iter_state(fsm, state, inc_cb, (void *)&res); + return res; +} + +struct get_env { + size_t count; + fsm_output_id_t *buf; +}; + +static int +append_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct get_env *env = opaque; + (void)state; + env->buf[env->count++] = id; + return 1; +} + +static int +cmp_fsm_output_id_t(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +void +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, fsm_output_id_t *buf) +{ + struct get_env env = { .buf = buf }; + fsm_eager_output_iter_state(fsm, state, append_cb, &env); + qsort(buf, env.count, sizeof(buf[0]), cmp_fsm_output_id_t); +} + void fsm_eager_output_iter_all(const struct fsm *fsm, fsm_eager_output_iter_cb *cb, void *opaque) @@ -327,26 +376,6 @@ fsm_eager_output_dump(FILE *f, const struct fsm *fsm) fprintf(f, "== %zu total\n", env.count); } -static int -inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) -{ - (void)state; - (void)id; - size_t *count = opaque; - (*count)++; - return 1; -} - -bool -fsm_eager_output_has_any(const struct fsm *fsm, - fsm_state_t state, size_t *count) -{ - size_t c = 0; - fsm_eager_output_iter_state(fsm, state, &inc_cb, &c); - if (count != NULL) { *count = c; } - return c > 0; -} - int fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count) { diff --git a/src/libfsm/eager_output.h b/src/libfsm/eager_output.h index 1b48ba4c4..6093adc9e 100644 --- a/src/libfsm/eager_output.h +++ b/src/libfsm/eager_output.h @@ -36,10 +36,6 @@ void fsm_eager_output_iter_all(const struct fsm *fsm, fsm_eager_output_iter_cb *cb, void *opaque); -bool -fsm_eager_output_has_any(const struct fsm *fsm, - fsm_state_t state, size_t *count); - int fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index adfcdec2a..8041c29d3 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -222,7 +222,8 @@ fsm_remove_epsilons(struct fsm *nfa) * in the current state's epsilon closure to the * current state. These will be added at the end. */ { - if (fsm_eager_output_has_any(nfa, es_id, NULL)) { + const size_t count = fsm_eager_output_count(nfa, es_id); + if (count > 0) { fsm_eager_output_iter_state(nfa, es_id, collect_eager_output_ids_cb, &eager_output_buf); if (!eager_output_buf.ok) { goto cleanup; } } diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index f645c4ceb..ab28b0a21 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -102,9 +102,11 @@ fsm_endid_dump fsm_seteageroutput fsm_seteageroutputonends +fsm_eager_output_count # short term hack fsm_eager_output_set_cb fsm_eager_output_dump +fsm_eager_output_get fsm_countedges fsm_countstates diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 86f00b46f..a2ff1b818 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -1026,8 +1026,8 @@ static int collect_eager_output_ids(const struct fsm *fsm, fsm_state_t state, struct end_metadata_eager_outputs *e) { - size_t count = 0; - if (!fsm_eager_output_has_any(fsm, state, &count)) { + size_t count = fsm_eager_output_count(fsm, state); + if (count == 0) { return 1; /* nothing to do */ } diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 81d5890e0..a18dadbbc 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -506,23 +506,6 @@ make_example(const struct fsm *fsm, fsm_state_t s, char **example) return 0; } -static int -append_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) -{ - struct ir_state_eager_output *outputs = opaque; - (void)state; - outputs->ids[outputs->count++] = id; - return 1; -} - -static int -cmp_fsm_output_id_t(const void *pa, const void *pb) -{ - const fsm_output_id_t a = *(fsm_output_id_t *)pa; - const fsm_output_id_t b = *(fsm_output_id_t *)pb; - return a < b ? -1 : a > b ? 1 : 0; -} - struct ir * make_ir(const struct fsm *fsm, const struct fsm_options *opt) { @@ -587,17 +570,16 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) ir->states[i].endids.count = count; } - size_t count; - if (fsm_eager_output_has_any(fsm, i, &count)) { + const size_t eager_output_count = fsm_eager_output_count(fsm, i); + if (eager_output_count > 0) { struct ir_state_eager_output *outputs = f_malloc(fsm->alloc, - sizeof(*outputs) + count * sizeof(outputs->ids[0])); + sizeof(*outputs) + eager_output_count * sizeof(outputs->ids[0])); if (outputs == NULL) { goto error; } - outputs->count = 0; - fsm_eager_output_iter_state(fsm, i, append_eager_output_cb, outputs); - assert(outputs->count == count); - qsort(outputs->ids, outputs->count, sizeof(outputs->ids[0]), cmp_fsm_output_id_t); + fsm_eager_output_get(fsm, i, outputs->ids); + outputs->count = eager_output_count; + ir->states[i].eager_outputs = outputs; } diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 0b18cd30c..126181992 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -287,7 +287,8 @@ fsm_union_repeated_pattern_group(size_t entry_count, /* for any state that has eager outputs and a self edge, * remove the self edge before further linkage */ for (fsm_state_t s = 0; s < fsm->statecount; s++) { - if (!fsm_eager_output_has_any(fsm, s, NULL)) { continue; } + const size_t eager_output_count = fsm_eager_output_count(fsm, s); + if (eager_output_count == 0) { continue; } struct edge_set *edges = fsm->states[s].edges; struct edge_set *new = edge_set_new(); From 981128f3b6456f9faeb8633f0728afe113785123 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 14:16:59 -0400 Subject: [PATCH 314/373] minimise_test_oracle.c: mismatched eager outputs also prevent merging. Do the same check with eager outputs as it does with endids -- any two states with distinct sets for either should end up in different equivalence classes. --- src/libfsm/minimise_test_oracle.c | 107 ++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index ec6d0d83c..20d4633a1 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -109,13 +109,19 @@ fsm_minimise_test_oracle(const struct fsm *fsm) fsm_state_t *tmp_map = NULL; fsm_state_t *mapping = NULL; - /* endid_group_assignments[X] = Y: state X is in endid group Y - * endid_group_leaders[X] = Y: see end state Y for endid group X */ - unsigned *endid_group_assignments = NULL; - size_t endid_group_count = 1; /* group 0 is the empty set */ - unsigned *endid_group_leaders = NULL; + /* End metadata grouping: The fixpoint algorithm here isn't aware + * of endids or eager outputs associated with particular states, + * so do a pass grouping them by matching end metadata. + * + * end_md_group_assignments[X] = Y: state X is in end_md group Y + * end_md_group_leaders[X] = Y: see end state Y for end_md group X */ + unsigned *end_md_group_assignments = NULL; + size_t end_md_group_count = 1; /* group 0 is the empty set */ + unsigned *end_md_group_leaders = NULL; fsm_end_id_t *ids_a = NULL; fsm_end_id_t *ids_b = NULL; + fsm_output_id_t *eo_ids_a = NULL; + fsm_output_id_t *eo_ids_b = NULL; table = calloc(row_words * table_states, sizeof(table[0])); if (table == NULL) { goto cleanup; } @@ -126,11 +132,11 @@ fsm_minimise_test_oracle(const struct fsm *fsm) mapping = malloc(state_count * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; } - endid_group_assignments = calloc(state_count, sizeof(tmp_map[0])); - if (endid_group_assignments == NULL) { goto cleanup; } + end_md_group_assignments = calloc(state_count, sizeof(tmp_map[0])); + if (end_md_group_assignments == NULL) { goto cleanup; } - endid_group_leaders = calloc(state_count, sizeof(tmp_map[0])); - if (endid_group_leaders == NULL) { goto cleanup; } + end_md_group_leaders = calloc(state_count, sizeof(tmp_map[0])); + if (end_md_group_leaders == NULL) { goto cleanup; } /* macros for NxN bit table */ #define POS(X,Y) ((X*table_states) + Y) @@ -139,6 +145,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) #define CHECK(X,Y) u64bitset_get(table, POS(X,Y)) size_t max_endid_count = 0; + size_t max_eager_output_count = 0; /* Mark all pairs of states where one is final and one is not. * This includes the dead state. */ @@ -150,6 +157,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) } } + /* count eager outputs, not just on end states */ + const size_t eo_count = fsm_eager_output_count(fsm, i); + if (eo_count > max_eager_output_count) { + max_eager_output_count = eo_count; + } + for (size_t j = 0; j < i; j++) { const bool end_i = i == dead_state ? false : fsm_isend(fsm, i); @@ -171,50 +184,70 @@ fsm_minimise_test_oracle(const struct fsm *fsm) ids_b = malloc(max_endid_count * sizeof(ids_b[0])); if (ids_b == NULL) { goto cleanup; } - /* For every end state, check if it has endids. If not, assign it - * to endid group 0 (none). Otherwise, check if its endids match - * any of the other end states. If so, assign it to the same endid + eo_ids_a = malloc(max_eager_output_count * sizeof(eo_ids_a[0])); + if (eo_ids_a == NULL) { goto cleanup; } + eo_ids_b = malloc(max_eager_output_count * sizeof(eo_ids_b[0])); + if (eo_ids_b == NULL) { goto cleanup; } + + /* For every end state, check if it has endids or eager outputs. + * If not, assign it to group 0 (none). Otherwise, check if its IDs match + * any of the other end states. If so, assign it to the same * group, otherwise assign a new one and mark it as the leader. */ for (size_t i = 0; i < state_count; i++) { if (!fsm_isend(fsm, i)) { - endid_group_assignments[i] = 0; /* none */ + end_md_group_assignments[i] = 0; /* none */ continue; } - size_t count_a = fsm_endid_count(fsm, i); - assert(count_a <= max_endid_count); - if (count_a == 0) { + const size_t endid_count_a = fsm_endid_count(fsm, i); + assert(endid_count_a <= max_endid_count); + + const size_t eager_output_count_a = fsm_eager_output_count(fsm, i); + assert(eager_output_count_a <= max_eager_output_count); + + if (endid_count_a == 0 && eager_output_count_a == 0) { continue; } - int eres = fsm_endid_get(fsm, i, count_a, ids_a); + int eres = fsm_endid_get(fsm, i, endid_count_a, ids_a); assert(eres == 1); + fsm_eager_output_get(fsm, i, eo_ids_a); + bool found = false; /* note: skipping eg 0 here since that's the empty set */ - for (size_t eg_i = 1; eg_i < endid_group_count; eg_i++) { - size_t count_b = fsm_endid_count(fsm, endid_group_leaders[eg_i]); - if (count_b != count_a) { + for (size_t eg_i = 1; eg_i < end_md_group_count; eg_i++) { + size_t endid_count_b = fsm_endid_count(fsm, end_md_group_leaders[eg_i]); + if (endid_count_b != endid_count_a) { continue; } - assert(count_b > 0); - assert(count_b <= max_endid_count); - eres = fsm_endid_get(fsm, endid_group_leaders[eg_i], - count_b, ids_b); + const size_t eager_output_count_b = fsm_eager_output_count(fsm, end_md_group_leaders[eg_i]); + assert(eager_output_count_b <= max_eager_output_count); + if (eager_output_count_b != eager_output_count_a) { + continue; + } + + assert(endid_count_b > 0 || eager_output_count_b > 0); + assert(endid_count_b <= max_endid_count); + eres = fsm_endid_get(fsm, end_md_group_leaders[eg_i], + endid_count_b, ids_b); assert(eres == 1); - if (0 == memcmp(ids_a, ids_b, count_a * sizeof(ids_a[0]))) { + fsm_eager_output_get(fsm, end_md_group_leaders[eg_i], eo_ids_b); + + if ((0 == memcmp(ids_a, ids_b, endid_count_a * sizeof(ids_a[0]))) && + (0 == memcmp(eo_ids_a, eo_ids_b, eager_output_count_a * sizeof(eo_ids_a[0])))) { found = true; - endid_group_assignments[i] = eg_i; + end_md_group_assignments[i] = eg_i; break; } } if (!found) { - endid_group_assignments[i] = endid_group_count; - endid_group_leaders[endid_group_count] = i; - endid_group_count++; + end_md_group_assignments[i] = end_md_group_count; + end_md_group_leaders[end_md_group_count] = i; + end_md_group_count++; } } @@ -222,10 +255,10 @@ fsm_minimise_test_oracle(const struct fsm *fsm) * group must be distinguishable. */ for (size_t i = 0; i < state_count; i++) { if (fsm_isend(fsm, i)) { - const unsigned i_group = endid_group_assignments[i]; + const unsigned i_group = end_md_group_assignments[i]; for (size_t j = 0; j < i; j++) { if (fsm_isend(fsm, j)) { - const unsigned j_group = endid_group_assignments[j]; + const unsigned j_group = end_md_group_assignments[j]; if (i_group != j_group) { MARK(i, j); } @@ -359,10 +392,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) free(table); free(tmp_map); free(mapping); - free(endid_group_assignments); - free(endid_group_leaders); + free(end_md_group_assignments); + free(end_md_group_leaders); free(ids_a); free(ids_b); + free(eo_ids_a); + free(eo_ids_b); return res; @@ -370,10 +405,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) if (table != NULL) { free(table); } if (tmp_map != NULL) { free(tmp_map); } if (mapping != NULL) { free(mapping); } - if (endid_group_assignments != NULL) { free(endid_group_assignments); } - if (endid_group_leaders != NULL) { free(endid_group_leaders); } + if (end_md_group_assignments != NULL) { free(end_md_group_assignments); } + if (end_md_group_leaders != NULL) { free(end_md_group_leaders); } if (ids_a != NULL) { free(ids_a); } if (ids_b != NULL) { free(ids_b); } if (res != NULL) { fsm_free(res); } + free(eo_ids_a); + free(eo_ids_b); return NULL; } From fa63fcd1fe272c1e7f68dbd48c25b40954945a90 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Sat, 12 Oct 2024 13:54:23 -0400 Subject: [PATCH 315/373] fuzz/target.c: re_is_anchor interface changes. --- fuzz/target.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index d56a9bf82..53b6003c1 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -503,13 +503,14 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - struct re_anchoring_info anchorage[MAX_PATTERNS] = {0}; + enum re_is_anchored_res anchorage[MAX_PATTERNS] = {0}; /* for each pattern, attempt to compile to a DFA */ for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { const char *p = env.patterns[p_i]; - if (!re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL, &anchorage[p_i])) { + enum re_is_anchored_res a = re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL); + if (a == RE_IS_ANCHORED_ERROR) { continue; /* unsupported regex */ } @@ -599,8 +600,8 @@ fuzz_eager_output(const uint8_t *data, size_t size) } entries[used].fsm = cp; - entries[used].anchored_start = anchorage[i].start; - entries[used].anchored_end = anchorage[i].end; + entries[used].anchored_start = anchorage[i] & RE_IS_ANCHORED_START; + entries[used].anchored_end = anchorage[i] & RE_IS_ANCHORED_END; used++; } From 496198d5b19230174802e69a62c30f27ae8ff25b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 10 Oct 2024 15:48:13 -0400 Subject: [PATCH 316/373] fuzzer: Add seed argument for fsm_generate_matches (interface change). --- fuzz/target.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 53b6003c1..4ff8b63bd 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -446,6 +446,8 @@ fuzz_eager_output(const uint8_t *data, size_t size) size_t max_pattern_length = 0; + const unsigned seed = size == 0 ? 0 : data[0]; + /* chop data into a series of patterns */ { size_t prev = 0; @@ -646,7 +648,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) * Use the combined DFA to generate matches, check that the * match behavior agrees with the individual DFA copies. */ env.current_pattern = (size_t)-1; - if (!fsm_generate_matches(env.combined, max_pattern_length, gen_combined_check_individual_cb, &env)) { + if (!fsm_generate_matches(env.combined, max_pattern_length, seed, gen_combined_check_individual_cb, &env)) { goto cleanup; } @@ -656,7 +658,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) /* check behavior against the combined DFA. */ for (size_t i = 0; i < env.pattern_count; i++) { env.current_pattern = i; - if (!fsm_generate_matches(env.combined, max_pattern_length, gen_individual_check_combined_cb, &env)) { + if (!fsm_generate_matches(env.combined, max_pattern_length, seed, gen_individual_check_combined_cb, &env)) { goto cleanup; } } From e2b913072545ee52e92efbc5f41f90c90cb44549 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 31 Jan 2025 15:09:24 -0500 Subject: [PATCH 317/373] Fix memory leak in fsm_eager_output_compact, found while fuzzing. --- src/libfsm/eager_output.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c index e00e96cd1..00fa1b5f0 100644 --- a/src/libfsm/eager_output.c +++ b/src/libfsm/eager_output.c @@ -408,7 +408,10 @@ fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_c assert(ob->state < mapping_count); const fsm_state_t nstate = mapping[ob->state]; - if (nstate == FSM_STATE_REMAP_NO_STATE) { continue; } + if (nstate == FSM_STATE_REMAP_NO_STATE) { + f_free(fsm->alloc, ob->entry); + continue; + } const uint64_t hash = hash_id(nstate); From 36d018726f3d5f2e252ee3c866865e0ffdbe8c28 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 4 Feb 2025 12:45:08 -0500 Subject: [PATCH 318/373] Fix fsm_union_repeated_pattern_group's anchoring linkage. Previously this didn't handle mixed anchoring correctly, potentially leading to false positives the case represented by eager_output_alt_mixing_anchored_and_unanchored.c. See comments in fsm_union_repeated_pattern_group for details. Fuzzing did not turn up any new issues. Another commit after this will make a few small interface changes and update callers. --- src/libfsm/union.c | 751 ++++++++++++++---- ...utput_alt_mixing_anchored_and_unanchored.c | 29 + .../eager_output_at_start_multiple_anchored.c | 15 + .../eager_output_at_start_single_anchored.c | 22 + tests/eager_output/utils.c | 90 +-- 5 files changed, 716 insertions(+), 191 deletions(-) create mode 100644 tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c create mode 100644 tests/eager_output/eager_output_at_start_multiple_anchored.c create mode 100644 tests/eager_output/eager_output_at_start_single_anchored.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 126181992..ed71384ff 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -21,9 +21,54 @@ #include "internal.h" #include +#include + #include "eager_output.h" +#include "endids.h" #define LOG_UNION_ARRAY 0 +#define LOG_ANALYZE_GROUP_NFA_RESULTS 0 +#define LOG_UNION_REPEATED_PATTERN_GROUP 0 + +#define NO_STATE (fsm_state_t)-1 + +/* State/edge info gathered about an NFA. Used by fsm_union_repeated_pattern_group. */ +struct analysis_info { + bool nullable; /* Does the NFA match the empty string? */ + fsm_state_t start; /* start state */ + + /* The states with a /./ self edge representing the unanchored + * start and end, or NO_STATE. There can be at most one of each. */ + fsm_state_t unanchored_start_loop; + fsm_state_t unanchored_end_loop; + + /* The end state following the unanchored end loop. */ + fsm_state_t unanchored_end_loop_end; + + /* State that links to paths only reachable from the beginning of input. */ + fsm_state_t anchored_start; + + /* States leading to an anchored end. */ + struct state_set *anchored_ends; + + /* States with an outgoing labeled edge to the unanchored end loop. Input + * following those edges has matched, but may still consume trailing input. + * These edges correspond to edges leaving capture group 0 in PCRE. */ + struct state_set *eager_matches; + + /* Edges leading to states that can only match at the start of input. */ + struct edge_set *anchored_firsts; + + /* Edges leading to states that can begin an unanchored match, + * potentially after other combined patterns have matched. */ + struct edge_set *repeatable_firsts; + + /* A new state that may be added while replacing the unanchored_end_loop, + * if present. This state exists to set an eager output ID and have an + * epsilon edge to the combined NFA's global_unanchored_end_loop. + * The old unanchored_end_loop will be disconnected. */ + fsm_state_t eager_match_state; +}; struct fsm * fsm_union(struct fsm *a, struct fsm *b, @@ -157,10 +202,441 @@ fsm_union_array(size_t fsm_count, return res; } -#define LOG_UNION_REPEATED_PATTERN_GROUP 0 +static bool +has_dot_self_edge(const struct fsm *nfa, fsm_state_t s_i) +{ + const struct fsm_state *s = &nfa->states[s_i]; + + struct edge_group_iter ei; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &ei); + struct edge_group_iter_info info; + while (edge_set_group_iter_next(&ei, &info)) { + if (info.to != s_i) { continue; } + for (size_t i = 0; i < 256/64; i++) { + if (info.symbols[i] != (uint64_t)-1) { continue; } + } + return true; + } + + return false; +} + +#if LOG_ANALYZE_GROUP_NFA_RESULTS +static void +dump_state_set(FILE *f, const char *name, const struct state_set *set) +{ + struct state_iter si; + fsm_state_t s; + if (state_set_empty(set)) { return; } + + fprintf(f, " - %s:", name); + state_set_reset(set, &si); + while (state_set_next(&si, &s)) { + fprintf(f, " %d", s); + } + fprintf(f, "\n"); +} + +static void +dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set *edges) +{ + struct edge_group_iter iter; + struct edge_group_iter_info info; + if (edge_set_empty(edges)) { return; } + + fprintf(f, " - %s:", name); + edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + fprintf(f, " %d->%d", from, info.to); + } + fprintf(f, "\n"); +} +#endif + +/* If there's a labeled edge to an end state, check if the label set is + * only [\n] and there's also an epsilon edge to the same end state. + * This represents an anchored end in the NFA. */ +static bool +has_epsilon_and_newline_edges_to_end(const struct fsm *nfa, fsm_state_t s_i, fsm_state_t *dst_end) +{ + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + + if (state_set_empty(s->epsilons)) { return false; } + if (edge_set_empty(s->edges)) { return false; } + + struct edge_group_iter iter; + struct edge_group_iter_info info; + + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + /* Look for an edge set with only '\n' */ + if ((info.symbols[0] != (1ULL << '\n')) + || info.symbols[1] || info.symbols[2] || info.symbols[3]) { + continue; + } + + if (fsm_isend(nfa, info.to)) { + struct state_iter si; + fsm_state_t os_i; + state_set_reset(s->epsilons, &si); + while (state_set_next(&si, &os_i)) { + if (os_i == info.to) { + *dst_end = info.to; + return true; + } + } + } + } + + return false; +} + +static bool +has_labeled_edge_to_unanchored_end_loop(const struct fsm *nfa, + fsm_state_t s_i, fsm_state_t unanchored_end_loop) +{ + if (unanchored_end_loop == NO_STATE) { return false; } + assert(unanchored_end_loop < nfa->statecount); + + /* The unanchored_end_loop's self-edge doesn't count here. */ + if (s_i == unanchored_end_loop) { return false; } + + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (info.to == unanchored_end_loop) { + return true; + } + } + return false; +} + +static bool +start_state_epsilon_closure_matches_empty_string__iter(const struct fsm *nfa, + fsm_state_t s_i, struct state_set **seen, bool *result) +{ + if (*result) { return true; } + if (!state_set_add(seen, nfa->alloc, s_i)) { return false; } + + if (fsm_isend(nfa, s_i)) { + *result = true; + return true; + } + + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (!state_set_contains(*seen, ns_i)) { + if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, ns_i, seen, result)) { + return false; + } + if (*result) { break; } + } + } + + return true; +} + +/* Does the start state's epsilon closure match the empty string? + * Returns false on error, otherwise returns true and sets *result. */ +static bool +start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, fsm_state_t start, bool *result) +{ + struct state_set *seen = NULL; /* empty set */ + if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, start, &seen, result)) { return false; } + state_set_free(seen); + + return true; +} + +static bool +analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) +{ + memset(ainfo, 0x00, sizeof(*ainfo)); + ainfo->start = NO_STATE; + ainfo->unanchored_start_loop = NO_STATE; + ainfo->anchored_start = NO_STATE; + ainfo->unanchored_end_loop = NO_STATE; + ainfo->unanchored_end_loop_end = NO_STATE; + ainfo->eager_match_state = NO_STATE; + + if (!fsm_getstart(nfa, &ainfo->start)) { + return false; + } + + const size_t state_count = fsm_countstates(nfa); + assert(ainfo->start < state_count); + + { + const struct fsm_state *s = &nfa->states[ainfo->start]; + + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (ns_i == ainfo->start) { continue; } + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + assert(ns_i < state_count); + const struct fsm_state *ns = &nfa->states[ns_i]; + + /* if there's a state in the start state's epsilon closure that + * has a dot self-edge, it's the unanchored start loop */ + if (has_dot_self_edge(nfa, ns_i)) { + assert(ainfo->unanchored_start_loop == NO_STATE); + ainfo->unanchored_start_loop = ns_i; + + /* copy its non-self labeled edges to ainfo->repeatable_firsts */ + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != ns_i) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } else { + /* likewise, a state without a dot self-edge is the anchored start */ + assert(ainfo->anchored_start == NO_STATE); + ainfo->anchored_start = ns_i; + + /* copy its labeled edges to ainfo->anchored_firsts */ + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (!edge_set_add_bulk(&ainfo->anchored_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } + } + + /* If the start state always matches, set a flag noting that it will need special handling + * later. It's arguably pointless to combine "" with other regexes, because it will always + * trivially match, but otherwise it would never match. */ + if (!start_state_epsilon_closure_matches_empty_string(nfa, ainfo->start, &ainfo->nullable)) { + goto alloc_fail; + } + + /* If there's a state with a dot self-edge and an epsilon edge to an end state, it's + * the unanchored end loop. There should only be one. */ + for (size_t s_i = 0; s_i < state_count; s_i++) { + const struct fsm_state *s = &nfa->states[s_i]; + if (has_dot_self_edge(nfa, s_i)) { + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (fsm_isend(nfa, ns_i)) { + assert(ainfo->unanchored_end_loop == NO_STATE); + ainfo->unanchored_end_loop = s_i; + ainfo->unanchored_end_loop_end = ns_i; + break; + } + } + if (ainfo->unanchored_end_loop != NO_STATE) { break; } + } + } + + /* Collect states that lead to an anchored end or eager match. */ + for (size_t s_i = 0; s_i < state_count; s_i++) { + fsm_state_t dst_end = NO_STATE; + if (has_epsilon_and_newline_edges_to_end(nfa, s_i, &dst_end)) { + if (!state_set_add(&ainfo->anchored_ends, nfa->alloc, dst_end)) { + goto alloc_fail; + } + } + + if (has_labeled_edge_to_unanchored_end_loop(nfa, s_i, ainfo->unanchored_end_loop)) { + if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { + goto alloc_fail; + } + } + } -/* Combine an array of FSMs into a single FSM in one pass, with an extra loop - * so that more than one pattern with eager outputs can match. */ +#if LOG_ANALYZE_GROUP_NFA_RESULTS + { + fprintf(stderr, "# analysis_info start %d, usl %d, uel %d, uele %d\n", + ainfo->start, ainfo->unanchored_start_loop, ainfo->unanchored_end_loop, ainfo->unanchored_end_loop_end); + dump_state_set(stderr, "anchored_ends", ainfo->anchored_ends); + dump_state_set(stderr, "eager_matches", ainfo->eager_matches); + dump_edge_set(stderr, "anchored_firsts", ainfo->anchored_start, ainfo->anchored_firsts); + dump_edge_set(stderr, "repeatable_firsts", ainfo->unanchored_start_loop, ainfo->repeatable_firsts); + } +#endif + return true; + +alloc_fail: + fprintf(stderr, "alloc fail\n"); + return false; +} + +/* Replace any labeled edges on nfa->states[from_state] going to old_to + * with a new edge leading to new_to. There currently isn't a function in + * the libfsm API for this (and it shouldn't be necessary in general), but + * if it gets one later this can be replaced. */ +static bool +replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to) +{ + if (old_to == NO_STATE) { + /* nothing to do */ + return true; + } + assert(new_to < nfa->statecount); + assert(from_state < nfa->statecount); + + struct fsm_state *from = &nfa->states[from_state]; + struct edge_set *old_edges = from->edges; + struct edge_set *new_edges = edge_set_new(); + + /* copy, replacing edges to old_to */ + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(old_edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (!edge_set_add_bulk(&new_edges, nfa->alloc, info.symbols, + info.to == old_to ? new_to : info.to)) { + return false; + } + } + + edge_set_free(nfa->alloc, old_edges); + from->edges = new_edges; + return true; +} + +/* Make a couple changes to the group NFA so that it can be combined correctly: + * + * - If the group NFA has an unanchored_end_loop, add a new state, + * eager_match_state, which will be a waypoint between edges that previously + * led to the unanchored_end_loop and the global NFA's global_unanchored_end_loop + * (so it can potentially also match other group NFAs with unanchored starts). + * This state will get an eager output ID. + * + * - Set an end ID on every anchored end state, so halting on these counts as a match. */ +static bool +modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t id_base) +{ + const bool nullable_and_unanchored_end = ainfo->nullable + && ainfo->unanchored_end_loop != NO_STATE; + + /* Add the eager match state if there are eager match states + * or a nullable unanchored end. This will link to the global NFA's + * unanchored_end_loop. */ + if (!state_set_empty(ainfo->eager_matches) || nullable_and_unanchored_end) { + if (!fsm_addstate(nfa, &ainfo->eager_match_state)) { + return false; + } + + /* Set eager match ID on new eager_match_state. */ + const fsm_output_id_t oid = (fsm_output_id_t)(id + id_base); + if (!fsm_seteageroutput(nfa, ainfo->eager_match_state, oid)) { + return false; + } + + /* For every state in eager_matches, replace every edge leading to + * the unanchored_end_loop with an edge with the same labels to + * eager_match_state. */ + struct state_iter si; + state_set_reset(ainfo->eager_matches, &si); + fsm_state_t ems_i; + while (state_set_next(&si, &ems_i)) { + if (!replace_labeled_edge(nfa, ems_i, + ainfo->unanchored_end_loop, ainfo->eager_match_state)) { + return false; + } + + /* The state must not link to the unanchored end loop anymore. + * Doing so will cause a combinatorial explosion that makes + * combining more ~10 NFAs incredibly expensive. */ + struct edge_group_iter iter; + struct edge_group_iter_info info; + assert(ems_i < nfa->statecount); + const struct fsm_state *ems = &nfa->states[ems_i]; + edge_set_group_iter_reset(ems->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to != ainfo->unanchored_end_loop); + } + } + } + + /* If the group NFA matches the empty string and has an unanchored end, then + * link its unanchored start state to the eager match state. This ensures + * all inputs will match this group NFA when combined. */ + if (nullable_and_unanchored_end) { + assert(ainfo->start != NO_STATE); + assert(ainfo->eager_match_state != NO_STATE); + + if (ainfo->unanchored_start_loop != NO_STATE) { + struct fsm_state *s = &nfa->states[ainfo->unanchored_start_loop]; + if (!state_set_add(&s->epsilons, nfa->alloc, ainfo->eager_match_state)) { + return false; + } + } + } + + /* If there are anchored ends, set an endid on them */ + if (!state_set_empty(ainfo->anchored_ends)) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t anchored_end_state; + const fsm_end_id_t end_id = (fsm_end_id_t)(id + id_base); + while (state_set_next(&si, &anchored_end_state)) { + if (!fsm_endid_set(nfa, anchored_end_state, end_id)) { + return false; + } + } + } + + return true; +} + +static void +rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) +{ + if (base == 0) { return; } + +#define SHIFT(S) if (ainfo-> S != NO_STATE) { ainfo-> S += base; } + SHIFT(start); + SHIFT(unanchored_start_loop); + SHIFT(unanchored_end_loop); + SHIFT(unanchored_end_loop_end); + SHIFT(anchored_start); + SHIFT(eager_match_state); +#undef SHIFT + + state_set_rebase(&ainfo->anchored_ends, base); + state_set_rebase(&ainfo->eager_matches, base); + + edge_set_rebase(&ainfo->anchored_firsts, base); + edge_set_rebase(&ainfo->repeatable_firsts, base); + +} + +static void +free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) +{ + state_set_free(ainfo->anchored_ends); + state_set_free(ainfo->eager_matches); + edge_set_free(alloc, ainfo->anchored_firsts); + edge_set_free(alloc, ainfo->repeatable_firsts); +} + +/* Combine an array of FSMs into a single FSM that attempts to match them + * all in one pass, with an extra loop so that more than one pattern with + * eager outputs can match. */ struct fsm * fsm_union_repeated_pattern_group(size_t entry_count, struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases) @@ -168,9 +644,11 @@ fsm_union_repeated_pattern_group(size_t entry_count, const struct fsm_alloc *alloc = entries[0].fsm->alloc; const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; - if (entry_count == 1) { - return entries[0].fsm; - } + /* TODO: make this an extra argument */ + const size_t id_base = 1; + + struct analysis_info *ainfos = f_calloc(alloc, entry_count, sizeof(ainfos[0])); + if (ainfos == NULL) { goto fail; } size_t est_total_states = 0; for (size_t i = 0; i < entry_count; i++) { @@ -183,205 +661,200 @@ fsm_union_repeated_pattern_group(size_t entry_count, est_total_states += count; } + for (size_t i = 0; i < entry_count; i++) { + struct fsm *fsm = entries[i].fsm; + + /* Identify various states in the NFA that will be relevant to combining. */ + if (!analyze_group_nfa(fsm, &ainfos[i])) { + goto fail; + } + + /* Change the NFA structure so it can better link into the combined FSM, + * and set endids and/or output IDs as appropriate. */ + if (!modify_group_nfa(fsm, i, &ainfos[i], id_base)) { + goto fail; + } + } + est_total_states += 5; /* new start and end, new unanchored start and end loops */ struct fsm *res = fsm_new_statealloc(alloc, est_total_states); if (res == NULL) { return NULL; } - /* collected end states */ - struct ends_buf { - size_t ceil; - size_t used; - fsm_state_t *states; - } ends = { .ceil = 0 }; - - /* The new overall start state, which will have an epsilon edge to... */ + /* The new overall start state */ fsm_state_t global_start; if (!fsm_addstate(res, &global_start)) { goto fail; } - /* states linking to the starts of unanchored and anchored subgraphs, respectively. */ - fsm_state_t global_start_loop, global_start_anchored; - if (!fsm_addstate(res, &global_start_loop)) { goto fail; } - if (!fsm_addstate(res, &global_start_anchored)) { goto fail; } + /* States linking to the starts of unanchored and anchored subgraphs, respectively. + * Matching other group NFAs loops back to the global_unanchored_start_loop, but + * patterns anchored at the ^start are only reachable via global_anchored_start. */ + fsm_state_t global_unanchored_start_loop, global_anchored_start; + if (!fsm_addstate(res, &global_unanchored_start_loop)) { goto fail; } + if (!fsm_addstate(res, &global_anchored_start)) { goto fail; } /* The unanchored end loop state, and an end state with no outgoing edges. */ - fsm_state_t global_end_loop, global_end; + fsm_state_t global_unanchored_end_loop, global_end; if (!fsm_addstate(res, &global_end)) { goto fail; } - if (!fsm_addstate(res, &global_end_loop)) { goto fail; } + if (!fsm_addstate(res, &global_unanchored_end_loop)) { goto fail; } - /* link the start to the start loop and anchored start, and the start loop to itself */ + /* link the start to the global unanchored start loop and anchored start. */ if (log) { - fprintf(stderr, "link_before: global_start %d -> global_start_loop %d and global_start_anchored %d\n", - global_start, global_start_loop, global_start_anchored); + fprintf(stderr, "link_before: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", + global_start, global_unanchored_start_loop, global_anchored_start); } - if (!fsm_addedge_epsilon(res, global_start, global_start_loop)) { goto fail; } - if (!fsm_addedge_epsilon(res, global_start, global_start_anchored)) { goto fail; } - if (!fsm_addedge_any(res, global_start_loop, global_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } - /* link the end loop and end */ + /* Link the global unanchored start loop to itself. */ + if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop and global end. */ if (log) { - fprintf(stderr, "link_before: global_end_loop %d -> global_end %d (and -> self)\n", global_end_loop, global_end); + fprintf(stderr, "link_before: global_unanchored_end_loop %d -> global_end %d (and -> self)\n", global_unanchored_end_loop, global_end); } - if (!fsm_addedge_epsilon(res, global_end_loop, global_end)) { goto fail; } - if (!fsm_addedge_any(res, global_end_loop, global_end_loop)) { goto fail; } + if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } if (bases != NULL) { memset(bases, 0x00, entry_count * sizeof(bases[0])); } + /* For each group FSM, link its unanchored and anchored start states + * and eager_match_state to the global ones. */ for (size_t fsm_i = 0; fsm_i < entry_count; fsm_i++) { - ends.used = 0; /* reset */ - struct fsm *fsm = entries[fsm_i].fsm; entries[fsm_i].fsm = NULL; /* transfer ownership */ const size_t state_count = fsm_countstates(fsm); - - fsm_state_t fsm_start; - if (!fsm_getstart(fsm, &fsm_start)) { + struct analysis_info *ainfo = &ainfos[fsm_i]; + if (ainfo->start == NO_STATE) { fsm_free(fsm); /* no start, just discard */ continue; } + assert(ainfo->start < state_count); - for (fsm_state_t s_i = 0; s_i < state_count; s_i++) { - if (fsm_isend(fsm, s_i)) { - if (ends.used == ends.ceil) { /* grow? */ - size_t nceil = (ends.ceil == 0 ? 4 : 2*ends.ceil); - fsm_state_t *nstates = f_realloc(alloc, - ends.states, nceil * sizeof(nstates[0])); - if (nstates == NULL) { goto fail; } - ends.ceil = nceil; - ends.states = nstates; - } - ends.states[ends.used++] = s_i; - } - } - - if (ends.used == 0) { - fsm_free(fsm); /* no ends, just discard */ - continue; - } - - /* When combining these, remove self-edges from any states on the FSMs to be - * combined that also have eager output IDs. We are about to add an epsilon edge - * from each to a shared state that won't have eager output IDs. - * - * Eager output matching should be idempotent, so carrying it to other reachable - * state is redundant, and it leads to a combinatorial explosion that blows up the - * state count while determinising the combined FSM otherwise. - * - * For example, if /aaa/, /bbb/, and /ccc/ are combined into a DFA that repeats - * the sub-patterns (like `^.*(?:(aaa)|(bbb)|(ccc))+.*$`), the self-edge at each - * eager output state would combine with every reachable state from then on, - * leading to a copy of the whole reachable subgraph colored by every - * combination of eager output IDs: aaa, bbb, ccc, aaa+bbb, aaa+ccc, - * bbb+ccc, aaa+bbb+ccc. Instead of three relatively separate subgraphs - * that set the eager output at their last state, one for each pattern, - * it leads to 8 (2**3) subgraph clusters because it encodes _each - * distinct combination_ in the DFA. This becomes incredibly expensive - * as the combined pattern count increases; it's essentially what I'm - * trying to avoid by adding eager output support in the first place. - * - * FIXME: instead of actively removing these, filter in fsm_determinise? */ - if (fsm_eager_output_has_eager_output(fsm)) { - /* for any state that has eager outputs and a self edge, - * remove the self edge before further linkage */ - for (fsm_state_t s = 0; s < fsm->statecount; s++) { - const size_t eager_output_count = fsm_eager_output_count(fsm, s); - if (eager_output_count == 0) { continue; } - struct edge_set *edges = fsm->states[s].edges; - struct edge_set *new = edge_set_new(); - - struct edge_group_iter iter; - struct edge_group_iter_info info; - edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - if (info.to != s) { - if (!edge_set_add_bulk(&new, fsm->alloc, - info.symbols, info.to)) { - goto fail; - } - } - } - edge_set_free(fsm->alloc, edges); - fsm->states[s].edges = new; - } - } - - /* call fsm_merge; we really don't care which is which */ + /* Call fsm_merge; we really don't care which is which. */ struct fsm_combine_info combine_info; struct fsm *merged = fsm_merge(res, fsm, &combine_info); if (merged == NULL) { goto fail; } - /* update offsets if res had its state IDs shifted forward */ + /* Update offsets if res had its state IDs shifted forward. */ global_start += combine_info.base_a; - global_start_loop += combine_info.base_a; - global_start_anchored += combine_info.base_a;; + global_unanchored_start_loop += combine_info.base_a; + global_anchored_start += combine_info.base_a; global_end += combine_info.base_a; - global_end_loop += combine_info.base_a; + global_unanchored_end_loop += combine_info.base_a; - /* also update offsets for the FSM's states */ - fsm_start += combine_info.base_b; - for (size_t i = 0; i < ends.used; i++) { - ends.states[i] += combine_info.base_b; - } + /* Also update offsets for the group FSM's states. */ + rebase_analysis_info(ainfo, combine_info.base_b); if (bases != NULL) { bases[fsm_i].state = combine_info.base_b; bases[fsm_i].capture = combine_info.capture_base_b; } - if (log) { - fprintf(stderr, "%s: fsm[%zd].start: %d\n", __func__, fsm_i, fsm_start); - for (size_t i = 0; i < ends.used; i++) { - fprintf(stderr, "%s: fsm[%zd].ends[%zd]: %d\n", __func__, fsm_i, i, ends.states[i]); + /* Link the FSM's eager match state back to the global_unanchored_end_loop, so that after + * matching it in an unanchored way it can continue attempting to match other combined + * patterns that aren't anchored at their start. Also link it to the global end, so + * it will be retained during determinisation and minimisation. */ + if (ainfo->eager_match_state != NO_STATE) { + if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_unanchored_end_loop)) { + goto fail; + } + if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_end)) { + goto fail; + } + + /* If the NFA matches the empty string and is not anchored at the end, then + * add an epsilon edge from the global start directly to its eager match state. + * This ensures all inputs will match this group NFA when combined. */ + if (ainfo->nullable) { + assert(ainfo->unanchored_end_loop != NO_STATE); + if (!fsm_addedge_epsilon(merged, global_start, ainfo->eager_match_state)) { + goto fail; + } + } + } else { + /* If the NFA matches an end-anchored empty string, then add an epsilon edge from + * the global start to an anchored end, which has an endid. */ + if (ainfo->nullable && !state_set_empty(ainfo->anchored_ends)) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t anchored_end_state; + + while (state_set_next(&si, &anchored_end_state)) { + if (!fsm_addedge_epsilon(merged, global_start, anchored_end_state)) { + goto fail; + } + /* It should only be necessary to link one, since that's enough + * for determinisation to carry the end id back to the start's + * epsilon closure. */ + break; + } } } - /* link to the FSM's start state */ - const fsm_state_t start_src = entries[fsm_i].anchored_start ? global_start_anchored : global_start_loop; - if (!fsm_addedge_epsilon(merged, start_src, fsm_start)) { goto fail; } - if (log) { - fprintf(stderr, "%s: linking %s %d to fsm[%zd]'s start %d (anchored? %d)\n", - __func__, - entries[fsm_i].anchored_start ? "global_start_anchored" : "global_start_loop", - start_src, fsm_i, fsm_start, entries[fsm_i].anchored_start); + struct edge_group_iter iter; + struct edge_group_iter_info info; + + /* Link the global_anchored_start to group FSM paths that are start-anchored + * and can only match at the start of input. */ + edge_set_group_iter_reset(ainfo->anchored_firsts, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(global_anchored_start < merged->statecount); + struct fsm_state *anchored_start = &merged->states[global_anchored_start]; + if (!edge_set_add_bulk(&anchored_start->edges, merged->alloc, + info.symbols, info.to)) { + goto fail; + } } - /* link from the FSM's ends */ - const fsm_state_t end_dst = entries[fsm_i].anchored_end ? global_end : global_end_loop; - for (size_t i = 0; i < ends.used; i++) { - if (log) { - fprintf(stderr, "%s: linking fsm[%zd]'s end[%zd] %d (anchored? %d) to %s %d\n", - __func__, fsm_i, i, ends.states[i], entries[fsm_i].anchored_end, - entries[fsm_i].anchored_end ? "global_end" : "global_end_loop", - end_dst); + /* Link the global_unanchored_start_loop to group FSM paths that aren't + * start-anchored. */ + edge_set_group_iter_reset(ainfo->repeatable_firsts, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + struct fsm_state *unanchored_start = &merged->states[global_unanchored_start_loop]; + if (!edge_set_add_bulk(&unanchored_start->edges, merged->alloc, + info.symbols, info.to)) { + goto fail; } - if (!fsm_addedge_epsilon(merged, ends.states[i], end_dst)) { goto fail; } } res = merged; } - /* Link from the global_end_loop to the global_start_loop, so patterns with an - * unanchored start can follow other patterns with an unanchored end. */ + /* Link from the global_unanchored_end_loop to the global_unanchored_start_loop, + * so patterns with an unanchored start can follow other patterns with an unanchored + * end, possibly with other ignored input between them. */ if (log) { fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", - __func__, global_start, global_start_loop, global_start_anchored, global_end_loop, global_end); - fprintf(stderr, "%s: linking global_end_loop %d to global_start_loop %d\n", - __func__, global_end_loop, global_start_loop); + __func__, global_start, global_unanchored_start_loop, global_anchored_start, global_unanchored_end_loop, global_end); + fprintf(stderr, "%s: linking global_unanchored_end_loop %d to global_unanchored_start_loop %d\n", + __func__, global_unanchored_end_loop, global_unanchored_start_loop); fprintf(stderr, "%s: setting global_start %d and end %d\n", __func__, global_start, global_end); } - if (!fsm_addedge_epsilon(res, global_end_loop, global_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } /* This needs to be set after merging, because that clears the start state. */ fsm_setstart(res, global_start); fsm_setend(res, global_end, 1); - f_free(alloc, ends.states); + for (size_t i = 0; i < entry_count; i++) { + free_analysis(alloc, &ainfos[i]); + } + + f_free(alloc, ainfos); + return res; fail: - f_free(alloc, ends.states); + if (ainfos != NULL) { + for (size_t i = 0; i < entry_count; i++) { + free_analysis(alloc, &ainfos[i]); + } + f_free(alloc, ainfos); + } + return NULL; } diff --git a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c new file mode 100644 index 000000000..06965e014 --- /dev/null +++ b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c @@ -0,0 +1,29 @@ +#include "utils.h" + +/* Test for false positive matches when combining patterns + * with both anchored and unanchored subtrees. */ + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "a(?:x|$)", + "a(?:y|$)", + }, + .inputs = { + { .input = "a", .expected_ids = { 1, 2 } }, + { .input = "aZ", .expect_fail = true }, + { .input = "Za", .expected_ids = { 1, 2 } }, + { .input = "ax", .expected_ids = { 1 } }, + { .input = "axZ", .expected_ids = { 1 } }, + { .input = "ay", .expected_ids = { 2 } }, + { .input = "ayZ", .expected_ids = { 2 } }, + { .input = "axa", .expected_ids = { 1, 2 } }, + { .input = "aya", .expected_ids = { 1, 2 } }, + { .input = "axay", .expected_ids = { 1, 2 } }, + { .input = "ayax", .expected_ids = { 1, 2 } }, + }, + }; + + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_at_start_multiple_anchored.c b/tests/eager_output/eager_output_at_start_multiple_anchored.c new file mode 100644 index 000000000..ce242472f --- /dev/null +++ b/tests/eager_output/eager_output_at_start_multiple_anchored.c @@ -0,0 +1,15 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + [0] = "$", + [1] = "^$", + }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test, false, false); +} diff --git a/tests/eager_output/eager_output_at_start_single_anchored.c b/tests/eager_output/eager_output_at_start_single_anchored.c new file mode 100644 index 000000000..e7ce16050 --- /dev/null +++ b/tests/eager_output/eager_output_at_start_single_anchored.c @@ -0,0 +1,22 @@ +#include "utils.h" + +int main(void) +{ + const struct eager_output_test test_unanchored_start = { + .patterns = { "$" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + + const struct eager_output_test test_anchored_start = { + .patterns = { "^$" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + + bool pass = run_test(&test_unanchored_start, false, false); + pass = run_test(&test_anchored_start, false, false) && pass; + return pass; +} diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c index 4bee8d848..f7eb23069 100644 --- a/tests/eager_output/utils.c +++ b/tests/eager_output/utils.c @@ -45,6 +45,8 @@ dump(const struct fsm *fsm) int run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids) { + (void)force_endids; /* TODO: unused, remove. */ + struct fsm_union_entry entries[MAX_PATTERNS] = {0}; allow_extra_outputs = false; @@ -82,45 +84,14 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); assert(fsm != NULL); - /* Zero is used to terminate expected_ids, so don't use it here. */ - const fsm_output_id_t output_id = (fsm_output_id_t) (i + 1); - const fsm_end_id_t end_id = (fsm_end_id_t) (i + 1); - - /* Set either an end ID or an eager output ID, depending on - * whether the fsm is anchored at the end or not. */ - if (e->anchored_end || force_endids) { - ret = fsm_setendid(fsm, end_id); - } else { - ret = fsm_seteageroutputonends(fsm, output_id); - } - assert(ret == 1); - if (log) { fprintf(stderr, "==== source DFA %zd (pre det+min)\n", i); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fsm_endid_dump(stderr, fsm); - fprintf(stderr, "====\n"); - } - - ret = fsm_determinise(fsm); - assert(ret == 1); - - if (log) { - fprintf(stderr, "==== source DFA %zd (post det)\n", i); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "====\n"); - } - - ret = fsm_minimise(fsm); - assert(ret == 1); - - if (log) { - fprintf(stderr, "==== source DFA %zd (post det+min)\n", i); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } e->fsm = fsm; @@ -133,11 +104,13 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo if (log) { fprintf(stderr, "==== combined (pre det+min)\n"); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "--- endids:\n"); - fsm_endid_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } if (log) { @@ -151,9 +124,11 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo if (log) { fprintf(stderr, "==== combined (post det)\n"); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } ret = fsm_minimise(fsm); @@ -164,11 +139,13 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo if (log) { fprintf(stderr, "==== combined (post det+min)\n"); - if (log > 1) { dump(fsm); } - fsm_eager_output_dump(stderr, fsm); - fprintf(stderr, "--- endids:\n"); - fsm_endid_dump(stderr, fsm); - fprintf(stderr, "====\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } } struct cb_info outputs = { 0 }; @@ -218,13 +195,22 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo /* Copy endid outputs into outputs.ids[], since for testing * purposes we don't care about the difference between eager - * output and endids here -- the values don't overlap. */ + * output and endids here. */ assert(outputs.used + endid_count <= MAX_IDS); for (size_t endid_i = 0; endid_i < endid_count; endid_i++) { if (log) { fprintf(stderr, "-- adding endid %zd: %d\n", endid_i, endid_buf[endid_i]); } - outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + bool found = false; + for (size_t o_i = 0; o_i < outputs.used; o_i++) { + if (outputs.ids[o_i] == endid_buf[endid_i]) { + found = true; + break; + } + } + if (!found) { + outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + } } } From af27a87c1d078465a8e22dda00a61c0dc00b26e7 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 4 Feb 2025 13:57:35 -0500 Subject: [PATCH 319/373] fsm_union_repeated_pattern_group: Interface changes. - Instead of taking an array of `struct fsm_union_array *` pointers, this now takes an array of `struct fsm *` pointers. The other fields on `fsm_union_array` are no longer used, so the extra struct layer has been removed. - This now takes an extra argument, id_base, because each nfa[i] will get end IDs and/or output IDs (i + id_base) set on them. Previously these were set by the caller. - Rename parameters, to emphasize that the FSMs must be NFAs. - Update the test code for the interface changes. - Remove flags from the test code that are no longer used. --- include/fsm/bool.h | 23 ++++++---- src/libfsm/union.c | 43 +++++++++---------- tests/eager_output/eager_output1.c | 2 +- tests/eager_output/eager_output2.c | 2 +- tests/eager_output/eager_output3.c | 2 +- tests/eager_output/eager_output4.c | 2 +- tests/eager_output/eager_output5.c | 2 +- tests/eager_output/eager_output6.c | 2 +- tests/eager_output/eager_output7.c | 30 +------------ ...utput_alt_mixing_anchored_and_unanchored.c | 2 +- tests/eager_output/eager_output_at_start.c | 2 +- .../eager_output_at_start_multiple.c | 15 +++++++ .../eager_output_at_start_multiple_anchored.c | 2 +- .../eager_output_at_start_single_anchored.c | 4 +- tests/eager_output/eager_output_fr1.c | 2 +- tests/eager_output/eager_output_fr2.c | 2 +- tests/eager_output/eager_output_fr3.c | 2 +- .../eager_output_mixed_anchored_unanchored.c | 2 +- tests/eager_output/utils.c | 38 ++++------------ tests/eager_output/utils.h | 2 +- 20 files changed, 76 insertions(+), 105 deletions(-) create mode 100644 tests/eager_output/eager_output_at_start_multiple.c diff --git a/include/fsm/bool.h b/include/fsm/bool.h index 4d9f1889a..c2c2d80ed 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -52,15 +52,22 @@ struct fsm * fsm_union_array(size_t fsm_count, struct fsm **fsms, struct fsm_combined_base_pair *bases); -struct fsm_union_entry { - struct fsm *fsm; - bool anchored_start; - bool anchored_end; -}; - +/* Combine an array of NFAs into a single NFA that attempts to match them + * all in one pass, with an extra loop so that more than one pattern with + * eager outputs can match. Ownership of the NFAs is transferred, they will + * be combined (or freed, if they don't have a start state). + * + * This MUST be called with NFAs constructed via re_comp, Calling it with + * manually constructed NFAs or DFAs is unsupported. + * + * This will set end IDs and/or output IDs representing matching each + * of the original NFAs on the combined result, where nfas[i] will + * get ID of (id_base + i). + * + * Returns NULL on error. */ struct fsm * -fsm_union_repeated_pattern_group(size_t entry_count, - struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases); +fsm_union_repeated_pattern_group(size_t nfa_count, + struct fsm **nfas, struct fsm_combined_base_pair *bases, size_t id_base); struct fsm * fsm_intersect(struct fsm *a, struct fsm *b); diff --git a/src/libfsm/union.c b/src/libfsm/union.c index ed71384ff..f381d7cd6 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -638,31 +638,28 @@ free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) * all in one pass, with an extra loop so that more than one pattern with * eager outputs can match. */ struct fsm * -fsm_union_repeated_pattern_group(size_t entry_count, - struct fsm_union_entry *entries, struct fsm_combined_base_pair *bases) +fsm_union_repeated_pattern_group(size_t nfa_count, + struct fsm **nfas, struct fsm_combined_base_pair *bases, size_t id_base) { - const struct fsm_alloc *alloc = entries[0].fsm->alloc; + const struct fsm_alloc *alloc = nfas[0]->alloc; const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; - /* TODO: make this an extra argument */ - const size_t id_base = 1; - - struct analysis_info *ainfos = f_calloc(alloc, entry_count, sizeof(ainfos[0])); + struct analysis_info *ainfos = f_calloc(alloc, nfa_count, sizeof(ainfos[0])); if (ainfos == NULL) { goto fail; } size_t est_total_states = 0; - for (size_t i = 0; i < entry_count; i++) { - assert(entries[i].fsm); - if (entries[i].fsm->alloc != alloc) { + for (size_t i = 0; i < nfa_count; i++) { + assert(nfas[i]); + if (nfas[i]->alloc != alloc) { errno = EINVAL; return NULL; } - const size_t count = fsm_countstates(entries[i].fsm); + const size_t count = fsm_countstates(nfas[i]); est_total_states += count; } - for (size_t i = 0; i < entry_count; i++) { - struct fsm *fsm = entries[i].fsm; + for (size_t i = 0; i < nfa_count; i++) { + struct fsm *fsm = nfas[i]; /* Identify various states in the NFA that will be relevant to combining. */ if (!analyze_group_nfa(fsm, &ainfos[i])) { @@ -716,17 +713,17 @@ fsm_union_repeated_pattern_group(size_t entry_count, if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } if (bases != NULL) { - memset(bases, 0x00, entry_count * sizeof(bases[0])); + memset(bases, 0x00, nfa_count * sizeof(bases[0])); } - /* For each group FSM, link its unanchored and anchored start states + /* For each group NFA, link its unanchored and anchored start states * and eager_match_state to the global ones. */ - for (size_t fsm_i = 0; fsm_i < entry_count; fsm_i++) { - struct fsm *fsm = entries[fsm_i].fsm; - entries[fsm_i].fsm = NULL; /* transfer ownership */ + for (size_t nfa_i = 0; nfa_i < nfa_count; nfa_i++) { + struct fsm *fsm = nfas[nfa_i]; + nfas[nfa_i] = NULL; /* transfer ownership */ const size_t state_count = fsm_countstates(fsm); - struct analysis_info *ainfo = &ainfos[fsm_i]; + struct analysis_info *ainfo = &ainfos[nfa_i]; if (ainfo->start == NO_STATE) { fsm_free(fsm); /* no start, just discard */ continue; @@ -749,8 +746,8 @@ fsm_union_repeated_pattern_group(size_t entry_count, rebase_analysis_info(ainfo, combine_info.base_b); if (bases != NULL) { - bases[fsm_i].state = combine_info.base_b; - bases[fsm_i].capture = combine_info.capture_base_b; + bases[nfa_i].state = combine_info.base_b; + bases[nfa_i].capture = combine_info.capture_base_b; } /* Link the FSM's eager match state back to the global_unanchored_end_loop, so that after @@ -840,7 +837,7 @@ fsm_union_repeated_pattern_group(size_t entry_count, fsm_setstart(res, global_start); fsm_setend(res, global_end, 1); - for (size_t i = 0; i < entry_count; i++) { + for (size_t i = 0; i < nfa_count; i++) { free_analysis(alloc, &ainfos[i]); } @@ -850,7 +847,7 @@ fsm_union_repeated_pattern_group(size_t entry_count, fail: if (ainfos != NULL) { - for (size_t i = 0; i < entry_count; i++) { + for (size_t i = 0; i < nfa_count; i++) { free_analysis(alloc, &ainfos[i]); } f_free(alloc, ainfos); diff --git a/tests/eager_output/eager_output1.c b/tests/eager_output/eager_output1.c index f20ef77b7..4900f89e0 100644 --- a/tests/eager_output/eager_output1.c +++ b/tests/eager_output/eager_output1.c @@ -8,5 +8,5 @@ int main(void) { .input = "abc", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output2.c b/tests/eager_output/eager_output2.c index cdac204e2..6a10eec1c 100644 --- a/tests/eager_output/eager_output2.c +++ b/tests/eager_output/eager_output2.c @@ -13,5 +13,5 @@ int main(void) { .input = "XabeX", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output3.c b/tests/eager_output/eager_output3.c index c11bc58a4..b6320ef79 100644 --- a/tests/eager_output/eager_output3.c +++ b/tests/eager_output/eager_output3.c @@ -12,5 +12,5 @@ int main(void) { .input = "abe", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output4.c b/tests/eager_output/eager_output4.c index 47cd32029..2e0f17f13 100644 --- a/tests/eager_output/eager_output4.c +++ b/tests/eager_output/eager_output4.c @@ -9,5 +9,5 @@ int main(void) { .input = "Xabcde", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output5.c b/tests/eager_output/eager_output5.c index 4551c68b1..6d2ce4eb8 100644 --- a/tests/eager_output/eager_output5.c +++ b/tests/eager_output/eager_output5.c @@ -10,5 +10,5 @@ int main(void) { .input = "abbc", .expected_ids = { 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output6.c b/tests/eager_output/eager_output6.c index 5431d0981..188541f39 100644 --- a/tests/eager_output/eager_output6.c +++ b/tests/eager_output/eager_output6.c @@ -30,5 +30,5 @@ int main(void) }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output7.c b/tests/eager_output/eager_output7.c index 3d123878b..94e9f1787 100644 --- a/tests/eager_output/eager_output7.c +++ b/tests/eager_output/eager_output7.c @@ -2,24 +2,6 @@ int main(void) { - /* Run this test with env FORCE_ENDIDS=N ... to see how much more - * expensive it is to combine the first N patterns using endids, - * rather than eager_outputs. It becomes VERY slow for >= 9 or so. - * (Note that the checks probably will not pass for N < 4, because - * it will start skipping appear in the early test inputs.) */ - bool force_endids = false; - size_t force_endid_count = 0; - { - const char *str = getenv("FORCE_ENDIDS"); - if (str != NULL) { - force_endid_count = atoi(str); - if (force_endid_count == 0) { - force_endid_count = 26; - } - force_endids = true; - } - } - struct eager_output_test test = { .patterns = { [0] = "apple", @@ -89,15 +71,5 @@ int main(void) }, }; - /* truncate patterns to the first N */ - if (force_endids) { - assert(force_endid_count > 0 && force_endid_count <= 26); - test.patterns[force_endid_count] = NULL; - - /* truncate test inputs to just the first couple, since - * later inputs use later patterns */ - test.inputs[5].input = NULL; - } - - return run_test(&test, false, force_endids); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c index 06965e014..ad125b9b1 100644 --- a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c +++ b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c @@ -25,5 +25,5 @@ int main(void) }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start.c b/tests/eager_output/eager_output_at_start.c index 407aa4e77..8ba5f2ad1 100644 --- a/tests/eager_output/eager_output_at_start.c +++ b/tests/eager_output/eager_output_at_start.c @@ -8,5 +8,5 @@ int main(void) { .input = "", .expected_ids = { 1 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start_multiple.c b/tests/eager_output/eager_output_at_start_multiple.c new file mode 100644 index 000000000..ddc9530f5 --- /dev/null +++ b/tests/eager_output/eager_output_at_start_multiple.c @@ -0,0 +1,15 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + [0] = "", + [1] = "", + }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_at_start_multiple_anchored.c b/tests/eager_output/eager_output_at_start_multiple_anchored.c index ce242472f..0cf7e9b70 100644 --- a/tests/eager_output/eager_output_at_start_multiple_anchored.c +++ b/tests/eager_output/eager_output_at_start_multiple_anchored.c @@ -11,5 +11,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_at_start_single_anchored.c b/tests/eager_output/eager_output_at_start_single_anchored.c index e7ce16050..a9c13ef67 100644 --- a/tests/eager_output/eager_output_at_start_single_anchored.c +++ b/tests/eager_output/eager_output_at_start_single_anchored.c @@ -16,7 +16,7 @@ int main(void) }, }; - bool pass = run_test(&test_unanchored_start, false, false); - pass = run_test(&test_anchored_start, false, false) && pass; + bool pass = run_test(&test_unanchored_start); + pass = run_test(&test_anchored_start) && pass; return pass; } diff --git a/tests/eager_output/eager_output_fr1.c b/tests/eager_output/eager_output_fr1.c index e8e5f3395..97eb34312 100644 --- a/tests/eager_output/eager_output_fr1.c +++ b/tests/eager_output/eager_output_fr1.c @@ -9,5 +9,5 @@ int main(void) { .input = "ab", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_fr2.c b/tests/eager_output/eager_output_fr2.c index 404e98644..23bd3103c 100644 --- a/tests/eager_output/eager_output_fr2.c +++ b/tests/eager_output/eager_output_fr2.c @@ -9,5 +9,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_fr3.c b/tests/eager_output/eager_output_fr3.c index c7e4127a6..0d15a4a68 100644 --- a/tests/eager_output/eager_output_fr3.c +++ b/tests/eager_output/eager_output_fr3.c @@ -9,5 +9,5 @@ int main(void) { .input = "", .expected_ids = { 1, 2 } }, }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/eager_output_mixed_anchored_unanchored.c b/tests/eager_output/eager_output_mixed_anchored_unanchored.c index a586f9840..7afb272db 100644 --- a/tests/eager_output/eager_output_mixed_anchored_unanchored.c +++ b/tests/eager_output/eager_output_mixed_anchored_unanchored.c @@ -42,5 +42,5 @@ int main(void) }, }; - return run_test(&test, false, false); + return run_test(&test); } diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c index f7eb23069..dfd2b952b 100644 --- a/tests/eager_output/utils.c +++ b/tests/eager_output/utils.c @@ -43,15 +43,11 @@ dump(const struct fsm *fsm) } int -run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids) +run_test(const struct eager_output_test *test) { - (void)force_endids; /* TODO: unused, remove. */ + struct fsm *nfas[MAX_PATTERNS] = {0}; - struct fsm_union_entry entries[MAX_PATTERNS] = {0}; - - allow_extra_outputs = false; - - size_t fsms_used = 0; + size_t nfas_used = 0; int ret = 0; int log = 0; @@ -68,24 +64,12 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo for (size_t i = 0; i < MAX_PATTERNS; i++) { const char *p = test->patterns[i]; if (test->patterns[i] == NULL) { break; } - const size_t len = strlen(p); - struct fsm_union_entry *e = &entries[fsms_used]; - - /* For sake of these patterns, they are anchored if the first/last - * character is '^' and '$', respectively. This is too simplistic - * for the general case, though. */ - if (len > 0) { - if (p[0] == '^') { e->anchored_start = true; } - if (p[len - 1] == '$') { e->anchored_end = true; } - /* fprintf(stderr, "%s: p[%zd]: '%s', start %d, end %d\n", */ - /* __func__, fsms_used, p, e->anchored_start, e->anchored_end); */ - } struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); assert(fsm != NULL); if (log) { - fprintf(stderr, "==== source DFA %zd (pre det+min)\n", i); + fprintf(stderr, "==== source NFA %zd\n", i); if (log > 1) { dump(fsm); fsm_eager_output_dump(stderr, fsm); @@ -94,12 +78,12 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo } } - e->fsm = fsm; - fsms_used++; + nfas[i] = fsm; + nfas_used++; } - /* If there's only one pattern this just returns fsms[0]. */ - struct fsm *fsm = fsm_union_repeated_pattern_group(fsms_used, entries, NULL); + const size_t id_base = 1; /* offset by 1 because 0 is used as end-of-list */ + struct fsm *fsm = fsm_union_repeated_pattern_group(nfas_used, nfas, NULL, id_base); assert(fsm != NULL); if (log) { @@ -238,11 +222,7 @@ run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool fo assert(ret == 1); } - if (!allow_extra_outputs) { - assert(outputs.used == expected_id_count); - } else { - assert(outputs.used >= expected_id_count); - } + assert(outputs.used >= expected_id_count); size_t floor = 0; for (size_t exp_i = 0; exp_i < outputs.used; exp_i++) { diff --git a/tests/eager_output/utils.h b/tests/eager_output/utils.h index 672c01977..02f8427c9 100644 --- a/tests/eager_output/utils.h +++ b/tests/eager_output/utils.h @@ -48,7 +48,7 @@ int cmp_output(const void *pa, const void *pb); int -run_test(const struct eager_output_test *test, bool allow_extra_outputs, bool force_endids); +run_test(const struct eager_output_test *test); struct cb_info { size_t used; From d44a6710caed008004d8ea752211ef366fa7917b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 5 Feb 2025 15:35:44 -0500 Subject: [PATCH 320/373] fsm_union_repeated_pattern_group: fix linkage for mixed start anchoring. --- src/libfsm/union.c | 64 ++++++++++++++++--- ...tput_mixed_anchored_and_unanchored_start.c | 18 ++++++ 2 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index f381d7cd6..e3c7311ae 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -357,6 +357,56 @@ start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, fsm_stat return true; } +static bool +copy_noncycle_labeled_edges_from_epsilon_closure_iter(struct analysis_info *ainfo, + const struct fsm *nfa, fsm_state_t usl_i, fsm_state_t s_i, struct state_set **seen) +{ + const struct fsm_state *s = &nfa->states[s_i]; + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + /* copy its labeled edges to ainfo->repeatable_firsts, + * unless they lead back to the unanchored_start_loop */ + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != usl_i) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, + nfa->alloc, info.symbols, info.to)) { + return false; + } + } + } + + if (!state_set_add(seen, nfa->alloc, usl_i)) { return false; } + + struct state_iter si; + state_set_reset(s->epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (state_set_contains(*seen, ns_i)) { continue; } + if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, usl_i, ns_i, seen)) { + return false; + } + } + + return true; +} + +static bool +copy_noncycle_labeled_edges_from_epsilon_closure(struct analysis_info *ainfo, + const struct fsm *nfa, fsm_state_t unanchored_start_loop_id) +{ + struct state_set *seen = NULL; /* empty set */ + if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, + unanchored_start_loop_id, unanchored_start_loop_id, &seen)) { + return false; + } + state_set_free(seen); + + return true; +} + static bool analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) { @@ -396,15 +446,11 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) assert(ainfo->unanchored_start_loop == NO_STATE); ainfo->unanchored_start_loop = ns_i; - /* copy its non-self labeled edges to ainfo->repeatable_firsts */ - edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); - while (edge_set_group_iter_next(&egi, &info)) { - if (info.to != ns_i) { - if (!edge_set_add_bulk(&ainfo->repeatable_firsts, - nfa->alloc, info.symbols, info.to)) { - goto alloc_fail; - } - } + /* Copy labeled edges from the unanchored start loop and its epsilon + * closure to ainfo->repeatable_firsts, except for edges leading back + * to the unanchored start loop. */ + if (!copy_noncycle_labeled_edges_from_epsilon_closure(ainfo, nfa, ns_i)) { + goto alloc_fail; } } else { /* likewise, a state without a dot self-edge is the anchored start */ diff --git a/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c b/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c new file mode 100644 index 000000000..d18a67de7 --- /dev/null +++ b/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c @@ -0,0 +1,18 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "(^|[^A-Z])abc", + }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "xabc", .expected_ids = { 1 } }, + { .input = "xyz abc", .expected_ids = { 1 } }, + { .input = "Xabc", .expect_fail = true }, + }, + }; + + return run_test(&test); +} From 36129d079d3e52546fcb7e2020118d0c893ec09c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 11:38:10 -0500 Subject: [PATCH 321/373] Add tests, fix anchoring bugs in fsm_union_repeated_pattern_group. There are two bugs captured in eager_output_unanchored_end_plus.c: - Regexes ending in '+' weren't combining correctly, because analysis wasn't properly handling the construction for matching but optionally repeating the last character. - Eager matching after consuming a single character from the start state wasn't linked correctly to the global_unanchored_start_loop, so while the labeled edges were copied the eager output was lost. The other test files are focused on variants of that -- the + and start cases individually, and when + precedes a `()` subtree with more than one character. --- src/libfsm/union.c | 474 ++++++++++++------ .../eager_output_unanchored_end_plus.c | 19 + .../eager_output_unanchored_end_plus_min.c | 18 + .../eager_output_unanchored_end_plus_min2.c | 19 + ...ger_output_unanchored_end_plus_multichar.c | 19 + 5 files changed, 394 insertions(+), 155 deletions(-) create mode 100644 tests/eager_output/eager_output_unanchored_end_plus.c create mode 100644 tests/eager_output/eager_output_unanchored_end_plus_min.c create mode 100644 tests/eager_output/eager_output_unanchored_end_plus_min2.c create mode 100644 tests/eager_output/eager_output_unanchored_end_plus_multichar.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index e3c7311ae..9acdddf01 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -27,8 +27,11 @@ #include "endids.h" #define LOG_UNION_ARRAY 0 -#define LOG_ANALYZE_GROUP_NFA_RESULTS 0 +#define LOG_ANALYZE_GROUP_NFA 0 +#define LOG_AFTER_MODIFY_GROUP_NFA 0 +#define LOG_ANALYZE_GROUP_NFA_RESULTS (0 || LOG_ANALYZE_GROUP_NFA > 1) #define LOG_UNION_REPEATED_PATTERN_GROUP 0 +#define LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT 0 #define NO_STATE (fsm_state_t)-1 @@ -68,6 +71,9 @@ struct analysis_info { * epsilon edge to the combined NFA's global_unanchored_end_loop. * The old unanchored_end_loop will be disconnected. */ fsm_state_t eager_match_state; + + /* These states need an epsilon edge added to the eager_matched_state. */ + struct state_set *needs_indirect_epsilon_edge_to_eager_match_state; }; struct fsm * @@ -253,37 +259,48 @@ dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set } #endif -/* If there's a labeled edge to an end state, check if the label set is - * only [\n] and there's also an epsilon edge to the same end state. - * This represents an anchored end in the NFA. */ +/* For each state in the epsilon closure, if there's a labeled edge + * to an end state, check if the label set is only [\n] and there's + * also an epsilon edge to the same end state. + * If so, this represents an anchored end in the NFA. */ static bool -has_epsilon_and_newline_edges_to_end(const struct fsm *nfa, fsm_state_t s_i, fsm_state_t *dst_end) +has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, struct state_set *eclosure, + fsm_state_t s_i, fsm_state_t *dst_end) { - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; + struct state_iter si; + state_set_reset(eclosure, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + assert(ns_i < nfa->statecount); + const struct fsm_state *ns = &nfa->states[ns_i]; - if (state_set_empty(s->epsilons)) { return false; } - if (edge_set_empty(s->edges)) { return false; } + if (state_set_empty(ns->epsilons)) { continue; } + if (edge_set_empty(ns->edges)) { continue; } - struct edge_group_iter iter; - struct edge_group_iter_info info; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + /* Look for an edge set with only '\n' */ + if ((info.symbols[0] != (1ULL << '\n')) + || info.symbols[1] || info.symbols[2] || info.symbols[3]) { + continue; + } - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - /* Look for an edge set with only '\n' */ - if ((info.symbols[0] != (1ULL << '\n')) - || info.symbols[1] || info.symbols[2] || info.symbols[3]) { - continue; - } + /* If it's an end, look for an epsilon leeding to the same destination */ + if (fsm_isend(nfa, info.to)) { + struct state_iter inner_si; + fsm_state_t os_i; - if (fsm_isend(nfa, info.to)) { - struct state_iter si; - fsm_state_t os_i; - state_set_reset(s->epsilons, &si); - while (state_set_next(&si, &os_i)) { - if (os_i == info.to) { - *dst_end = info.to; - return true; + assert(s_i < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_i]; + + state_set_reset(s->epsilons, &inner_si); + while (state_set_next(&inner_si, &os_i)) { + if (os_i == info.to) { + *dst_end = info.to; + return true; + } } } } @@ -293,123 +310,89 @@ has_epsilon_and_newline_edges_to_end(const struct fsm *nfa, fsm_state_t s_i, fsm } static bool -has_labeled_edge_to_unanchored_end_loop(const struct fsm *nfa, - fsm_state_t s_i, fsm_state_t unanchored_end_loop) +has_labeled_edge_to_eclosure_with_unanchored_end_loop(const struct fsm *nfa, + struct state_set **eclosures, + fsm_state_t s_i, fsm_state_t unanchored_end_loop, + fsm_state_t *indirect_dst) { if (unanchored_end_loop == NO_STATE) { return false; } assert(unanchored_end_loop < nfa->statecount); - /* The unanchored_end_loop's self-edge doesn't count here. */ - if (s_i == unanchored_end_loop) { return false; } - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; - struct edge_group_iter iter; - struct edge_group_iter_info info; - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - if (info.to == unanchored_end_loop) { - return true; - } - } - return false; -} - -static bool -start_state_epsilon_closure_matches_empty_string__iter(const struct fsm *nfa, - fsm_state_t s_i, struct state_set **seen, bool *result) -{ - if (*result) { return true; } - if (!state_set_add(seen, nfa->alloc, s_i)) { return false; } - - if (fsm_isend(nfa, s_i)) { - *result = true; - return true; - } - - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; + const struct state_set *s_eclosure = eclosures[s_i]; + /* For every state in s_i's epsilon closure, check if it has + * a labeled edge to a state with the unanchored_end_loop + * in its epsilon closure. */ struct state_iter si; - state_set_reset(s->epsilons, &si); + state_set_reset(s_eclosure, &si); fsm_state_t ns_i; while (state_set_next(&si, &ns_i)) { - if (!state_set_contains(*seen, ns_i)) { - if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, ns_i, seen, result)) { - return false; - } - if (*result) { break; } - } - } + /* The unanchored_end_loop's self-edge doesn't count here. */ + if (ns_i == unanchored_end_loop) { continue; } - return true; -} - -/* Does the start state's epsilon closure match the empty string? - * Returns false on error, otherwise returns true and sets *result. */ -static bool -start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, fsm_state_t start, bool *result) -{ - struct state_set *seen = NULL; /* empty set */ - if (!start_state_epsilon_closure_matches_empty_string__iter(nfa, start, &seen, result)) { return false; } - state_set_free(seen); - - return true; -} - -static bool -copy_noncycle_labeled_edges_from_epsilon_closure_iter(struct analysis_info *ainfo, - const struct fsm *nfa, fsm_state_t usl_i, fsm_state_t s_i, struct state_set **seen) -{ - const struct fsm_state *s = &nfa->states[s_i]; + /* FIXME: this should only apply to the original state, not its epsilon closure...right? */ + if (ns_i != s_i) { continue; } - struct edge_group_iter egi; - struct edge_group_iter_info info; + assert(ns_i < nfa->statecount); + const struct fsm_state *ns = &nfa->states[ns_i]; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to < nfa->statecount); + const struct state_set *to_eclosure = eclosures[info.to]; + + struct state_iter dst_si; + state_set_reset(to_eclosure, &dst_si); + fsm_state_t dst_s_i; + while (state_set_next(&dst_si, &dst_s_i)) { + if (dst_s_i == unanchored_end_loop) { + if (info.to != unanchored_end_loop) { + *indirect_dst = info.to; + } - /* copy its labeled edges to ainfo->repeatable_firsts, - * unless they lead back to the unanchored_start_loop */ - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); - while (edge_set_group_iter_next(&egi, &info)) { - if (info.to != usl_i) { - if (!edge_set_add_bulk(&ainfo->repeatable_firsts, - nfa->alloc, info.symbols, info.to)) { - return false; + return true; + } } } } - if (!state_set_add(seen, nfa->alloc, usl_i)) { return false; } - - struct state_iter si; - state_set_reset(s->epsilons, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - if (state_set_contains(*seen, ns_i)) { continue; } - if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, usl_i, ns_i, seen)) { - return false; - } - } - - return true; + return false; } static bool -copy_noncycle_labeled_edges_from_epsilon_closure(struct analysis_info *ainfo, - const struct fsm *nfa, fsm_state_t unanchored_start_loop_id) +start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, const struct state_set *eclosure) { - struct state_set *seen = NULL; /* empty set */ - if (!copy_noncycle_labeled_edges_from_epsilon_closure_iter(ainfo, nfa, - unanchored_start_loop_id, unanchored_start_loop_id, &seen)) { - return false; + struct state_iter si; + state_set_reset(eclosure, &si); + + fsm_state_t s_i; + while (state_set_next(&si, &s_i)) { + if (fsm_isend(nfa, s_i)) { return true; } } - state_set_free(seen); - return true; + return false; } +static const struct fsm_options dump_nfa_opt = { + .io = FSM_IO_STR, + .ambig = AMBIG_MULTIPLE, + .case_ranges = 1, + .consolidate_edges = 1, + .group_edges = 1, +}; + static bool analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) { + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "==== %s\n", __func__); + if (LOG_ANALYZE_GROUP_NFA > 1) { + fsm_print(stderr, nfa, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + } + } + memset(ainfo, 0x00, sizeof(*ainfo)); ainfo->start = NO_STATE; ainfo->unanchored_start_loop = NO_STATE; @@ -425,42 +408,67 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) const size_t state_count = fsm_countstates(nfa); assert(ainfo->start < state_count); - { - const struct fsm_state *s = &nfa->states[ainfo->start]; + struct state_set **eclosures = epsilon_closure(nfa); + if (eclosures == NULL) { + return false; + } + /* First pass: Iterate over the start state's epsilon edges, + * attempting to identify the unanchored start loop and anchored + * start states (if present). + * + * Note: This uses the start state's epsilon set rather than its + * epsilon closure because (by construction) the unanchored + * start loop and anchored start states will both be directly + * connected to the start state. Using the epsilon closure can + * mis-identify the unanchored *end* loop as the start loop, if + * there is a path with only epsilon edges between them. */ + struct state_iter si; + state_set_reset(nfa->states[ainfo->start].epsilons, &si); + fsm_state_t ns_i; + while (state_set_next(&si, &ns_i)) { + if (ns_i == ainfo->start) { continue; } + + /* If there's a state in the start state's epsilon set that + * has a dot self-edge, it's the unanchored start loop. */ + if (has_dot_self_edge(nfa, ns_i)) { + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "%s: unanchored_start_loop found on state %d\n", __func__, ns_i); + } + /* there can be only one */ + assert(ainfo->unanchored_start_loop == NO_STATE + || ainfo->unanchored_start_loop == ns_i); + ainfo->unanchored_start_loop = ns_i; + continue; + } else { + /* Otherwise, a state without a dot self-edge is the anchored start. */ + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); + } + assert(ainfo->anchored_start == NO_STATE || ainfo->anchored_start == ns_i); + ainfo->anchored_start = ns_i; + continue; + } + } + + /* Copy labeled edges from the unanchored start loop and + * its epsilon closure to ainfo->repeatable_firsts, except + * for edges leading back to the unanchored start loop. */ + if (ainfo->unanchored_start_loop != NO_STATE) { struct state_iter si; - state_set_reset(s->epsilons, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - if (ns_i == ainfo->start) { continue; } + state_set_reset(eclosures[ainfo->unanchored_start_loop], &si); + fsm_state_t cs_i; + while (state_set_next(&si, &cs_i)) { + assert(cs_i < nfa->statecount); + const struct fsm_state *cs = &nfa->states[cs_i]; /* closure state */ struct edge_group_iter egi; struct edge_group_iter_info info; - assert(ns_i < state_count); - const struct fsm_state *ns = &nfa->states[ns_i]; - - /* if there's a state in the start state's epsilon closure that - * has a dot self-edge, it's the unanchored start loop */ - if (has_dot_self_edge(nfa, ns_i)) { - assert(ainfo->unanchored_start_loop == NO_STATE); - ainfo->unanchored_start_loop = ns_i; - - /* Copy labeled edges from the unanchored start loop and its epsilon - * closure to ainfo->repeatable_firsts, except for edges leading back - * to the unanchored start loop. */ - if (!copy_noncycle_labeled_edges_from_epsilon_closure(ainfo, nfa, ns_i)) { - goto alloc_fail; - } - } else { - /* likewise, a state without a dot self-edge is the anchored start */ - assert(ainfo->anchored_start == NO_STATE); - ainfo->anchored_start = ns_i; - - /* copy its labeled edges to ainfo->anchored_firsts */ - edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &egi); - while (edge_set_group_iter_next(&egi, &info)) { - if (!edge_set_add_bulk(&ainfo->anchored_firsts, + edge_set_group_iter_reset(cs->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != ainfo->unanchored_start_loop) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, nfa->alloc, info.symbols, info.to)) { goto alloc_fail; } @@ -469,12 +477,33 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) } } + /* Copy labeled edges from the anchored start and its epsilon + * closure to ainfo->anchored_firsts. */ + if (ainfo->anchored_start != NO_STATE) { + struct state_iter si; + state_set_reset(eclosures[ainfo->anchored_start], &si); + fsm_state_t cs_i; + while (state_set_next(&si, &cs_i)) { + assert(cs_i < nfa->statecount); + const struct fsm_state *cs = &nfa->states[cs_i]; + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + edge_set_group_iter_reset(cs->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (!edge_set_add_bulk(&ainfo->anchored_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } + /* If the start state always matches, set a flag noting that it will need special handling * later. It's arguably pointless to combine "" with other regexes, because it will always * trivially match, but otherwise it would never match. */ - if (!start_state_epsilon_closure_matches_empty_string(nfa, ainfo->start, &ainfo->nullable)) { - goto alloc_fail; - } + ainfo->nullable = start_state_epsilon_closure_matches_empty_string(nfa, eclosures[ainfo->start]); /* If there's a state with a dot self-edge and an epsilon edge to an end state, it's * the unanchored end loop. There should only be one. */ @@ -499,17 +528,24 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) /* Collect states that lead to an anchored end or eager match. */ for (size_t s_i = 0; s_i < state_count; s_i++) { fsm_state_t dst_end = NO_STATE; - if (has_epsilon_and_newline_edges_to_end(nfa, s_i, &dst_end)) { + if (has_epsilon_and_newline_edges_to_same_end(nfa, eclosures[s_i], s_i, &dst_end)) { if (!state_set_add(&ainfo->anchored_ends, nfa->alloc, dst_end)) { goto alloc_fail; } } - if (has_labeled_edge_to_unanchored_end_loop(nfa, s_i, ainfo->unanchored_end_loop)) { + fsm_state_t indirect_dst = NO_STATE; + if (has_labeled_edge_to_eclosure_with_unanchored_end_loop(nfa, eclosures, s_i, ainfo->unanchored_end_loop, &indirect_dst)) { if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { goto alloc_fail; } } + + if (indirect_dst != NO_STATE) { + if (!state_set_add(&ainfo->needs_indirect_epsilon_edge_to_eager_match_state, nfa->alloc, indirect_dst)) { + goto alloc_fail; + } + } } #if LOG_ANALYZE_GROUP_NFA_RESULTS @@ -522,10 +558,17 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) dump_edge_set(stderr, "repeatable_firsts", ainfo->unanchored_start_loop, ainfo->repeatable_firsts); } #endif + + closure_free(nfa, eclosures, state_count); + return true; alloc_fail: fprintf(stderr, "alloc fail\n"); + if (eclosures != NULL) { + closure_free(nfa, eclosures, state_count); + } + return false; } @@ -534,7 +577,8 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) * the libfsm API for this (and it shouldn't be necessary in general), but * if it gets one later this can be replaced. */ static bool -replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to) +replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to, + bool *found) { if (old_to == NO_STATE) { /* nothing to do */ @@ -552,6 +596,7 @@ replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to struct edge_group_iter_info info; edge_set_group_iter_reset(old_edges, EDGE_GROUP_ITER_ALL, &iter); while (edge_set_group_iter_next(&iter, &info)) { + if (info.to == old_to) { *found = true; } if (!edge_set_add_bulk(&new_edges, nfa->alloc, info.symbols, info.to == old_to ? new_to : info.to)) { return false; @@ -577,6 +622,7 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t { const bool nullable_and_unanchored_end = ainfo->nullable && ainfo->unanchored_end_loop != NO_STATE; + const bool log = 0 || (LOG_ANALYZE_GROUP_NFA > 0); /* Add the eager match state if there are eager match states * or a nullable unanchored end. This will link to the global NFA's @@ -585,25 +631,47 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t if (!fsm_addstate(nfa, &ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: added eager_match_state %d\n", __func__, ainfo->eager_match_state); + } /* Set eager match ID on new eager_match_state. */ const fsm_output_id_t oid = (fsm_output_id_t)(id + id_base); if (!fsm_seteageroutput(nfa, ainfo->eager_match_state, oid)) { return false; } + if (log) { + fprintf(stderr, "%s: set eager_output id %d on eager_match_state %d\n", + __func__, oid, ainfo->eager_match_state); + } /* For every state in eager_matches, replace every edge leading to * the unanchored_end_loop with an edge with the same labels to - * eager_match_state. */ + * eager_match_state. + * + * If the labeled edge does not directly lead to the unanchored_end_loop, + * then add an epsilon edge from wherever it leads to eager_match_state + * instead. */ struct state_iter si; state_set_reset(ainfo->eager_matches, &si); fsm_state_t ems_i; while (state_set_next(&si, &ems_i)) { + bool found = false; if (!replace_labeled_edge(nfa, ems_i, - ainfo->unanchored_end_loop, ainfo->eager_match_state)) { + ainfo->unanchored_end_loop, ainfo->eager_match_state, &found)) { return false; } + if (log) { + if (found) { + fprintf(stderr, "%s: replacing labeled edges from eager_match_state %d to unanchored_end_loop %d with edge to new eager_match_state %d\n", + __func__, ems_i, ainfo->unanchored_end_loop, ainfo->eager_match_state); + } else if (!found && ainfo->unanchored_end_loop != NO_STATE) { + fprintf(stderr, "%s: not found: labeled edges from eager_match_state %d to unanchored_end_loop %d\n", + __func__, ems_i, ainfo->unanchored_end_loop); + } + } + /* The state must not link to the unanchored end loop anymore. * Doing so will cause a combinatorial explosion that makes * combining more ~10 NFAs incredibly expensive. */ @@ -616,6 +684,16 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t assert(info.to != ainfo->unanchored_end_loop); } } + + state_set_reset(ainfo->needs_indirect_epsilon_edge_to_eager_match_state, &si); + fsm_state_t intermediate_i; + while (state_set_next(&si, &intermediate_i)) { + if (!fsm_addedge_epsilon(nfa, intermediate_i, ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: adding epsilon edge from intermediate eager match state %d to new eager_match_state %d\n", + __func__, intermediate_i, ainfo->eager_match_state); + } + } } /* If the group NFA matches the empty string and has an unanchored end, then @@ -630,6 +708,10 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t if (!state_set_add(&s->epsilons, nfa->alloc, ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: adding epsilon edge from unanchored_start_loop %d to eager_match_state %d\n", + __func__, ainfo->unanchored_start_loop, ainfo->eager_match_state); + } } } @@ -643,9 +725,21 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t if (!fsm_endid_set(nfa, anchored_end_state, end_id)) { return false; } + if (log) { + fprintf(stderr, "%s: setting endid %d on anchored_end_state %d\n", + __func__, end_id, anchored_end_state); + } } } +#if LOG_AFTER_MODIFY_GROUP_NFA + fprintf(stderr, "=== after %s\n", __func__); + fsm_print(stderr, nfa, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + fsm_endid_dump(stderr, nfa); + fsm_eager_output_dump(stderr, nfa); + +#endif + return true; } @@ -740,6 +834,8 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addstate(res, &global_end)) { goto fail; } if (!fsm_addstate(res, &global_unanchored_end_loop)) { goto fail; } + /* do this later, combining NFAs may rebase the state IDs */ +#if 0 /* link the start to the global unanchored start loop and anchored start. */ if (log) { fprintf(stderr, "link_before: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", @@ -757,6 +853,7 @@ fsm_union_repeated_pattern_group(size_t nfa_count, } if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } +#endif if (bases != NULL) { memset(bases, 0x00, nfa_count * sizeof(bases[0])); @@ -781,6 +878,10 @@ fsm_union_repeated_pattern_group(size_t nfa_count, struct fsm *merged = fsm_merge(res, fsm, &combine_info); if (merged == NULL) { goto fail; } + if (log) { + fprintf(stderr, "merged: bases a %d and b %d\n", combine_info.base_a, combine_info.base_b); + } + /* Update offsets if res had its state IDs shifted forward. */ global_start += combine_info.base_a; global_unanchored_start_loop += combine_info.base_a; @@ -807,6 +908,11 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_end)) { goto fail; } + if (log) { + fprintf(stderr, "eager_match_state: adding epsilon EMS %d -> global_unanchored_end_loop %d and EMS %d -> global_end %d\n", + ainfo->eager_match_state, global_unanchored_end_loop, + ainfo->eager_match_state, global_end); + } /* If the NFA matches the empty string and is not anchored at the end, then * add an epsilon edge from the global start directly to its eager match state. @@ -816,6 +922,10 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(merged, global_start, ainfo->eager_match_state)) { goto fail; } + if (log) { + fprintf(stderr, "nullable: global_start %d -> eager_match_state %d\n", + global_start, ainfo->eager_match_state); + } } } else { /* If the NFA matches an end-anchored empty string, then add an epsilon edge from @@ -829,6 +939,11 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(merged, global_start, anchored_end_state)) { goto fail; } + if (log) { + fprintf(stderr, "nullable & anchored ends: global_start %d -> anchored_end_state %d\n", + global_start, anchored_end_state); + } + /* It should only be necessary to link one, since that's enough * for determinisation to carry the end id back to the start's * epsilon closure. */ @@ -850,6 +965,10 @@ fsm_union_repeated_pattern_group(size_t nfa_count, info.symbols, info.to)) { goto fail; } + if (log) { + fprintf(stderr, "anchored_firsts: adding global_anchored_start %d -> info.to %d (with same labels)\n", + global_anchored_start, info.to); + } } /* Link the global_unanchored_start_loop to group FSM paths that aren't @@ -861,15 +980,53 @@ fsm_union_repeated_pattern_group(size_t nfa_count, info.symbols, info.to)) { goto fail; } + + if (log) { + fprintf(stderr, "repeatable_firsts: adding global_unanchored_start_loop %d -> info.to %d (same edges)\n", + global_unanchored_start_loop, info.to); + } + } + + + /* Add an epsilon edge from the global unanchored start loop to the NFA's. + * Without this, eager outputs for eager matches in the start state's epsilon + * closure may get lost during determinisation. */ + if (ainfo->unanchored_start_loop != NO_STATE) { + if (!fsm_addedge_epsilon(merged, global_unanchored_start_loop, ainfo->unanchored_start_loop)) { + goto fail; + } + + if (log) { + fprintf(stderr, "repeatable_firsts: adding an epsilon edge from global_unanchored_start_loop %d to NFA unanchored_start_loop %d\n", + global_unanchored_start_loop, ainfo->unanchored_start_loop); + } } res = merged; } + /* link the start to the global unanchored start loop and anchored start. */ + if (log) { + fprintf(stderr, "linking: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", + global_start, global_unanchored_start_loop, global_anchored_start); + } + if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } + + /* Link the global unanchored start loop to itself. */ + if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop and global end. */ + if (log) { + fprintf(stderr, "linking: global_unanchored_end_loop %d -> global_end %d (and -> self)\n", global_unanchored_end_loop, global_end); + } + if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } + /* Link from the global_unanchored_end_loop to the global_unanchored_start_loop, * so patterns with an unanchored start can follow other patterns with an unanchored * end, possibly with other ignored input between them. */ - if (log) { + if (log || LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT) { fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", __func__, global_start, global_unanchored_start_loop, global_anchored_start, global_unanchored_end_loop, global_end); fprintf(stderr, "%s: linking global_unanchored_end_loop %d to global_unanchored_start_loop %d\n", @@ -887,6 +1044,13 @@ fsm_union_repeated_pattern_group(size_t nfa_count, free_analysis(alloc, &ainfos[i]); } + if (LOG_UNION_REPEATED_PATTERN_GROUP || LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT) { + fprintf(stderr, "==== %s output (combined, pre det+min)\n", __func__); + fsm_print(stderr, res, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + fsm_endid_dump(stderr, res); + fsm_eager_output_dump(stderr, res); + } + f_free(alloc, ainfos); return res; diff --git a/tests/eager_output/eager_output_unanchored_end_plus.c b/tests/eager_output/eager_output_unanchored_end_plus.c new file mode 100644 index 000000000..81354ffcb --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abcx+", + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcx", .expected_ids = { 1 } }, + { .input = "abcxx", .expected_ids = { 1 } }, + { .input = "z", .expected_ids = { 2 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_min.c b/tests/eager_output/eager_output_unanchored_end_plus_min.c new file mode 100644 index 000000000..3d270514e --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_min.c @@ -0,0 +1,18 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abcx+", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcx", .expected_ids = { 1 } }, + { .input = "abcxx", .expected_ids = { 1 } }, + { .input = "z", .expect_fail = true }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_min2.c b/tests/eager_output/eager_output_unanchored_end_plus_min2.c new file mode 100644 index 000000000..20d735fcc --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_min2.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "z", .expected_ids = { 1 } }, + { .input = "Xz", .expected_ids = { 1 } }, + { .input = "XzX", .expected_ids = { 1 } }, + { .input = "zX", .expected_ids = { 1 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_multichar.c b/tests/eager_output/eager_output_unanchored_end_plus_multichar.c new file mode 100644 index 000000000..40d3e5d24 --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_multichar.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abc(xy)+", + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcxy", .expected_ids = { 1 } }, + { .input = "abcxyxy", .expected_ids = { 1 } }, + { .input = "z", .expected_ids = { 2 } }, + }, + }; + + return run_test(&test); +} From 7b8b16948241a40253f191339848995f30443b9c Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 12:12:15 -0500 Subject: [PATCH 322/373] Interface change: Add 'const'. --- src/libfsm/closure.c | 4 ++-- src/libfsm/internal.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index 3993afcda..165fa4964 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -128,7 +128,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s } struct state_set ** -epsilon_closure(struct fsm *fsm) +epsilon_closure(const struct fsm *fsm) { struct state_set **closures; fsm_state_t s; @@ -190,7 +190,7 @@ epsilon_closure(struct fsm *fsm) } void -closure_free(struct fsm *fsm, struct state_set **closures, size_t n) +closure_free(const struct fsm *fsm, struct state_set **closures, size_t n) { fsm_state_t s; diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 46997c82a..094723fdb 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -94,10 +94,10 @@ state_hasnondeterminism(const struct fsm *fsm, fsm_state_t state, struct bm *bm) * for states, with wrapper to populate malloced array of user-facing structs. */ struct state_set ** -epsilon_closure(struct fsm *fsm); +epsilon_closure(const struct fsm *fsm); void -closure_free(struct fsm *fsm, struct state_set **closures, size_t n); +closure_free(const struct fsm *fsm, struct state_set **closures, size_t n); /* * Internal free function that invokes free(3) by default, or a user-provided From 6b07bd2dfd943bda45d6d8e5641f18b6118f1ed1 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 14:56:49 -0500 Subject: [PATCH 323/373] union: Fix trivial memory leak. --- src/libfsm/union.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 9acdddf01..7ef7bb278 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -770,6 +770,7 @@ free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) { state_set_free(ainfo->anchored_ends); state_set_free(ainfo->eager_matches); + state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); edge_set_free(alloc, ainfo->anchored_firsts); edge_set_free(alloc, ainfo->repeatable_firsts); } From 7546f81e4b461f9b42ec70ec748eb42daae0a62b Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 12 Feb 2025 14:57:01 -0500 Subject: [PATCH 324/373] union.c: Add comments for assertions. Fuzzing has produced inputs that cause this to fail, but they all depend on embedded '\0' characters. I wasn't able to reproduce the failure without those present, but I will investigate further later. For now, adding a TODO. --- src/libfsm/union.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 7ef7bb278..a0790c3f9 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -435,7 +435,17 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) if (LOG_ANALYZE_GROUP_NFA) { fprintf(stderr, "%s: unanchored_start_loop found on state %d\n", __func__, ns_i); } - /* there can be only one */ + + /* TODO: There is only one unanchored start loop, but in obscure cases it may + * be difficult to distinguish between the USL and the unanchored end loop or + * other intermediate .* loops. The real USL will strictly appear before any + * other such loops in the graph. + * + * For now, assert that there is only one, because it's safer to have this + * loudly fail at compile time than produce an incorrect graph. Fuzzing has + * produced some inputs that make this fail, but currently they seem to + * depend on having a '\0' character embedded in the middle, which would + * normally be rejected by this point. */ assert(ainfo->unanchored_start_loop == NO_STATE || ainfo->unanchored_start_loop == ns_i); ainfo->unanchored_start_loop = ns_i; @@ -445,7 +455,10 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) if (LOG_ANALYZE_GROUP_NFA) { fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); } + + /* TODO: This, too, can fail in obscure cases and needs further investigation. */ assert(ainfo->anchored_start == NO_STATE || ainfo->anchored_start == ns_i); + ainfo->anchored_start = ns_i; continue; } From bb9f6203274d6ffbffa94ab52905a8b978deae6a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 14 Feb 2025 13:41:24 -0500 Subject: [PATCH 325/373] Switch to collecting an anchored_start state set, not just one state. `(^|wax-)((?:banana|^apple))` is an example of a regex that needs multiple anchored_start states linked in order to combine correctly. --- src/libfsm/union.c | 22 ++++++++++--------- ...ger_output_mixed_start_anchor_regression.c | 20 +++++++++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 tests/eager_output/eager_output_mixed_start_anchor_regression.c diff --git a/src/libfsm/union.c b/src/libfsm/union.c index a0790c3f9..9062e1dbd 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -48,8 +48,8 @@ struct analysis_info { /* The end state following the unanchored end loop. */ fsm_state_t unanchored_end_loop_end; - /* State that links to paths only reachable from the beginning of input. */ - fsm_state_t anchored_start; + /* States that link to paths only reachable from the beginning of input. */ + struct state_set *anchored_starts; /* States leading to an anchored end. */ struct state_set *anchored_ends; @@ -396,7 +396,6 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) memset(ainfo, 0x00, sizeof(*ainfo)); ainfo->start = NO_STATE; ainfo->unanchored_start_loop = NO_STATE; - ainfo->anchored_start = NO_STATE; ainfo->unanchored_end_loop = NO_STATE; ainfo->unanchored_end_loop_end = NO_STATE; ainfo->eager_match_state = NO_STATE; @@ -456,10 +455,9 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); } - /* TODO: This, too, can fail in obscure cases and needs further investigation. */ - assert(ainfo->anchored_start == NO_STATE || ainfo->anchored_start == ns_i); - - ainfo->anchored_start = ns_i; + if (!state_set_add(&ainfo->anchored_starts, nfa->alloc, ns_i)) { + goto alloc_fail; + } continue; } } @@ -492,9 +490,12 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) /* Copy labeled edges from the anchored start and its epsilon * closure to ainfo->anchored_firsts. */ - if (ainfo->anchored_start != NO_STATE) { + struct state_iter si_anchored_start; + state_set_reset(ainfo->anchored_starts, &si_anchored_start); + fsm_state_t anchored_start; + while (state_set_next(&si_anchored_start, &anchored_start)) { struct state_iter si; - state_set_reset(eclosures[ainfo->anchored_start], &si); + state_set_reset(eclosures[anchored_start], &si); fsm_state_t cs_i; while (state_set_next(&si, &cs_i)) { assert(cs_i < nfa->statecount); @@ -766,11 +767,11 @@ rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) SHIFT(unanchored_start_loop); SHIFT(unanchored_end_loop); SHIFT(unanchored_end_loop_end); - SHIFT(anchored_start); SHIFT(eager_match_state); #undef SHIFT state_set_rebase(&ainfo->anchored_ends, base); + state_set_rebase(&ainfo->anchored_starts, base); state_set_rebase(&ainfo->eager_matches, base); edge_set_rebase(&ainfo->anchored_firsts, base); @@ -782,6 +783,7 @@ static void free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) { state_set_free(ainfo->anchored_ends); + state_set_free(ainfo->anchored_starts); state_set_free(ainfo->eager_matches); state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); edge_set_free(alloc, ainfo->anchored_firsts); diff --git a/tests/eager_output/eager_output_mixed_start_anchor_regression.c b/tests/eager_output/eager_output_mixed_start_anchor_regression.c new file mode 100644 index 000000000..2965dc7aa --- /dev/null +++ b/tests/eager_output/eager_output_mixed_start_anchor_regression.c @@ -0,0 +1,20 @@ +#include "utils.h" + +/* Regression: This is a case that requires an anchored_start state set + * rather than a single optional anchored_start state ID to link + * correctly. */ + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "(^|wax-)((?:banana|^apple))", + "(^|wax-)(orange)", + }, + .inputs = { + { .input = "banana", .expected_ids = { 1 } }, + }, + }; + + return run_test(&test); +} From a54095a4813ee6cfc7d0d3dfe8f3a7abf3094e51 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 19 Sep 2025 12:13:03 -0400 Subject: [PATCH 326/373] Rename test file -- make tests looks for build files matching "*res*". `build/tests/eager_output/run_mixed_start_anchor_regression` matches "*res*" because of "regression", so `make test` incorrectly treats it like any of the other output test result files. Rename it, to remove "regression". This is an unexpected consequence of cb42d58f. --- ...tart_anchor_regression.c => eager_output_mixed_start_anchor.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/eager_output/{eager_output_mixed_start_anchor_regression.c => eager_output_mixed_start_anchor.c} (100%) diff --git a/tests/eager_output/eager_output_mixed_start_anchor_regression.c b/tests/eager_output/eager_output_mixed_start_anchor.c similarity index 100% rename from tests/eager_output/eager_output_mixed_start_anchor_regression.c rename to tests/eager_output/eager_output_mixed_start_anchor.c From eca233da96c54edc6f8b148a1c244763f8a19feb Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 9 Sep 2025 17:18:57 -0400 Subject: [PATCH 327/373] union.c: Updates to fsm_union_repeated_pattern_group and its internals. - Remove some dead code. - Update several comments. - Rename a couple functions to clarify their use. - Refine detection of the unanchored start loop. --- src/libfsm/union.c | 224 +++++++++++++++++++++++++++------------------ 1 file changed, 134 insertions(+), 90 deletions(-) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 9062e1dbd..7e011af8e 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -74,6 +74,11 @@ struct analysis_info { /* These states need an epsilon edge added to the eager_matched_state. */ struct state_set *needs_indirect_epsilon_edge_to_eager_match_state; + + /* States which are reachable from any state besides the start + * state. This can be necessary to correctly identify the + * unanchored start loop. */ + struct state_set *reachable_from_nonstart_state; }; struct fsm * @@ -209,7 +214,7 @@ fsm_union_array(size_t fsm_count, } static bool -has_dot_self_edge(const struct fsm *nfa, fsm_state_t s_i) +state_has_dot_self_edge(const struct fsm *nfa, fsm_state_t s_i) { const struct fsm_state *s = &nfa->states[s_i]; @@ -260,15 +265,15 @@ dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set #endif /* For each state in the epsilon closure, if there's a labeled edge - * to an end state, check if the label set is only [\n] and there's - * also an epsilon edge to the same end state. + * to an end state with no outgoing edges, check if the label set is + * only [\n] and there's also an epsilon edge to the same end state. * If so, this represents an anchored end in the NFA. */ static bool -has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, struct state_set *eclosure, - fsm_state_t s_i, fsm_state_t *dst_end) +state_has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, + fsm_state_t s_id, struct state_set *s_eclosure, fsm_state_t *dst_end) { struct state_iter si; - state_set_reset(eclosure, &si); + state_set_reset(s_eclosure, &si); fsm_state_t ns_i; while (state_set_next(&si, &ns_i)) { assert(ns_i < nfa->statecount); @@ -289,11 +294,18 @@ has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, struct state_se /* If it's an end, look for an epsilon leeding to the same destination */ if (fsm_isend(nfa, info.to)) { + assert(info.to < nfa->statecount); + const struct fsm_state *end_candidate = &nfa->states[info.to]; + if (!state_set_empty(end_candidate->epsilons) || + !edge_set_empty(end_candidate->edges)) { + continue; /* not an anchored end */ + } + struct state_iter inner_si; fsm_state_t os_i; - assert(s_i < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_i]; + assert(s_id < nfa->statecount); + const struct fsm_state *s = &nfa->states[s_id]; state_set_reset(s->epsilons, &inner_si); while (state_set_next(&inner_si, &os_i)) { @@ -310,50 +322,39 @@ has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, struct state_se } static bool -has_labeled_edge_to_eclosure_with_unanchored_end_loop(const struct fsm *nfa, - struct state_set **eclosures, - fsm_state_t s_i, fsm_state_t unanchored_end_loop, +state_has_labeled_edge_to_eclosure_with_unanchored_end_loop(const struct fsm *nfa, + fsm_state_t s_i, struct state_set **eclosures, + fsm_state_t unanchored_end_loop, fsm_state_t *indirect_dst) { if (unanchored_end_loop == NO_STATE) { return false; } assert(unanchored_end_loop < nfa->statecount); assert(s_i < nfa->statecount); - const struct state_set *s_eclosure = eclosures[s_i]; - - /* For every state in s_i's epsilon closure, check if it has - * a labeled edge to a state with the unanchored_end_loop - * in its epsilon closure. */ - struct state_iter si; - state_set_reset(s_eclosure, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - /* The unanchored_end_loop's self-edge doesn't count here. */ - if (ns_i == unanchored_end_loop) { continue; } - /* FIXME: this should only apply to the original state, not its epsilon closure...right? */ - if (ns_i != s_i) { continue; } + /* The unanchored_end_loop doesn't count, here. */ + if (s_i == unanchored_end_loop) { return false; } - assert(ns_i < nfa->statecount); - const struct fsm_state *ns = &nfa->states[ns_i]; - struct edge_group_iter iter; - struct edge_group_iter_info info; - edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - assert(info.to < nfa->statecount); - const struct state_set *to_eclosure = eclosures[info.to]; - - struct state_iter dst_si; - state_set_reset(to_eclosure, &dst_si); - fsm_state_t dst_s_i; - while (state_set_next(&dst_si, &dst_s_i)) { - if (dst_s_i == unanchored_end_loop) { - if (info.to != unanchored_end_loop) { - *indirect_dst = info.to; - } - - return true; + /* Check whether the state has a labeled edge to a state with the + * unanchored_end_loop in its epsilon closure. */ + const struct fsm_state *s = &nfa->states[s_i]; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to < nfa->statecount); + const struct state_set *to_eclosure = eclosures[info.to]; + + struct state_iter dst_si; + state_set_reset(to_eclosure, &dst_si); + fsm_state_t dst_s_i; + while (state_set_next(&dst_si, &dst_s_i)) { + if (dst_s_i == unanchored_end_loop) { + if (info.to != unanchored_end_loop) { + *indirect_dst = info.to; } + + return true; } } } @@ -412,9 +413,41 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) return false; } - /* First pass: Iterate over the start state's epsilon edges, - * attempting to identify the unanchored start loop and anchored - * start states (if present). + /* Mark any states that are reachable from any state besides the + * start state -- this means they cannot be the unanchored start + * loop, in cases where the pass below would otherwise detect + * more than one. */ + for (fsm_state_t s_i = 0; s_i < state_count; s_i++) { + if (s_i == ainfo->start) { continue; } + + struct state_iter si; + state_set_reset(nfa->states[s_i].epsilons, &si); + fsm_state_t eps_i; + while (state_set_next(&si, &eps_i)) { + /* Ignore self edges */ + if (eps_i == s_i) { continue; } + + if (!state_set_add(&ainfo->reachable_from_nonstart_state, nfa->alloc, eps_i)) { + return false; + } + } + + struct edge_group_iter egi; + struct edge_group_iter_info info; + edge_set_group_iter_reset(nfa->states[s_i].edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + /* Ignore self edges */ + if (info.to == s_i) { continue; } + + if (!state_set_add(&ainfo->reachable_from_nonstart_state, nfa->alloc, info.to)) { + return false; + } + } + } + + /* Iterate over the start state's epsilon edges, attempting to + * identify the unanchored start loop and anchored start states + * (if present). * * Note: This uses the start state's epsilon set rather than its * epsilon closure because (by construction) the unanchored @@ -426,31 +459,55 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) state_set_reset(nfa->states[ainfo->start].epsilons, &si); fsm_state_t ns_i; while (state_set_next(&si, &ns_i)) { + /* Ignore self edges. */ if (ns_i == ainfo->start) { continue; } /* If there's a state in the start state's epsilon set that * has a dot self-edge, it's the unanchored start loop. */ - if (has_dot_self_edge(nfa, ns_i)) { + if (state_has_dot_self_edge(nfa, ns_i)) { if (LOG_ANALYZE_GROUP_NFA) { fprintf(stderr, "%s: unanchored_start_loop found on state %d\n", __func__, ns_i); } - /* TODO: There is only one unanchored start loop, but in obscure cases it may - * be difficult to distinguish between the USL and the unanchored end loop or - * other intermediate .* loops. The real USL will strictly appear before any - * other such loops in the graph. + /* By construction, the true unanchored start loop is only reachable + * via an epsilon edge from the start state, so if any other state + * has an epsilon or labeled edge to this one, it cannot be the + * unanchored start loop. + * + * This is necessary for cases like '^|x', which produces: + * + * 0 -> 2; + * 0 -> 3; + * 2 -> 2 "\x00" .. "\xff"; + * 2 -> 3 "x"; + * 3 -> 1; + * 3 -> 3 ?; * - * For now, assert that there is only one, because it's safer to have this - * loudly fail at compile time than produce an incorrect graph. Fuzzing has - * produced some inputs that make this fail, but currently they seem to - * depend on having a '\0' character embedded in the middle, which would - * normally be rejected by this point. */ - assert(ainfo->unanchored_start_loop == NO_STATE - || ainfo->unanchored_start_loop == ns_i); + * start: 0; + * end: 1 = [0]; + * + * where this analysis would otherwise identify both 2 (correct) + * and 3 (incorrect) as the unanchored start loop. Both are reachable + * from the start state via an epsilon edge, but the labeled edge + * 2->3 'x' rules 3 out. + * */ + if (state_set_contains(ainfo->reachable_from_nonstart_state, ns_i)) { + continue; + } + + /* The reachable_from_nonstart_state check handles the other cases, + * but for `$|^` other attempts to distinguish them will fail, + * but by construction the USL will have the earlier state ID. */ + if (ainfo->unanchored_start_loop != NO_STATE && + ainfo->unanchored_start_loop < ns_i) { + continue; + } + ainfo->unanchored_start_loop = ns_i; continue; } else { - /* Otherwise, a state without a dot self-edge is the anchored start. */ + /* Otherwise, a state without a dot self-edge is an anchored start. + * There may be more than one. */ if (LOG_ANALYZE_GROUP_NFA) { fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); } @@ -523,7 +580,7 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) * the unanchored end loop. There should only be one. */ for (size_t s_i = 0; s_i < state_count; s_i++) { const struct fsm_state *s = &nfa->states[s_i]; - if (has_dot_self_edge(nfa, s_i)) { + if (state_has_dot_self_edge(nfa, s_i)) { struct state_iter si; state_set_reset(s->epsilons, &si); fsm_state_t ns_i; @@ -542,14 +599,14 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) /* Collect states that lead to an anchored end or eager match. */ for (size_t s_i = 0; s_i < state_count; s_i++) { fsm_state_t dst_end = NO_STATE; - if (has_epsilon_and_newline_edges_to_same_end(nfa, eclosures[s_i], s_i, &dst_end)) { + if (state_has_epsilon_and_newline_edges_to_same_end(nfa, s_i, eclosures[s_i], &dst_end)) { if (!state_set_add(&ainfo->anchored_ends, nfa->alloc, dst_end)) { goto alloc_fail; } } fsm_state_t indirect_dst = NO_STATE; - if (has_labeled_edge_to_eclosure_with_unanchored_end_loop(nfa, eclosures, s_i, ainfo->unanchored_end_loop, &indirect_dst)) { + if (state_has_labeled_edge_to_eclosure_with_unanchored_end_loop(nfa, s_i, eclosures, ainfo->unanchored_end_loop, &indirect_dst)) { if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { goto alloc_fail; } @@ -773,6 +830,7 @@ rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) state_set_rebase(&ainfo->anchored_ends, base); state_set_rebase(&ainfo->anchored_starts, base); state_set_rebase(&ainfo->eager_matches, base); + state_set_rebase(&ainfo->reachable_from_nonstart_state, base); edge_set_rebase(&ainfo->anchored_firsts, base); edge_set_rebase(&ainfo->repeatable_firsts, base); @@ -786,6 +844,7 @@ free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) state_set_free(ainfo->anchored_starts); state_set_free(ainfo->eager_matches); state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); + state_set_free(ainfo->reachable_from_nonstart_state); edge_set_free(alloc, ainfo->anchored_firsts); edge_set_free(alloc, ainfo->repeatable_firsts); } @@ -838,9 +897,11 @@ fsm_union_repeated_pattern_group(size_t nfa_count, fsm_state_t global_start; if (!fsm_addstate(res, &global_start)) { goto fail; } - /* States linking to the starts of unanchored and anchored subgraphs, respectively. - * Matching other group NFAs loops back to the global_unanchored_start_loop, but - * patterns anchored at the ^start are only reachable via global_anchored_start. */ + /* States linking to the starts of unanchored and anchored + * subgraphs, respectively. Matching group NFAs with unanchored + * ends will loop back to the global_unanchored_start_loop, but + * patterns anchored at the start are only reachable via + * global_anchored_start. */ fsm_state_t global_unanchored_start_loop, global_anchored_start; if (!fsm_addstate(res, &global_unanchored_start_loop)) { goto fail; } if (!fsm_addstate(res, &global_anchored_start)) { goto fail; } @@ -850,27 +911,6 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addstate(res, &global_end)) { goto fail; } if (!fsm_addstate(res, &global_unanchored_end_loop)) { goto fail; } - /* do this later, combining NFAs may rebase the state IDs */ -#if 0 - /* link the start to the global unanchored start loop and anchored start. */ - if (log) { - fprintf(stderr, "link_before: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", - global_start, global_unanchored_start_loop, global_anchored_start); - } - if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } - if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } - - /* Link the global unanchored start loop to itself. */ - if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } - - /* Link the global unanchored end loop and global end. */ - if (log) { - fprintf(stderr, "link_before: global_unanchored_end_loop %d -> global_end %d (and -> self)\n", global_unanchored_end_loop, global_end); - } - if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } - if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } -#endif - if (bases != NULL) { memset(bases, 0x00, nfa_count * sizeof(bases[0])); } @@ -889,7 +929,7 @@ fsm_union_repeated_pattern_group(size_t nfa_count, } assert(ainfo->start < state_count); - /* Call fsm_merge; we really don't care which is which. */ + /* Call fsm_merge; the argument order shouldn't matter. */ struct fsm_combine_info combine_info; struct fsm *merged = fsm_merge(res, fsm, &combine_info); if (merged == NULL) { goto fail; } @@ -1021,7 +1061,7 @@ fsm_union_repeated_pattern_group(size_t nfa_count, res = merged; } - /* link the start to the global unanchored start loop and anchored start. */ + /* Link the global start to the global unanchored start loop and anchored start states. */ if (log) { fprintf(stderr, "linking: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", global_start, global_unanchored_start_loop, global_anchored_start); @@ -1029,7 +1069,8 @@ fsm_union_repeated_pattern_group(size_t nfa_count, if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } - /* Link the global unanchored start loop to itself. */ + /* Link the global unanchored start loop to itself, so it can + * consume and ignore input preceding each matching group NFA. */ if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } /* Link the global unanchored end loop and global end. */ @@ -1050,9 +1091,12 @@ fsm_union_repeated_pattern_group(size_t nfa_count, fprintf(stderr, "%s: setting global_start %d and end %d\n", __func__, global_start, global_end); } if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop to the global end, so + * reaching the end of input there is considered a match. */ if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } - /* This needs to be set after merging, because that clears the start state. */ + /* These need to be set after merging, because that clears the start state. */ fsm_setstart(res, global_start); fsm_setend(res, global_end, 1); From e7125757382b1a13b625968d67a6392c98ed5715 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 15 Sep 2025 16:44:11 -0400 Subject: [PATCH 328/373] Eager outputs: Comment on use, rename some functions, clean up. --- include/fsm/fsm.h | 97 ++++++++++++++++++++----------- src/libfsm/clone.c | 2 +- src/libfsm/consolidate.c | 2 +- src/libfsm/determinise.c | 2 +- src/libfsm/eager_output.c | 26 ++++++--- src/libfsm/epsilons.c | 2 +- src/libfsm/libfsm.syms | 6 +- src/libfsm/merge.c | 2 +- src/libfsm/minimise_test_oracle.c | 4 +- src/libfsm/print/ir.c | 2 +- src/libfsm/union.c | 2 +- 11 files changed, 95 insertions(+), 52 deletions(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index f78d91d71..d57d43b1e 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -270,36 +270,48 @@ fsm_mapendids(struct fsm * fsm, fsm_endid_remap_fun remap, void *opaque); void fsm_increndids(struct fsm * fsm, int delta); -/* Associate an eagerly matched numeric ID with the end states in an fsm. - * - * This is similar to fsm_setendid, but has different performance - * trade-offs. In particular, it can become extremely expensive to - * combine multiple DFAs with endids on their end states when they - * representing regexes with unanchored ends, because the FSM has to - * explicitly represent all the possible combinations of matches by - * copying the entire path to every reachable end state. Eager endids - * are associated with the edge leaving the main pattern match. - * - * Returns 1 on success, 0 on error. - * */ -int -fsm_seteagerendid(struct fsm *fsm, fsm_end_id_t id); - /* Set an eager output ID to emit every time the state is entered. - * This turns the automata into a Moore machine. */ + * This is similar to fsm_setendid, but has different performance + * trade-offs for determinisation, and can be applied to + * non-end states. + * + * During DFA execution, states with eager outputs will output their + * ID when output reaches them. With fsm_exec, this happens via a + * callback (see fsm_eager_output_set_cb). Some print languages + * will eventually eager outputs. + * + * One use case for eager outputs is combining multiple unanchored + * regexes into a single DFA and detecting when input matches more than + * one of them. With endids, determinisation has to represent every + * possible reachable combination of endids as a distinct copy of the + * DFA subgraph, leading to a combinatorial explosion that makes + * combining more than a 8 or so regexes (even very simple ones) + * prohibitively expensive. With eager outputs, the graph no longer + * needs a separate subgraph copy for each combination of IDs, so it is + * possible to combine several dozen or even hundreds of FSMs into a + * single DFA. See fsm_union_repeated_pattern_group for more details. */ int -fsm_seteageroutput(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id); +fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id); /* Set an eager output ID on all current end states. */ int -fsm_seteageroutputonends(struct fsm *fsm, fsm_output_id_t id); +fsm_eager_output_set_on_ends(struct fsm *fsm, fsm_output_id_t id); -/* HACK */ +/* Callback for eager output processing. + * If set (using fsm_eager_output_set_cb), this may be called while fsm_exec runs. */ typedef void fsm_eager_output_cb(fsm_output_id_t id, void *opaque); + +/* Set a callback and opaque argument on an FSM for eager outputs encountered + * while fsm_exec is running. Rather than adding another pair of arguments to + * fsm_exec, this is called as a separate step -- most DFAs will not use eager + * outputs, or use them with code generation rather than fsm_exec. + * + * See fsm_eager_output_set for more details about eager output functionality. */ void fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque); +/* Get the eager output callback set on a FSM and its opaque pointer, if any. */ void fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque); @@ -307,11 +319,39 @@ fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void ** size_t fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state); -/* Get eager output associated with a state. It's expected that buf[] has - * sufficient space -- call fsm_eager_output_count first to get the count. - * The contents of buf will be sorted and unique. */ -void -fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, fsm_output_id_t *buf); +/* Get eager output IDs associated with a state, if any. + * id_buf is expected to have enough cells (according to id_buf_count) + * to store all the end IDs. You can find this with fsm_eager_output_count(). + * + * The IDs in the buffer are sorted and do not have duplicates. + * + * Unlike end IDs, eager outputs can appear on states that are + * not marked as end states. + * + * Returns 0 if there is not enough space in id_buf for the + * eager output IDs, or 1 if zero more IDs were returned. */ +int +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, + size_t buf_count, fsm_output_id_t *id_buf); + +/* Get the end IDs associated with an end state, if any. + * id_buf is expected to have enough cells (according to id_buf_count) + * to store all the end IDs. You can find this with fsm_endid_count(). + * + * The end IDs in the buffer are sorted and do not have duplicates. + * + * A state with no end IDs set is considered equivalent to a state + * that has the empty set, this API does not distinguish these cases. + * This is not an error. + * + * It is an error to attempt to get end IDs associated with a state + * that is not marked as an end state. + * + * Returns 0 if there is not enough space in id_buf for the + * end IDs, or 1 if zero or more end IDs were returned. */ +int +fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, + size_t id_buf_count, fsm_end_id_t *id_buf); /* * Find the state (if there is just one), or add epsilon edges from all states, @@ -498,15 +538,6 @@ fsm_shortest(const struct fsm *fsm, fsm_state_t start, fsm_state_t goal, unsigned (*cost)(fsm_state_t from, fsm_state_t to, char c)); -/* HACK */ -typedef void -fsm_eager_endid_cb(fsm_end_id_t id, void *opaque); -void -fsm_eager_endid_set_cb(struct fsm *fsm, fsm_eager_endid_cb *cb, void *opaque); - -void -fsm_eager_endid_get_cb(const struct fsm *fsm, fsm_eager_endid_cb **cb, void **opaque); - /* * Execute an FSM reading input from the user-specified callback fsm_getc(). * fsm_getc() is passed the opaque pointer given, and is expected to return diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index 2161599ae..068aca1c2 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -179,7 +179,7 @@ static int copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) { struct copy_eager_output_ids_env *env = opaque; - if (!fsm_seteageroutput(env->dst, state, id)) { + if (!fsm_eager_output_set(env->dst, state, id)) { env->ok = false; return 0; } diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index b7a8905b2..55c3bfd64 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -294,7 +294,7 @@ consolidate_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opa assert(state < env->mapping_count); const fsm_state_t dst_state = env->mapping[state]; - if (!fsm_seteageroutput(env->dst, dst_state, id)) { + if (!fsm_eager_output_set(env->dst, dst_state, id)) { env->ok = false; return 0; } diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 9833fd878..29a32d4ea 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -2599,7 +2599,7 @@ remap_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) { (void)state; struct remap_eager_output_env *env = opaque; - if (!fsm_seteageroutput(env->dst, env->dst_state, id)) { + if (!fsm_eager_output_set(env->dst, env->dst_state, id)) { env->ok = false; return 0; } diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c index 00fa1b5f0..04e067178 100644 --- a/src/libfsm/eager_output.c +++ b/src/libfsm/eager_output.c @@ -107,13 +107,13 @@ fsm_eager_output_free(struct fsm *fsm) } int -fsm_seteageroutputonends(struct fsm *fsm, fsm_output_id_t id) +fsm_eager_output_set_on_ends(struct fsm *fsm, fsm_output_id_t id) { assert(fsm != NULL); const size_t count = fsm_countstates(fsm); for (size_t i = 0; i < count; i++) { if (fsm_isend(fsm, i)) { - if (!fsm_seteageroutput(fsm, i, id)) { return 0; } + if (!fsm_eager_output_set(fsm, i, id)) { return 0; } } } return 1; @@ -157,7 +157,7 @@ grow_htab(const struct fsm_alloc *alloc, struct eager_output_htab *htab) } int -fsm_seteageroutput(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) +fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) { assert(fsm != NULL); @@ -296,7 +296,9 @@ fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state) } struct get_env { + bool ok; size_t count; + size_t ceil; fsm_output_id_t *buf; }; @@ -305,6 +307,10 @@ append_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) { struct get_env *env = opaque; (void)state; + if (env->count == env->ceil) { + env->ok = false; + return 0; + } env->buf[env->count++] = id; return 1; } @@ -317,12 +323,18 @@ cmp_fsm_output_id_t(const void *pa, const void *pb) return a < b ? -1 : a > b ? 1 : 0; } -void -fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, fsm_output_id_t *buf) +int +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, + size_t buf_count, fsm_output_id_t *id_buf) { - struct get_env env = { .buf = buf }; + struct get_env env = { + .ok = true, + .buf = id_buf, + .ceil = buf_count, + }; fsm_eager_output_iter_state(fsm, state, append_cb, &env); - qsort(buf, env.count, sizeof(buf[0]), cmp_fsm_output_id_t); + qsort(id_buf, env.count, sizeof(id_buf[0]), cmp_fsm_output_id_t); + return env.ok ? 1 : 0; } void diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 8041c29d3..834d88cbf 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -246,7 +246,7 @@ fsm_remove_epsilons(struct fsm *nfa) } for (size_t i = 0; i < eager_output_buf.used; i++) { - if (!fsm_seteageroutput(nfa, s, eager_output_buf.ids[i])) { + if (!fsm_eager_output_set(nfa, s, eager_output_buf.ids[i])) { goto cleanup; } } diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index ab28b0a21..a498ced4d 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -100,10 +100,10 @@ fsm_increndids fsm_endid_dump -fsm_seteageroutput -fsm_seteageroutputonends +fsm_eager_output_set +fsm_eager_output_set_on_ends fsm_eager_output_count -# short term hack +fsm_eager_output_get_cb fsm_eager_output_set_cb fsm_eager_output_dump fsm_eager_output_get diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index ccc1568ff..267b5b1df 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -214,7 +214,7 @@ static int copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) { struct copy_eager_output_ids_env *env = opaque; - if (!fsm_seteageroutput(env->dst, state + env->base_src, id)) { + if (!fsm_eager_output_set(env->dst, state + env->base_src, id)) { env->ok = false; return 0; } diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index 20d4633a1..65bd3b43d 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -212,7 +212,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) int eres = fsm_endid_get(fsm, i, endid_count_a, ids_a); assert(eres == 1); - fsm_eager_output_get(fsm, i, eo_ids_a); + fsm_eager_output_get(fsm, i, max_eager_output_count, eo_ids_a); bool found = false; /* note: skipping eg 0 here since that's the empty set */ @@ -234,7 +234,7 @@ fsm_minimise_test_oracle(const struct fsm *fsm) endid_count_b, ids_b); assert(eres == 1); - fsm_eager_output_get(fsm, end_md_group_leaders[eg_i], eo_ids_b); + fsm_eager_output_get(fsm, end_md_group_leaders[eg_i], max_eager_output_count, eo_ids_b); if ((0 == memcmp(ids_a, ids_b, endid_count_a * sizeof(ids_a[0]))) && (0 == memcmp(eo_ids_a, eo_ids_b, eager_output_count_a * sizeof(eo_ids_a[0])))) { diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index a18dadbbc..d8e29b2e7 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -577,7 +577,7 @@ make_ir(const struct fsm *fsm, const struct fsm_options *opt) if (outputs == NULL) { goto error; } - fsm_eager_output_get(fsm, i, outputs->ids); + fsm_eager_output_get(fsm, i, eager_output_count, outputs->ids); outputs->count = eager_output_count; ir->states[i].eager_outputs = outputs; diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 7e011af8e..c3e8dad81 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -708,7 +708,7 @@ modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t /* Set eager match ID on new eager_match_state. */ const fsm_output_id_t oid = (fsm_output_id_t)(id + id_base); - if (!fsm_seteageroutput(nfa, ainfo->eager_match_state, oid)) { + if (!fsm_eager_output_set(nfa, ainfo->eager_match_state, oid)) { return false; } if (log) { From eeea923ff91f5f0f9a3d349763153c9096152afd Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 19 Sep 2025 12:35:19 -0400 Subject: [PATCH 329/373] fuzz/target.c: Updates for interface changes. --- fuzz/target.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/fuzz/target.c b/fuzz/target.c index 4ff8b63bd..fd59aeea4 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -505,8 +505,6 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - enum re_is_anchored_res anchorage[MAX_PATTERNS] = {0}; - /* for each pattern, attempt to compile to a DFA */ for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { const char *p = env.patterns[p_i]; @@ -526,7 +524,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) } const fsm_output_id_t endid = (fsm_output_id_t)p_i; - ret = fsm_seteageroutputonends(fsm, endid); + ret = fsm_eager_output_set_on_ends(fsm, endid); assert(ret == 1); if (verbose) { @@ -575,7 +573,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) /* copy and combine fsms into one DFA */ { size_t used = 0; - struct fsm_union_entry entries[MAX_PATTERNS] = {0}; + struct fsm *nfas[MAX_PATTERNS] = {0}; for (size_t i = 0; i < env.fsm_count; i++) { /* there can be gaps, fsms[] lines up with patterns[] */ @@ -601,9 +599,7 @@ fuzz_eager_output(const uint8_t *data, size_t size) } } - entries[used].fsm = cp; - entries[used].anchored_start = anchorage[i] & RE_IS_ANCHORED_START; - entries[used].anchored_end = anchorage[i] & RE_IS_ANCHORED_END; + nfas[used] = cp; used++; } @@ -611,8 +607,8 @@ fuzz_eager_output(const uint8_t *data, size_t size) goto cleanup; /* nothing to do */ } - /* consumes entries[] */ - struct fsm *fsm = fsm_union_repeated_pattern_group(used, entries, NULL); + /* consumes nfas[] */ + struct fsm *fsm = fsm_union_repeated_pattern_group(used, nfas, NULL, 0); assert(fsm != NULL); if (verbose) { From d6db021c79214858c488fb4c96345af0b9a4a12a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 19 Sep 2025 13:20:01 -0400 Subject: [PATCH 330/373] Misc. cleanup before integration. --- src/libfsm/eager_output.c | 14 +------------- src/libfsm/eager_output.h | 12 ++++++++++++ src/libfsm/epsilons.c | 4 +--- src/libfsm/exec.c | 2 -- src/libfsm/state.c | 1 - .../eager_output_mixed_anchored_unanchored.c | 3 --- 6 files changed, 14 insertions(+), 22 deletions(-) diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c index 04e067178..a6dcdc89f 100644 --- a/src/libfsm/eager_output.c +++ b/src/libfsm/eager_output.c @@ -172,15 +172,11 @@ fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) const uint64_t mask = info->htab.bucket_count - 1; assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ - /* fprintf(stderr, "%s: bucket_count %zd\n", __func__, info->htab.bucket_count); */ for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { const size_t b_i = (hash + probes) & mask; struct eager_output_bucket *b = &info->htab.buckets[b_i]; - /* fprintf(stderr, "%s: state %d -> b_i %zd, state %d, entry %p\n", */ - /* __func__, state, b_i, b->state, (void *)b->entry); */ struct eager_output_entry *e = b->entry; - if (e == NULL) { /* empty */ - /* add */ + if (e == NULL) { /* empty, add */ const size_t alloc_sz = sizeof(*e) + DEF_ENTRY_CEIL * sizeof(e->ids[0]); e = f_calloc(fsm->alloc, 1, alloc_sz); @@ -191,8 +187,6 @@ fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) b->state = state; b->entry = e; info->htab.buckets_used++; - /* fprintf(stderr, "%s: buckets_used %zd\n", __func__, info->htab.buckets_used); */ - /* fprintf(stderr, "%s: saved new entry in bucket %zd\n", __func__, b_i); */ } else if (b->state != state) { /* collision */ continue; } @@ -214,7 +208,6 @@ fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) } e->ids[e->used++] = id; - /* fprintf(stderr, "%s: e->ids_used %u\n", __func__, e->used); */ fsm->states[state].has_eager_outputs = 1; return 1; } @@ -259,8 +252,6 @@ fsm_eager_output_iter_state(const struct fsm *fsm, for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { const size_t b_i = (hash + probes) & mask; struct eager_output_bucket *b = &info->htab.buckets[b_i]; - /* fprintf(stderr, "%s: state %d -> b_i %zd, state %d, entry %p\n", */ - /* __func__, state, b_i, b->state, (void *)b->entry); */ struct eager_output_entry *e = b->entry; if (e == NULL) { /* empty */ return; @@ -347,12 +338,9 @@ fsm_eager_output_iter_all(const struct fsm *fsm, struct eager_output_info *info = fsm->eager_output_info; - /* fprintf(stderr, "%s: bucket_count %zd\n", __func__, info->htab.bucket_count); */ for (size_t b_i = 0; b_i < info->htab.bucket_count; b_i++) { struct eager_output_bucket *b = &info->htab.buckets[b_i]; struct eager_output_entry *e = b->entry; - /* fprintf(stderr, "%s: b_i %zd, state %d, entry %p\n", */ - /* __func__, b_i, b->state, (void *)b->entry); */ if (e == NULL) { /* empty */ continue; } diff --git a/src/libfsm/eager_output.h b/src/libfsm/eager_output.h index 6093adc9e..b90da935c 100644 --- a/src/libfsm/eager_output.h +++ b/src/libfsm/eager_output.h @@ -1,3 +1,9 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + #ifndef EAGER_OUTPUT_H #define EAGER_OUTPUT_H @@ -13,12 +19,15 @@ fsm_eager_output_init(struct fsm *fsm); void fsm_eager_output_free(struct fsm *fsm); +/* Does an FSM have eager outputs? */ bool fsm_eager_output_has_eager_output(const struct fsm *fsm); +/* Does a particular state have eager outputs? */ bool fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state); +/* Dump eager outputs on an FSM. (For debugging.) */ void fsm_eager_output_dump(FILE *f, const struct fsm *fsm); @@ -28,14 +37,17 @@ fsm_eager_output_dump(FILE *f, const struct fsm *fsm); typedef int fsm_eager_output_iter_cb(fsm_state_t state, fsm_output_id_t id, void *opaque); +/* Iterate over eager outputs on a state. */ void fsm_eager_output_iter_state(const struct fsm *fsm, fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque); +/* Iterate over all eager outputs on an FSM. */ void fsm_eager_output_iter_all(const struct fsm *fsm, fsm_eager_output_iter_cb *cb, void *opaque); +/* Compact eager output metadata. */ int fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index 834d88cbf..926e6d9bf 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -291,7 +291,7 @@ fsm_remove_epsilons(struct fsm *nfa) * reachable. This doesn't check that the FROM state is reachable from * the start state (trim will do that soon enough), it's just used to * check which states will become unreachable once epsilon edges are - * removed. We don't need to add eager endids for them, because they + * removed. We don't need to add eager outputs for them, because they * will soon be disconnected from the epsilon-free NFA. */ static void mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label) @@ -311,9 +311,7 @@ mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_lab struct fsm_state *s = &nfa->states[s_i]; /* Clear the visited flag, it will be used to avoid cycles. */ -#if 1 assert(s->visited == 0); /* stale */ -#endif s->visited = 0; edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); diff --git a/src/libfsm/exec.c b/src/libfsm/exec.c index 077494b8f..d8cebd843 100644 --- a/src/libfsm/exec.c +++ b/src/libfsm/exec.c @@ -55,7 +55,6 @@ struct check_eager_outputs_for_state_env { static int match_eager_outputs_for_state_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) { - /* HACK update the types here once it's working */ (void)state; struct check_eager_outputs_for_state_env *env = opaque; #if LOG_EAGER @@ -68,7 +67,6 @@ match_eager_outputs_for_state_cb(fsm_state_t state, fsm_end_id_t id, void *opaqu static int match_eager_outputs_for_state(const struct fsm *fsm, fsm_state_t state) { - /* HACK update the types here once it's working */ fsm_eager_output_cb *cb = NULL; void *opaque = NULL; fsm_eager_output_get_cb(fsm, &cb, &opaque); diff --git a/src/libfsm/state.c b/src/libfsm/state.c index 8f1146038..786bc3a25 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -90,7 +90,6 @@ fsm_addstate_bulk(struct fsm *fsm, size_t n) new->visited = 0; new->epsilons = NULL; new->edges = NULL; - new->has_eager_outputs = 0; } diff --git a/tests/eager_output/eager_output_mixed_anchored_unanchored.c b/tests/eager_output/eager_output_mixed_anchored_unanchored.c index 7afb272db..376b49d8a 100644 --- a/tests/eager_output/eager_output_mixed_anchored_unanchored.c +++ b/tests/eager_output/eager_output_mixed_anchored_unanchored.c @@ -2,9 +2,6 @@ int main(void) { - /* fprintf(stderr, "%s: skipping for now, this doesn't pass yet.\n", __FILE__); */ - /* return EXIT_SUCCESS; */ - struct eager_output_test test = { .patterns = { "^abc$", From 3293b7b95a464bcd11573e57b0a6241dc9995878 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 19 Sep 2025 13:40:26 -0400 Subject: [PATCH 331/373] Restore FORCE_ENDIDS behavior for tests/eager_output/eager_output7.c. This was removed in af27a87c, while updating for interface changes, but it's worth keeping this. It makes the performance difference between using endids and eager outputs for combining larger sets of unanchored patterns obvious -- on my current machine, `build/tests/eager_output/run7` completes in about 55 msec, whereas `env FORCE_ENDIDS=7 build/tests/eager_output/run7` takes about 88 msec, 8 takes 200 msec, 9 takes 639 msec, 10 takes 1.62 seconds, 11 takes 3.9 seconds, ... without FORCE_ENDIDS, it's running all 26 patterns. (FORCE_ENDIDS < 7 finishes faster, but that's because it's only using the first couple patterns and combining into a much smaller DFA.) Also, a minor bugfix -- instead of asserting that the outputs were >= the expected ID count, check that it's exactly equal. This was due to IDs set by both eager outputs and endids being counted twice; some of the eager_output_alt_mixing_anchored_and_unanchored.c cases could lead to this, due to the mixing of anchored and unanchored subtrees in the regexes. --- tests/eager_output/eager_output7.c | 29 +++++++++++++++++++++++++++ tests/eager_output/utils.c | 32 +++++++++++++++++++++++++++--- tests/eager_output/utils.h | 1 + 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/tests/eager_output/eager_output7.c b/tests/eager_output/eager_output7.c index 94e9f1787..e764ac946 100644 --- a/tests/eager_output/eager_output7.c +++ b/tests/eager_output/eager_output7.c @@ -2,7 +2,26 @@ int main(void) { + /* Run this test with env FORCE_ENDIDS=N ... to see how much more + * expensive it is to combine the first N patterns using endids, + * rather than eager_outputs. It becomes VERY slow for >= 9 or so. + * (Note that the checks probably will not pass for N < 4, because + * it will start skipping appear in the early test inputs.) */ + bool force_endids = false; + size_t force_endid_count = 0; + { + const char *str = getenv("FORCE_ENDIDS"); + if (str != NULL) { + force_endid_count = atoi(str); + if (force_endid_count == 0) { + force_endid_count = 26; + } + force_endids = true; + } + } + struct eager_output_test test = { + .force_endids = force_endids, .patterns = { [0] = "apple", [1] = "banana", @@ -71,5 +90,15 @@ int main(void) }, }; + /* truncate patterns to the first N */ + if (force_endids) { + assert(force_endid_count > 0 && force_endid_count <= 26); + test.patterns[force_endid_count] = NULL; + + /* truncate test inputs to just the first couple, since + * later inputs use later patterns */ + test.inputs[5].input = NULL; + } + return run_test(&test); } diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c index dfd2b952b..725da7e41 100644 --- a/tests/eager_output/utils.c +++ b/tests/eager_output/utils.c @@ -83,7 +83,16 @@ run_test(const struct eager_output_test *test) } const size_t id_base = 1; /* offset by 1 because 0 is used as end-of-list */ - struct fsm *fsm = fsm_union_repeated_pattern_group(nfas_used, nfas, NULL, id_base); + struct fsm *fsm; + if (test->force_endids) { + for (size_t i = 0; i < nfas_used; i++) { + fsm_setendid(nfas[i], i + id_base); + } + fsm = fsm_union_array(nfas_used, nfas, NULL); + } else { + /* This function sets the eager output IDs. */ + fsm = fsm_union_repeated_pattern_group(nfas_used, nfas, NULL, id_base); + } assert(fsm != NULL); if (log) { @@ -167,6 +176,7 @@ run_test(const struct eager_output_test *test) fsm_state_t end; /* only set on match */ ret = fsm_exec(fsm, fsm_sgetc, &input, &end, NULL); + size_t match_id_count = 0; if (ret == 1) { #define ENDID_BUF_SIZE 32 fsm_end_id_t endid_buf[ENDID_BUF_SIZE] = {0}; @@ -177,10 +187,26 @@ run_test(const struct eager_output_test *test) assert(!"fsm_endid_get failed"); } + match_id_count += outputs.used; + for (size_t e_i = 0; e_i < endid_count; e_i++) { + fsm_end_id_t endid = endid_buf[e_i]; + bool found = false; + for (size_t o_i = 0; o_i < outputs.used; o_i++) { + if (outputs.ids[o_i] == endid) { + found = true; + break; + } + } + if (!found) { + /* Don't count IDs set by both endids AND eager outputs twice. */ + match_id_count++; + } + } + /* Copy endid outputs into outputs.ids[], since for testing * purposes we don't care about the difference between eager * output and endids here. */ - assert(outputs.used + endid_count <= MAX_IDS); + assert(match_id_count <= MAX_IDS); for (size_t endid_i = 0; endid_i < endid_count; endid_i++) { if (log) { fprintf(stderr, "-- adding endid %zd: %d\n", endid_i, endid_buf[endid_i]); @@ -222,7 +248,7 @@ run_test(const struct eager_output_test *test) assert(ret == 1); } - assert(outputs.used >= expected_id_count); + assert(match_id_count == expected_id_count); size_t floor = 0; for (size_t exp_i = 0; exp_i < outputs.used; exp_i++) { diff --git a/tests/eager_output/utils.h b/tests/eager_output/utils.h index 02f8427c9..ee5f941c5 100644 --- a/tests/eager_output/utils.h +++ b/tests/eager_output/utils.h @@ -32,6 +32,7 @@ struct eager_output_test { const char *patterns[MAX_PATTERNS]; + bool force_endids; struct { const char *input; From 255e426ca2672f5929e72e2f8ef864f85579fa1a Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 22 Sep 2025 10:53:32 -0400 Subject: [PATCH 332/373] fsm.h: Fix missing word in comment. --- include/fsm/fsm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index d57d43b1e..a811c16ed 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -278,7 +278,7 @@ fsm_increndids(struct fsm * fsm, int delta); * During DFA execution, states with eager outputs will output their * ID when output reaches them. With fsm_exec, this happens via a * callback (see fsm_eager_output_set_cb). Some print languages - * will eventually eager outputs. + * will eventually support eager outputs. * * One use case for eager outputs is combining multiple unanchored * regexes into a single DFA and detecting when input matches more than From 610f2a3c2d2aad4a32c65c46c8401ac0aaf85d2e Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 22 Sep 2025 11:02:15 -0400 Subject: [PATCH 333/373] Note why fsm_union_repeated_pattern_group depends on re_comp. This may not be necessary in the future, but currently analyze_group_nfa in union.c depends on knowing that re_comp will always assign the true unanchored start loop an earlier state ID than the unanchored end loop in order to properly link a few ambiguous cases. --- include/fsm/bool.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/fsm/bool.h b/include/fsm/bool.h index c2c2d80ed..25d456b55 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -57,8 +57,11 @@ fsm_union_array(size_t fsm_count, * eager outputs can match. Ownership of the NFAs is transferred, they will * be combined (or freed, if they don't have a start state). * - * This MUST be called with NFAs constructed via re_comp, Calling it with - * manually constructed NFAs or DFAs is unsupported. + * This MUST be called with NFAs constructed via re_comp, Calling it + * with manually constructed NFAs or DFAs may lead to incorrect loop + * linking, because in a few ambiguous cases (e.g. `$|^`) it relies on + * internal details of re_comp's normal construction to correctly + * identify the state representing the unanchored start loop. * * This will set end IDs and/or output IDs representing matching each * of the original NFAs on the combined result, where nfas[i] will From a32aff3cc8b1624d55a8389d26bff456ebc505be Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 28 Oct 2025 12:36:01 +0000 Subject: [PATCH 334/373] Stray comment. --- src/rx/main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rx/main.c b/src/rx/main.c index 8fe40eb58..4ac357bc8 100644 --- a/src/rx/main.c +++ b/src/rx/main.c @@ -464,9 +464,6 @@ build_literals_fsm(bool show_stats, } } - /* We don't minimise here because this fsm has multiple endids, - * and the resulting FSM would be very similar to the current DFA */ - #ifndef NDEBUG /* * We could test to see that the fsm isn't any different. @@ -481,6 +478,9 @@ build_literals_fsm(bool show_stats, */ #endif + /* We don't minimise here because this fsm has multiple endids, + * and the resulting FSM would be very similar to the current DFA */ + return fsm; } From fcde3b700ab43ac66b0aebd5caf1f2dea8e297ce Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 16 Oct 2025 15:40:47 -0400 Subject: [PATCH 335/373] Optionally save linkage_info during NFA construction in re_comp. This adds a new flag for re_comp, RE_SAVE_LINKAGE_INFO. If that's set, then save information about the unanchored start and end loops, the end loop end, anchored starts, and anchored ends while building the NFA from the regex. Previously, this info was recovered by analyzing the NFA structure inside of fsm_union_repeated_pattern_group, but it seems less brittle to save those details during construction. This commit adds the flag, saves the info during construction, and after doing analysis it checks the results match. Since there is one case where it doesn't (eager_output_fr3.c), print a message and override it. This was actually a bug: that particular edge case leads to detecting the same state as both the unanchored start and end loops. In that specific case, it seems to be harmless, but this commit also adds an assert that the USL and UEL are not the same state. From now on, fsm_union_repeated_pattern_group depends on all of the NFAs passed to it having the linkage_info struct populated. They should come fresh from re_comp, without determinising or other transformations. I added an assert at the end of determinisation checking that the DFA does not have a linkage_info struct. Note: fsm_clone currently doesn't copy the linkage_info. Maybe it should. --- include/re/re.h | 3 ++ src/libfsm/determinise.c | 3 ++ src/libfsm/fsm.c | 7 ++++ src/libfsm/internal.h | 24 +++++++++++ src/libfsm/union.c | 84 +++++++++++++++++++++++++++++++++++++- src/libre/ast_compile.c | 46 ++++++++++++++++++++- tests/eager_output/utils.c | 2 +- 7 files changed, 166 insertions(+), 3 deletions(-) diff --git a/include/re/re.h b/include/re/re.h index 841e4e946..69551d39f 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -29,6 +29,9 @@ enum re_flags { RE_ANCHORED = 1 << 6, RE_EXTENDED = 1 << 7, /* PCRE extended mode */ RE_END_NL = 1 << 8, /* end anchor matches '\n' */ + /* save info about linkage at construction time, to inform + * later operations -- see fsm_union_repeated_pattern_group */ + RE_SAVE_LINKAGE_INFO = 1 << 9, RE_FLAGS_NONE = 0 }; diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 29a32d4ea..3f748aeae 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -280,6 +280,9 @@ fsm_determinise_with_config(struct fsm *nfa, assert(fsm_all(nfa, fsm_isdfa)); #endif + /* This should not be carried over from the NFA. */ + assert(nfa->linkage_info == NULL); + res = FSM_DETERMINISE_WITH_CONFIG_OK; cleanup: diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index c442c8262..866650a9c 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -42,6 +42,11 @@ free_contents(struct fsm *fsm) fsm_endid_free(fsm); fsm_eager_output_free(fsm); + if (fsm->linkage_info != NULL) { + state_set_free(fsm->linkage_info->anchored_starts); + state_set_free(fsm->linkage_info->anchored_ends); + f_free(fsm->alloc, fsm->linkage_info); + } f_free(fsm->alloc, fsm->states); } @@ -72,6 +77,7 @@ fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc) new->endcount = 0; new->capture_info = NULL; new->endid_info = NULL; + new->linkage_info = NULL; new->states = f_malloc(new->alloc, new->statealloc * sizeof *new->states); if (new->states == NULL) { @@ -144,6 +150,7 @@ fsm_move(struct fsm *dst, struct fsm *src) dst->capture_info = src->capture_info; dst->endid_info = src->endid_info; dst->eager_output_info = src->eager_output_info; + dst->linkage_info = src->linkage_info; f_free(src->alloc, src); } diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 094723fdb..06658a78e 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -18,6 +18,7 @@ struct bm; struct edge_set; struct state_set; struct state_array; +struct linkage_info; /* * The alphabet (Sigma) for libfsm's FSM is arbitrary octets. @@ -80,6 +81,29 @@ struct fsm { struct fsm_capture_info *capture_info; struct endid_info *endid_info; struct eager_output_info *eager_output_info; + struct linkage_info *linkage_info; +}; + +#define LINKAGE_NO_STATE ((fsm_state_t)-1) + +/* Internal structure for storing structural info about an NFA. + * This is currently only used by fsm_union_repeated_pattern_group, + * which needs to identify a couple components of the NFA in order + * to link groups of repeated pattern together correctly. */ +struct linkage_info { + /* The states with a /./ self edge representing the unanchored + * start and end, or LINKAGE_NO_STATE. There can be at most one + * of each. */ + fsm_state_t unanchored_start_loop; + fsm_state_t unanchored_end_loop; + + /* The end state following the unanchored end loop. */ + fsm_state_t unanchored_end_loop_end; + + /* States that link to paths only reachable from the beginning of input. */ + struct state_set *anchored_starts; + /* States leading to an anchored end. */ + struct state_set *anchored_ends; }; struct fsm * diff --git a/src/libfsm/union.c b/src/libfsm/union.c index c3e8dad81..f4e7bd6e2 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -619,19 +619,91 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) } } + /* Compare/log the linkage info */ +#define COMPARE_LINKAGE_INFO 1 +#define LOG_LINKAGE_INFO 1 + if (LOG_LINKAGE_INFO) { + struct state_iter si; + state_set_reset(ainfo->anchored_starts, &si); + fsm_state_t s_i; + + fprintf(stderr, "ainfo->anchored_starts count: %zd\n", state_set_count(ainfo->anchored_starts)); + state_set_reset(ainfo->anchored_starts, &si); + while (state_set_next(&si, &s_i)) { + fprintf(stderr, "ainfo->anchored_starts: %d\n", s_i); + } + + fprintf(stderr, "linkage_info->anchored_starts count: %zd\n", state_set_count(nfa->linkage_info->anchored_starts)); + state_set_reset(nfa->linkage_info->anchored_starts, &si); + while (state_set_next(&si, &s_i)) { + fprintf(stderr, "linkage_info->anchored_starts: %d\n", s_i); + } + + assert(state_set_count(nfa->linkage_info->anchored_starts) >= state_set_count(ainfo->anchored_starts)); + state_set_reset(ainfo->anchored_starts, &si); + while (state_set_next(&si, &s_i)) { + assert(state_set_contains(nfa->linkage_info->anchored_starts, s_i)); + } + } + + if (LOG_LINKAGE_INFO) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t s_i; + + fprintf(stderr, "ainfo->anchored_ends count: %zd\n", state_set_count(ainfo->anchored_ends)); + state_set_reset(ainfo->anchored_ends, &si); + while (state_set_next(&si, &s_i)) { + fprintf(stderr, "ainfo->anchored_ends: %d\n", s_i); + } + + fprintf(stderr, "linkage_info->anchored_ends count: %zd\n", state_set_count(nfa->linkage_info->anchored_ends)); + state_set_reset(nfa->linkage_info->anchored_ends, &si); + while (state_set_next(&si, &s_i)) { + fprintf(stderr, "linkage_info->anchored_ends: %d\n", s_i); + } + + assert(state_set_count(nfa->linkage_info->anchored_ends) >= state_set_count(ainfo->anchored_ends)); + state_set_reset(ainfo->anchored_ends, &si); + while (state_set_next(&si, &s_i)) { + assert(state_set_contains(nfa->linkage_info->anchored_ends, s_i)); + } + } + #if LOG_ANALYZE_GROUP_NFA_RESULTS { fprintf(stderr, "# analysis_info start %d, usl %d, uel %d, uele %d\n", ainfo->start, ainfo->unanchored_start_loop, ainfo->unanchored_end_loop, ainfo->unanchored_end_loop_end); dump_state_set(stderr, "anchored_ends", ainfo->anchored_ends); dump_state_set(stderr, "eager_matches", ainfo->eager_matches); - dump_edge_set(stderr, "anchored_firsts", ainfo->anchored_start, ainfo->anchored_firsts); dump_edge_set(stderr, "repeatable_firsts", ainfo->unanchored_start_loop, ainfo->repeatable_firsts); } #endif closure_free(nfa, eclosures, state_count); + if (COMPARE_LINKAGE_INFO) { + /* Check that the analysis and saved linkage_info from ast_compile.c match */ + fprintf(stderr, "%s: checking that build-time data matches... usl %d, %d; uel %d, %d; uele %d, %d\n", + __func__, + nfa->linkage_info->unanchored_start_loop, ainfo->unanchored_start_loop, + nfa->linkage_info->unanchored_end_loop, ainfo->unanchored_end_loop, + nfa->linkage_info->unanchored_end_loop_end, ainfo->unanchored_end_loop_end); + + if (nfa->linkage_info->unanchored_start_loop != ainfo->unanchored_start_loop) { + fprintf(stderr, "DISAGREEMENT, overriding\n"); + ainfo->unanchored_start_loop = nfa->linkage_info->unanchored_start_loop; + } + + assert(nfa->linkage_info->unanchored_start_loop == ainfo->unanchored_start_loop); + assert(nfa->linkage_info->unanchored_end_loop == ainfo->unanchored_end_loop); + assert(nfa->linkage_info->unanchored_end_loop_end == ainfo->unanchored_end_loop_end); + } + + /* The unanchored start and end loop cannot be the same state. */ + assert(nfa->linkage_info->unanchored_start_loop == NO_STATE + || nfa->linkage_info->unanchored_start_loop != nfa->linkage_info->unanchored_end_loop); + return true; alloc_fail: @@ -869,6 +941,16 @@ fsm_union_repeated_pattern_group(size_t nfa_count, errno = EINVAL; return NULL; } + + /* Any NFAs passed to this function must be built with + * an re_comp flag of RE_SAVE_LINKAGE_INFO, because some + * of the info saved during construction informs + * linking. */ + if (nfas[i]->linkage_info == NULL) { + errno = EINVAL; + return NULL; + } + const size_t count = fsm_countstates(nfas[i]); est_total_states += count; } diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 0ea0b08d8..5d31ef0a2 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -20,6 +20,8 @@ #include +#include + #include "class.h" #include "ast.h" #include "ast_compile.h" @@ -258,6 +260,10 @@ intern_start_any_loop(struct comp_env *env) env->start_any_loop = loop; env->has_start_any_loop = 1; + if (env->fsm->linkage_info) { + env->fsm->linkage_info->unanchored_start_loop = loop; + } + return 1; } @@ -290,6 +296,11 @@ intern_end_any_loop(struct comp_env *env) env->end_any_loop = loop; env->has_end_any_loop = 1; + if (env->fsm->linkage_info != NULL) { + env->fsm->linkage_info->unanchored_end_loop = loop; + env->fsm->linkage_info->unanchored_end_loop_end = env->end; + } + return 1; } @@ -327,6 +338,14 @@ intern_end_nl(struct comp_env *env) env->end_nl = end_nl; env->has_end_nl = 1; + + if (env->fsm->linkage_info != NULL) { + if (!state_set_add(&env->fsm->linkage_info->anchored_ends, + env->fsm->alloc, env->end)) { + return 0; + } + } + return 1; } @@ -718,7 +737,7 @@ comp_iter(struct comp_env *env, } #if LOG_LINKAGE - fprintf(stderr, " ---> x: %d, y: %d\n", x, y); + fprintf(stderr, " ---> x: %d, y: %d, type: %s\n", x, y, ast_node_type_name(n->type)); #endif switch (n->type) { @@ -871,6 +890,20 @@ comp_iter(struct comp_env *env, case AST_EXPR_ANCHOR: EPSILON(x, y); + + if (env->fsm->linkage_info != NULL + && x == env->start + && n->u.anchor.type == AST_ANCHOR_START) { + /* This state is directly linked from the global start. */ +#if LOG_LINKAGE + fprintf(stderr, "%s: adding %d to anchored_starts due to start anchor\n", + __func__, y); +#endif + if (!state_set_add(&env->fsm->linkage_info->anchored_starts, + env->fsm->alloc, y)) { + return 0; + } + } break; case AST_EXPR_SUBTRACT: { @@ -973,6 +1006,17 @@ ast_compile(const struct ast *ast, return NULL; } + if (re_flags & RE_SAVE_LINKAGE_INFO) { + struct linkage_info *li = f_malloc(alloc, sizeof(*fsm->linkage_info)); + if (li == NULL) { goto error; } + li->unanchored_start_loop = LINKAGE_NO_STATE; + li->unanchored_end_loop = LINKAGE_NO_STATE; + li->unanchored_end_loop_end = LINKAGE_NO_STATE; + li->anchored_starts = NULL; + li->anchored_ends = NULL; + fsm->linkage_info = li; + } + if (!fsm_addstate(fsm, &x)) { goto error; } diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c index 725da7e41..d1ff4f7b4 100644 --- a/tests/eager_output/utils.c +++ b/tests/eager_output/utils.c @@ -65,7 +65,7 @@ run_test(const struct eager_output_test *test) const char *p = test->patterns[i]; if (test->patterns[i] == NULL) { break; } - struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, RE_SAVE_LINKAGE_INFO, NULL); assert(fsm != NULL); if (log) { From 9971fe126f6ecb8a41dac546ce224bdf7767972f Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 5 Nov 2025 10:27:58 -0500 Subject: [PATCH 336/373] Copy some fields from linkage_info, remove analysis. They need to be copied over (transferring ownership of the state sets) because the linkage_info struct gets freed during the call to fsm_merge. This removes some code that tended to reconstruct the same information by analyzing the NFA structure, but was significantly more compliacted, and in at least one case it led to the wrong result (albeit harmlessly). It's simpler to just save the relevant state IDs during NFA construction. --- src/libfsm/union.c | 288 +++++---------------------------------------- 1 file changed, 28 insertions(+), 260 deletions(-) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index f4e7bd6e2..409553d1d 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -41,17 +41,19 @@ struct analysis_info { fsm_state_t start; /* start state */ /* The states with a /./ self edge representing the unanchored - * start and end, or NO_STATE. There can be at most one of each. */ + * start and end, or LINKAGE_NO_STATE. There can be at most one + * of each. Copied from linkage_info. */ fsm_state_t unanchored_start_loop; fsm_state_t unanchored_end_loop; - /* The end state following the unanchored end loop. */ + /* The end state following the unanchored end loop. + * Copied from linkage_info.*/ fsm_state_t unanchored_end_loop_end; - /* States that link to paths only reachable from the beginning of input. */ + /* States that link to paths only reachable from the beginning of input. + * Copied from linkage_info. */ struct state_set *anchored_starts; - - /* States leading to an anchored end. */ + /* States leading to an anchored end. Copied from linkage_info. */ struct state_set *anchored_ends; /* States with an outgoing labeled edge to the unanchored end loop. Input @@ -213,25 +215,6 @@ fsm_union_array(size_t fsm_count, return res; } -static bool -state_has_dot_self_edge(const struct fsm *nfa, fsm_state_t s_i) -{ - const struct fsm_state *s = &nfa->states[s_i]; - - struct edge_group_iter ei; - edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &ei); - struct edge_group_iter_info info; - while (edge_set_group_iter_next(&ei, &info)) { - if (info.to != s_i) { continue; } - for (size_t i = 0; i < 256/64; i++) { - if (info.symbols[i] != (uint64_t)-1) { continue; } - } - return true; - } - - return false; -} - #if LOG_ANALYZE_GROUP_NFA_RESULTS static void dump_state_set(FILE *f, const char *name, const struct state_set *set) @@ -264,63 +247,6 @@ dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set } #endif -/* For each state in the epsilon closure, if there's a labeled edge - * to an end state with no outgoing edges, check if the label set is - * only [\n] and there's also an epsilon edge to the same end state. - * If so, this represents an anchored end in the NFA. */ -static bool -state_has_epsilon_and_newline_edges_to_same_end(const struct fsm *nfa, - fsm_state_t s_id, struct state_set *s_eclosure, fsm_state_t *dst_end) -{ - struct state_iter si; - state_set_reset(s_eclosure, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - assert(ns_i < nfa->statecount); - const struct fsm_state *ns = &nfa->states[ns_i]; - - if (state_set_empty(ns->epsilons)) { continue; } - if (edge_set_empty(ns->edges)) { continue; } - - struct edge_group_iter iter; - struct edge_group_iter_info info; - edge_set_group_iter_reset(ns->edges, EDGE_GROUP_ITER_ALL, &iter); - while (edge_set_group_iter_next(&iter, &info)) { - /* Look for an edge set with only '\n' */ - if ((info.symbols[0] != (1ULL << '\n')) - || info.symbols[1] || info.symbols[2] || info.symbols[3]) { - continue; - } - - /* If it's an end, look for an epsilon leeding to the same destination */ - if (fsm_isend(nfa, info.to)) { - assert(info.to < nfa->statecount); - const struct fsm_state *end_candidate = &nfa->states[info.to]; - if (!state_set_empty(end_candidate->epsilons) || - !edge_set_empty(end_candidate->edges)) { - continue; /* not an anchored end */ - } - - struct state_iter inner_si; - fsm_state_t os_i; - - assert(s_id < nfa->statecount); - const struct fsm_state *s = &nfa->states[s_id]; - - state_set_reset(s->epsilons, &inner_si); - while (state_set_next(&inner_si, &os_i)) { - if (os_i == info.to) { - *dst_end = info.to; - return true; - } - } - } - } - } - - return false; -} - static bool state_has_labeled_edge_to_eclosure_with_unanchored_end_loop(const struct fsm *nfa, fsm_state_t s_i, struct state_set **eclosures, @@ -394,11 +320,7 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) } } - memset(ainfo, 0x00, sizeof(*ainfo)); ainfo->start = NO_STATE; - ainfo->unanchored_start_loop = NO_STATE; - ainfo->unanchored_end_loop = NO_STATE; - ainfo->unanchored_end_loop_end = NO_STATE; ainfo->eager_match_state = NO_STATE; if (!fsm_getstart(nfa, &ainfo->start)) { @@ -445,80 +367,6 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) } } - /* Iterate over the start state's epsilon edges, attempting to - * identify the unanchored start loop and anchored start states - * (if present). - * - * Note: This uses the start state's epsilon set rather than its - * epsilon closure because (by construction) the unanchored - * start loop and anchored start states will both be directly - * connected to the start state. Using the epsilon closure can - * mis-identify the unanchored *end* loop as the start loop, if - * there is a path with only epsilon edges between them. */ - struct state_iter si; - state_set_reset(nfa->states[ainfo->start].epsilons, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - /* Ignore self edges. */ - if (ns_i == ainfo->start) { continue; } - - /* If there's a state in the start state's epsilon set that - * has a dot self-edge, it's the unanchored start loop. */ - if (state_has_dot_self_edge(nfa, ns_i)) { - if (LOG_ANALYZE_GROUP_NFA) { - fprintf(stderr, "%s: unanchored_start_loop found on state %d\n", __func__, ns_i); - } - - /* By construction, the true unanchored start loop is only reachable - * via an epsilon edge from the start state, so if any other state - * has an epsilon or labeled edge to this one, it cannot be the - * unanchored start loop. - * - * This is necessary for cases like '^|x', which produces: - * - * 0 -> 2; - * 0 -> 3; - * 2 -> 2 "\x00" .. "\xff"; - * 2 -> 3 "x"; - * 3 -> 1; - * 3 -> 3 ?; - * - * start: 0; - * end: 1 = [0]; - * - * where this analysis would otherwise identify both 2 (correct) - * and 3 (incorrect) as the unanchored start loop. Both are reachable - * from the start state via an epsilon edge, but the labeled edge - * 2->3 'x' rules 3 out. - * */ - if (state_set_contains(ainfo->reachable_from_nonstart_state, ns_i)) { - continue; - } - - /* The reachable_from_nonstart_state check handles the other cases, - * but for `$|^` other attempts to distinguish them will fail, - * but by construction the USL will have the earlier state ID. */ - if (ainfo->unanchored_start_loop != NO_STATE && - ainfo->unanchored_start_loop < ns_i) { - continue; - } - - ainfo->unanchored_start_loop = ns_i; - continue; - } else { - /* Otherwise, a state without a dot self-edge is an anchored start. - * There may be more than one. */ - if (LOG_ANALYZE_GROUP_NFA) { - fprintf(stderr, "%s: anchored_start found on state %d\n", __func__, ns_i); - } - - if (!state_set_add(&ainfo->anchored_starts, nfa->alloc, ns_i)) { - goto alloc_fail; - } - continue; - } - } - /* Copy labeled edges from the unanchored start loop and * its epsilon closure to ainfo->repeatable_firsts, except * for edges leading back to the unanchored start loop. */ @@ -576,35 +424,8 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) * trivially match, but otherwise it would never match. */ ainfo->nullable = start_state_epsilon_closure_matches_empty_string(nfa, eclosures[ainfo->start]); - /* If there's a state with a dot self-edge and an epsilon edge to an end state, it's - * the unanchored end loop. There should only be one. */ - for (size_t s_i = 0; s_i < state_count; s_i++) { - const struct fsm_state *s = &nfa->states[s_i]; - if (state_has_dot_self_edge(nfa, s_i)) { - struct state_iter si; - state_set_reset(s->epsilons, &si); - fsm_state_t ns_i; - while (state_set_next(&si, &ns_i)) { - if (fsm_isend(nfa, ns_i)) { - assert(ainfo->unanchored_end_loop == NO_STATE); - ainfo->unanchored_end_loop = s_i; - ainfo->unanchored_end_loop_end = ns_i; - break; - } - } - if (ainfo->unanchored_end_loop != NO_STATE) { break; } - } - } - /* Collect states that lead to an anchored end or eager match. */ for (size_t s_i = 0; s_i < state_count; s_i++) { - fsm_state_t dst_end = NO_STATE; - if (state_has_epsilon_and_newline_edges_to_same_end(nfa, s_i, eclosures[s_i], &dst_end)) { - if (!state_set_add(&ainfo->anchored_ends, nfa->alloc, dst_end)) { - goto alloc_fail; - } - } - fsm_state_t indirect_dst = NO_STATE; if (state_has_labeled_edge_to_eclosure_with_unanchored_end_loop(nfa, s_i, eclosures, ainfo->unanchored_end_loop, &indirect_dst)) { if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { @@ -619,57 +440,6 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) } } - /* Compare/log the linkage info */ -#define COMPARE_LINKAGE_INFO 1 -#define LOG_LINKAGE_INFO 1 - if (LOG_LINKAGE_INFO) { - struct state_iter si; - state_set_reset(ainfo->anchored_starts, &si); - fsm_state_t s_i; - - fprintf(stderr, "ainfo->anchored_starts count: %zd\n", state_set_count(ainfo->anchored_starts)); - state_set_reset(ainfo->anchored_starts, &si); - while (state_set_next(&si, &s_i)) { - fprintf(stderr, "ainfo->anchored_starts: %d\n", s_i); - } - - fprintf(stderr, "linkage_info->anchored_starts count: %zd\n", state_set_count(nfa->linkage_info->anchored_starts)); - state_set_reset(nfa->linkage_info->anchored_starts, &si); - while (state_set_next(&si, &s_i)) { - fprintf(stderr, "linkage_info->anchored_starts: %d\n", s_i); - } - - assert(state_set_count(nfa->linkage_info->anchored_starts) >= state_set_count(ainfo->anchored_starts)); - state_set_reset(ainfo->anchored_starts, &si); - while (state_set_next(&si, &s_i)) { - assert(state_set_contains(nfa->linkage_info->anchored_starts, s_i)); - } - } - - if (LOG_LINKAGE_INFO) { - struct state_iter si; - state_set_reset(ainfo->anchored_ends, &si); - fsm_state_t s_i; - - fprintf(stderr, "ainfo->anchored_ends count: %zd\n", state_set_count(ainfo->anchored_ends)); - state_set_reset(ainfo->anchored_ends, &si); - while (state_set_next(&si, &s_i)) { - fprintf(stderr, "ainfo->anchored_ends: %d\n", s_i); - } - - fprintf(stderr, "linkage_info->anchored_ends count: %zd\n", state_set_count(nfa->linkage_info->anchored_ends)); - state_set_reset(nfa->linkage_info->anchored_ends, &si); - while (state_set_next(&si, &s_i)) { - fprintf(stderr, "linkage_info->anchored_ends: %d\n", s_i); - } - - assert(state_set_count(nfa->linkage_info->anchored_ends) >= state_set_count(ainfo->anchored_ends)); - state_set_reset(ainfo->anchored_ends, &si); - while (state_set_next(&si, &s_i)) { - assert(state_set_contains(nfa->linkage_info->anchored_ends, s_i)); - } - } - #if LOG_ANALYZE_GROUP_NFA_RESULTS { fprintf(stderr, "# analysis_info start %d, usl %d, uel %d, uele %d\n", @@ -682,27 +452,9 @@ analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) closure_free(nfa, eclosures, state_count); - if (COMPARE_LINKAGE_INFO) { - /* Check that the analysis and saved linkage_info from ast_compile.c match */ - fprintf(stderr, "%s: checking that build-time data matches... usl %d, %d; uel %d, %d; uele %d, %d\n", - __func__, - nfa->linkage_info->unanchored_start_loop, ainfo->unanchored_start_loop, - nfa->linkage_info->unanchored_end_loop, ainfo->unanchored_end_loop, - nfa->linkage_info->unanchored_end_loop_end, ainfo->unanchored_end_loop_end); - - if (nfa->linkage_info->unanchored_start_loop != ainfo->unanchored_start_loop) { - fprintf(stderr, "DISAGREEMENT, overriding\n"); - ainfo->unanchored_start_loop = nfa->linkage_info->unanchored_start_loop; - } - - assert(nfa->linkage_info->unanchored_start_loop == ainfo->unanchored_start_loop); - assert(nfa->linkage_info->unanchored_end_loop == ainfo->unanchored_end_loop); - assert(nfa->linkage_info->unanchored_end_loop_end == ainfo->unanchored_end_loop_end); - } - /* The unanchored start and end loop cannot be the same state. */ - assert(nfa->linkage_info->unanchored_start_loop == NO_STATE - || nfa->linkage_info->unanchored_start_loop != nfa->linkage_info->unanchored_end_loop); + assert(ainfo->unanchored_start_loop == NO_STATE + || ainfo->unanchored_start_loop != ainfo->unanchored_end_loop); return true; @@ -912,8 +664,6 @@ rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) static void free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) { - state_set_free(ainfo->anchored_ends); - state_set_free(ainfo->anchored_starts); state_set_free(ainfo->eager_matches); state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); state_set_free(ainfo->reachable_from_nonstart_state); @@ -958,8 +708,26 @@ fsm_union_repeated_pattern_group(size_t nfa_count, for (size_t i = 0; i < nfa_count; i++) { struct fsm *fsm = nfas[i]; + struct analysis_info *ainfo = &ainfos[i]; + + /* Copy these fields over, because fsm->linkage_info will be + * freed during the call to fsm_merge below. */ + { + struct linkage_info *linkage_info = fsm->linkage_info; + + ainfo->unanchored_start_loop = linkage_info->unanchored_start_loop; + ainfo->unanchored_end_loop = linkage_info->unanchored_end_loop; + ainfo->unanchored_end_loop_end = linkage_info->unanchored_end_loop_end; + + /* Transfer ownership of these. */ + ainfo->anchored_starts = linkage_info->anchored_starts; + linkage_info->anchored_starts = NULL; + ainfo->anchored_ends = linkage_info->anchored_ends; + linkage_info->anchored_ends = NULL; + } + /* Identify various states in the NFA that will be relevant to combining. */ - if (!analyze_group_nfa(fsm, &ainfos[i])) { + if (!analyze_group_nfa(fsm, ainfo)) { goto fail; } From 50a5fe76b5cc06b77eb7fd915aaaf14622ed7912 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 5 Nov 2025 10:52:00 -0500 Subject: [PATCH 337/373] Restore freeing of state sets copied from linkage_info. --- src/libfsm/union.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 409553d1d..4b07ca6cf 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -664,6 +664,8 @@ rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) static void free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) { + state_set_free(ainfo->anchored_starts); + state_set_free(ainfo->anchored_ends); state_set_free(ainfo->eager_matches); state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); state_set_free(ainfo->reachable_from_nonstart_state); From c65d68c9d7802f5b18c6bf7cfa953ad8c46ad3e8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Thu, 20 Nov 2025 16:12:41 -0500 Subject: [PATCH 338/373] Update stale comment. This wasn't updated to mention RE_SAVE_LINKAGE_INFO. --- include/fsm/bool.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/fsm/bool.h b/include/fsm/bool.h index 25d456b55..150d63f5d 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -57,11 +57,12 @@ fsm_union_array(size_t fsm_count, * eager outputs can match. Ownership of the NFAs is transferred, they will * be combined (or freed, if they don't have a start state). * - * This MUST be called with NFAs constructed via re_comp, Calling it - * with manually constructed NFAs or DFAs may lead to incorrect loop - * linking, because in a few ambiguous cases (e.g. `$|^`) it relies on - * internal details of re_comp's normal construction to correctly - * identify the state representing the unanchored start loop. + * This must be called with NFAs constructed via re_comp, using its + * RE_SAVE_LINKAGE_INFO flag. That saves details during construction + * that are necessary to correctly handle anchoring while linking + * them into the combined NFA. If any of the NFAs do not have that + * information populated, the whole set will be rejected and it + * will return NULL. * * This will set end IDs and/or output IDs representing matching each * of the original NFAs on the combined result, where nfas[i] will From 8debf240a40242333e177377e36c4912818d8ab8 Mon Sep 17 00:00:00 2001 From: Grace Susanto Date: Tue, 25 Nov 2025 14:02:12 +0000 Subject: [PATCH 339/373] Add documentation guide on how to use Libfsm effectively --- README.md | 6 ++ doc/GUIDE.md | 186 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 doc/GUIDE.md diff --git a/README.md b/README.md index 545725fe9..d72b4fece 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,12 @@ lx is an attempt to produce a simple, expressive, and unobtrusive lexer generator which is good at lexing, does just lexing, is language independent, and has no other features. +### Performance and Requirements + +libfsm is not a drop-in replacement for other regex engines and it only supports patterns that can be compiled to deterministic FSMs. In return, supported patterns run in linear time. + +**→ See [doc/GUIDE.md](doc/GUIDE.md) for detailed unsupported features, usage patterns, and examples.** + ### Building from source Clone with submodules (contains required .mk files): diff --git a/doc/GUIDE.md b/doc/GUIDE.md new file mode 100644 index 000000000..3fe458c41 --- /dev/null +++ b/doc/GUIDE.md @@ -0,0 +1,186 @@ +# Using libfsm for High-Performance Pattern Matching + +libfsm compiles regular expressions to deterministic finite state machines (FSMs) and generates executable code. FSM-based matching runs in **linear time O(n)** with **no backtracking**. + +**libfsm is not a drop-in replacement for traditional regex engines.** It only supports patterns that can be compiled to FSMs. + +## What libfsm Cannot Do + +These PCRE features will not compile: + +* Word boundaries (`\b`) +* Non-greedy quantifiers (`*?`, `+?`, `??`) +* Group capture and backreferences +* Lookahead/lookbehind assertions (`(?=`, `(?!`, `(?<=`, `(? user_detector.go +``` + +This produces a standalone matcher function. + +--- + +## Supported Code Generation Targets + +libfsm provides stable, “first-class” code generation for: + +| Category | Output | +| -------------------- | ------------------------------ | +| High-level languages | **C (via `-l vmc`), Go, Rust** | +| Toolchains | **LLVM IR** | +| Virtualization | **Native WebAssembly** | + +> Adding code generation for new languages is template-driven and straightforward. + +--- + +## Workflow Overview + +libfsm provides two main tools: **`re`** takes patterns from command line, **`rx`** takes patterns from file. + +### 1. Validate the Regex + +Test behavior using any PCRE-compatible tool (e.g., [https://regex101.com/](https://regex101.com/)). + +### 2. Verify libfsm Compatibility + +```bash +re -r pcre -l ast 'x*?' +# Output: /x*?/:3: Unsupported operator + +rx -r pcre -l ast -d declined.txt 'x*?' +# Unsupported character in declined.txt +``` + +If unsupported constructs exist, libfsm reports the failing location. + +### 3. Generate Code + +```bash +re -p -r pcre -l rust -k str '^item-[A-Z]{3}$' > item_detector.rs +``` + +### 4. Multiple Patterns + +```bash +# re - patterns from command line: +re -p -r pcre -l go -k str '^x?a b+c$' '^x*def?$' '^x$' + +# rx - patterns from file: +rx -p -r pcre -l vmc -k str -d skipped.txt patterns.txt > detectors.c +``` + +Both tools: +* Combine all patterns into one function (like using `|` to join them) +* Return `(bool, int)` - match status and pattern ID +* Pattern ID is argument position for `re`, line number for `rx` +* When encountering unsupported patterns: `rx` skips them to `-d` file and generates code with working patterns; `re` fails completely + +--- + +### Flag Reference +| Flag | Purpose | Common Options | Notes | +| ---- | --------------------------- | ------------------------------------------ | ------------------------------------------ | +| `-r` | Select regex dialect | `pcre`, `literal`, `glob`, `native`, `sql` | `pcre` supports the widest set of features | +| `-l` | Choose output language | `go`, `rust`, `vmc`, `llvm`, `wasm`, `dot` | Use `vmc` for `C` code, pipe `dot` into `idot` for visualization | +| `-k` | Generated function I/O API | `str`, `getc`, `pair` | `str` takes string, `pair` takes byte array, `getc` uses callback for streaming | +| `-p` | Production mode | *(no value)* | Generates optimized code | +| `-d` | Output unsupported patterns | filename | Only applies to `rx` (batch mode) | + +For more detailed information on flags, see [include/fsm/options.h](../include/fsm/options.h) and the man pages (by running `build/man/re.1/re.1` after `bmake doc`). + +--- + +## Writing Effective libfsm Patterns + +For additional regex best practices, see [Fastly's regex guide](https://www.fastly.com/documentation/reference/vcl/regex/#best-practices-and-common-mistakes). + +### 1. Replace Broad Wildcards + +Avoid `.*` whenever possible. Use negated character classes: + +| Avoid | Better | +| ---------- | -------------- | +| `<.*>` | `<[^>]*>` | +| `\((.*)\)` | `\([^)]*\)` | +| `price=.*` | `price=[0-9]+` | + +--- + +### 2. Anchor When You Require Full Matches + +FSMs only do what’s specified. Explicitly anchor when matching entire strings: + +```regex +^task-[a-z]+-[0-9]{2}\z +``` + +Use `\z` for end-of-string. + +--- + +## Byte Search Optimization (Optional) + +Patterns that start with an **uncommon character** can be accelerated using an initial byte scan before running the FSM. +This quickly jumps to likely match positions instead of scanning every byte. + +### Common fast byte search APIs + +| Language | Function | +| -------- | -------------------------- | +| Go | `strings.IndexByte` | +| Rust | `memchr::memchr` | +| C | `memchr` from `` | + + +### Good candidates + +Patterns that always start with uncommon prefix characters, for example: + +``` +#tag-[a-z]+ +@user-[0-9]+ +\[section\] +{"key": +"name='[^']+'" +``` + +These prefixes (`#`, `@`, `[`, `{`, `'`, `"`) rarely appear in normal text, making a byte search highly effective. + +--- + +## Troubleshooting + +### Pattern Matches Empty String Unintentionally + +Pattern: + +```regex +\s* +``` + +Will compile to code that always returns true. + +**Fix options:** + +* Require at least one match: `\s+` +* Anchor context: `^\s+$` +* Or alternatively, use `-Fb` flag + +### Compilation Takes Too Long + +Likely caused by unrestricted wildcards (`.*`, `.+`). Fix with: + +* Negated classes (`[^)]*`) +* Bounded repeats (`{0,50}`) +* Pattern splitting From 7306570e3b1556c9ab8a216463753f893797ec51 Mon Sep 17 00:00:00 2001 From: Grace Susanto Date: Thu, 27 Nov 2025 12:16:28 +0000 Subject: [PATCH 340/373] Update docs based on reviews --- doc/GUIDE.md | 156 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 39 deletions(-) diff --git a/doc/GUIDE.md b/doc/GUIDE.md index 3fe458c41..ff7abbc7d 100644 --- a/doc/GUIDE.md +++ b/doc/GUIDE.md @@ -2,15 +2,33 @@ libfsm compiles regular expressions to deterministic finite state machines (FSMs) and generates executable code. FSM-based matching runs in **linear time O(n)** with **no backtracking**. +> Regex engines like PCRE use backtracking to explore multiple possible match paths at **runtime**. +> This means the same pattern can have different execution costs depending on the input. +> +> libfsm instead resolves all match decisions at **compile time** by constructing a Deterministic Finite Automaton (DFA). +> At runtime, matching is a single linear pass over the input with no alternative paths to explore. +> +> As a result, libfsm avoids input-dependent slowdowns and is not susceptible to regular expression–based denial-of-service (ReDoS) attacks. + **libfsm is not a drop-in replacement for traditional regex engines.** It only supports patterns that can be compiled to FSMs. +### **Topics** + +- [What libfsm Cannot Do](#what-libfsm-cannot-do) +- [Quick Start](#quick-start) +- [Supported Code Generation Targets](#supported-code-generation-targets) +- [Workflow Overview](#workflow-overview) +- [Writing Effective libfsm Patterns](#writing-effective-libfsm-patterns) +- [Byte Search Optimization (Optional)](#byte-search-optimization-optional) +- [Troubleshooting](#troubleshooting) + ## What libfsm Cannot Do These PCRE features will not compile: * Word boundaries (`\b`) * Non-greedy quantifiers (`*?`, `+?`, `??`) -* Group capture and backreferences +* Group capture (coming soon!) and backreferences * Lookahead/lookbehind assertions (`(?=`, `(?!`, `(?<=`, `(? Adding code generation for new languages is template-driven and straightforward. +> Adding code generation for new languages is straightforward and is defined in [src/libfsm/print/](../src/libfsm/print/). --- ## Workflow Overview -libfsm provides two main tools: **`re`** takes patterns from command line, **`rx`** takes patterns from file. +libfsm provides two main tools: + - **`re`** takes patterns from command line + - **`rx`** takes patterns from file + +A recommended workflow when using libfsm is: ### 1. Validate the Regex -Test behavior using any PCRE-compatible tool (e.g., [https://regex101.com/](https://regex101.com/)). +Test behavior using any PCRE-compatible tool (e.g., [pcregrep(1)](https://man7.org/linux/man-pages/man1/pcregrep.1.html) on the CLI or [https://regex101.com/](https://regex101.com/) in the browser). ### 2. Verify libfsm Compatibility ```bash re -r pcre -l ast 'x*?' # Output: /x*?/:3: Unsupported operator +# :3 indicates that the character at offset 3 in the pattern is rejected. rx -r pcre -l ast -d declined.txt 'x*?' # Unsupported character in declined.txt @@ -67,7 +90,7 @@ If unsupported constructs exist, libfsm reports the failing location. ### 3. Generate Code ```bash -re -p -r pcre -l rust -k str '^item-[A-Z]{3}$' > item_detector.rs +re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs ``` ### 4. Multiple Patterns @@ -89,63 +112,112 @@ Both tools: --- ### Flag Reference -| Flag | Purpose | Common Options | Notes | -| ---- | --------------------------- | ------------------------------------------ | ------------------------------------------ | -| `-r` | Select regex dialect | `pcre`, `literal`, `glob`, `native`, `sql` | `pcre` supports the widest set of features | -| `-l` | Choose output language | `go`, `rust`, `vmc`, `llvm`, `wasm`, `dot` | Use `vmc` for `C` code, pipe `dot` into `idot` for visualization | -| `-k` | Generated function I/O API | `str`, `getc`, `pair` | `str` takes string, `pair` takes byte array, `getc` uses callback for streaming | -| `-p` | Production mode | *(no value)* | Generates optimized code | -| `-d` | Output unsupported patterns | filename | Only applies to `rx` (batch mode) | +| Flag | Purpose | Common Options | Notes | +| ---- | ---------------------------- | ------------------------------------------ | ---------------------------------------------------------------- | +| `-r` | Regex dialect | `pcre`, `literal`, `glob`, `native`, `sql` | `pcre` supports the widest set of features | +| `-l` | Output language for printing | `go`, `rust`, `vmc`, `llvm`, `wasm`, `dot` | Use `vmc` for `C` code. Pipe `dot` into `idot` for visualization | +| `-k` | Generated function I/O API | `str`, `getc`, `pair` | `str` takes string, `pair` takes byte array, `getc` uses callback for streaming | +| `-p` | Print mode | *(no value)* | Abbrv. of `-l fsm`. Print the constructed fsm, rather than executing it. | +| `-d` | Declined | filename | Only applies to `rx` (batch mode) | -For more detailed information on flags, see [include/fsm/options.h](../include/fsm/options.h) and the man pages (by running `build/man/re.1/re.1` after `bmake doc`). +This is not exhausted list. For full flag details, see [include/fsm/options.h](../include/fsm/options.h) and the [man pages](../man). +The man pages can be built by running `bmake doc`, then view with `build/man/re.1/re.1`. --- ## Writing Effective libfsm Patterns -For additional regex best practices, see [Fastly's regex guide](https://www.fastly.com/documentation/reference/vcl/regex/#best-practices-and-common-mistakes). - ### 1. Replace Broad Wildcards -Avoid `.*` whenever possible. Use negated character classes: +Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise and forces libfsm to build a large DFA. + +For example, a double-quoted string should not use `".*?"` because the content cannot contain an unescaped quote. +Instead, restrict it to the actual valid characters `"[^"\r\n]*"`, which matches only what is allowd and will keep the DFA more compact. + +Use negated character classes: | Avoid | Better | | ---------- | -------------- | | `<.*>` | `<[^>]*>` | | `\((.*)\)` | `\([^)]*\)` | -| `price=.*` | `price=[0-9]+` | +| `price=.+` | `price=[0-9]+` | +| `var\s.+=` | `var\s[^=]+=` | + +> This is often the cause of an “explosion” in the size of the generated FSM. +> +> See [Compilation Takes Too Long](#compilation-takes-too-long) for more details. --- -### 2. Anchor When You Require Full Matches +### 2. Anchor When Matching Full String -FSMs only do what’s specified. Explicitly anchor when matching entire strings: +When the intention is to match an entire string, use anchors. +Use `^` at the beginning and `\z` for the true end of the string. ```regex -^task-[a-z]+-[0-9]{2}\z +# Correct: matches only this exact hostname +^web\d+\.example\.com\z  + +# Incorrect: would match inside a larger string +web\d+\.example\.com # also matches "foo-web12.example.com-bar" ``` -Use `\z` for end-of-string. +--- + +### 3. Prefer `\z` Over `$` for End-of-String + +`\z` always matches the end of the string. +`$` will also match a trailing newline at the end of the string, +so if you use this in combination with capturing groups, you may not be capturing what you expect. +Also, `\z` is more efficient, so it is better to use it in places where `\n` cannot appear. + +```regex +# Preferred +/foo\z + +# Risky: $ may allow an extra newline +/foo$ +``` --- -## Byte Search Optimization (Optional) +### 4. Escape Special Characters When Used As Literal -Patterns that start with an **uncommon character** can be accelerated using an initial byte scan before running the FSM. -This quickly jumps to likely match positions instead of scanning every byte. +Many characters have special meaning in regex (for example `.`, `+`, `*`, `?`, `[`, `(`). +If you mean to match them literally, escape them: -### Common fast byte search APIs +| Literal You Want | Correct Regex | Explanation | +|----------------------------|-----------------------------|--------------------------------------------| +| `example.com` | `example\.com` | `.` matches any character unless escaped | +| `a+b` | `a\+b` | `+` means “one or more” | +| `price?` | `price\?` | `?` means “optional” | +| `[value]` | `\[value\]` | `[` and `]` start/end a character class | +| `(test)` | `\(test\)` | `(` and `)` begin/end a group | +| Markdown link `[t](u)` | `(\[[^]]*\]\([^)]*\))` | Matches `[text](url)` without crossing `]` or `)` | -| Language | Function | -| -------- | -------------------------- | -| Go | `strings.IndexByte` | -| Rust | `memchr::memchr` | -| C | `memchr` from `` | +--- +### 5. Use Non-Capturing Groups -### Good candidates +Capture groups are _currently_ not supported (coming soon!). +If you need grouping for alternation or precedence, use non-capturing syntax `(?:...)`: -Patterns that always start with uncommon prefix characters, for example: +```regex +# Correct +(?:private|no-store) + +# Unsupported +(private|no-store) +``` + +--- + +## Byte Search Optimization (Optional) + +Patterns that start with an **uncommon character** can be accelerated using an initial byte scan before running the FSM. +This quickly jumps to likely match positions instead of scanning every byte. + +Good candidates are patterns that start with uncommon prefix characters, for example: ``` #tag-[a-z]+ @@ -155,7 +227,9 @@ Patterns that always start with uncommon prefix characters, for example: "name='[^']+'" ``` -These prefixes (`#`, `@`, `[`, `{`, `'`, `"`) rarely appear in normal text, making a byte search highly effective. +These prefixes (`#`, `@`, `[`, `{`, `'`, `"`) are rare in normal text, so a byte search can skip ahead before running the matcher. + +We found using `strings.IndexByte` before calling the generated matcher in Go code significantly improved performance when matching strings with a large (>5k) leading prefix. --- @@ -171,16 +245,20 @@ Pattern: Will compile to code that always returns true. +This is only an issue if that is not what you intend. + **Fix options:** * Require at least one match: `\s+` -* Anchor context: `^\s+$` -* Or alternatively, use `-Fb` flag +* Anchor context: `^\s+$` or alternatively, use `-Fb` flag ### Compilation Takes Too Long -Likely caused by unrestricted wildcards (`.*`, `.+`). Fix with: +This is often caused by unrestricted wildcards (`.*`, `.+`). +Although they look compact, libfsm must enumerate every possible byte and every possible continuation, causing the state machine to grow quickly. + +For example, to match `var anything =`, a pattern such as `var\s.+=` looks simple, but `.+` forces libfsm to encode every possible byte +and every possible continuation -- including both the presence and absence of `=`. This drastically increases the number of states. -* Negated classes (`[^)]*`) -* Bounded repeats (`{0,50}`) -* Pattern splitting +When compilation is slow, look for broad wildcards and replace them with more specific character classes (as shown [above](#writing-effective-libfsm-patterns)), +such as: `var\s[^=]+=`. From 20f262cbaf40454b32ed1245da99555db84a1347 Mon Sep 17 00:00:00 2001 From: Grace Susanto Date: Thu, 27 Nov 2025 15:30:16 +0000 Subject: [PATCH 341/373] Revision #2 --- doc/GUIDE.md | 121 ++++++++++++++++++++------------------------------- 1 file changed, 46 insertions(+), 75 deletions(-) diff --git a/doc/GUIDE.md b/doc/GUIDE.md index ff7abbc7d..26ea155c6 100644 --- a/doc/GUIDE.md +++ b/doc/GUIDE.md @@ -2,13 +2,13 @@ libfsm compiles regular expressions to deterministic finite state machines (FSMs) and generates executable code. FSM-based matching runs in **linear time O(n)** with **no backtracking**. -> Regex engines like PCRE use backtracking to explore multiple possible match paths at **runtime**. -> This means the same pattern can have different execution costs depending on the input. -> -> libfsm instead resolves all match decisions at **compile time** by constructing a Deterministic Finite Automaton (DFA). -> At runtime, matching is a single linear pass over the input with no alternative paths to explore. -> -> As a result, libfsm avoids input-dependent slowdowns and is not susceptible to regular expression–based denial-of-service (ReDoS) attacks. +Regex engines like PCRE use backtracking to explore multiple possible match paths at **runtime**. +This means the same pattern can have different execution costs depending on the input. + +libfsm instead resolves all match decisions at **compile time** by constructing a Deterministic Finite Automaton (DFA). +At runtime, matching is a single linear pass over the input with no alternative paths to explore. + +As a result, libfsm avoids input-dependent slowdowns and is not susceptible to regular expression–based denial-of-service (ReDoS) attacks. **libfsm is not a drop-in replacement for traditional regex engines.** It only supports patterns that can be compiled to FSMs. @@ -21,6 +21,7 @@ libfsm compiles regular expressions to deterministic finite state machines (FSMs - [Writing Effective libfsm Patterns](#writing-effective-libfsm-patterns) - [Byte Search Optimization (Optional)](#byte-search-optimization-optional) - [Troubleshooting](#troubleshooting) +- [Pattern Matches Empty String Unintentionally](#pattern-matches-empty-string-unintentionally) ## What libfsm Cannot Do @@ -33,8 +34,6 @@ These PCRE features will not compile: * Conditional expressions (`(?(condition)then|else)`) * Recursion and subroutines (`(?R)`, `(?1)`) ---- - ## Quick Start Generate a matcher from a regex: @@ -46,21 +45,14 @@ re -p -r pcre -l go -k str 'user\d+' > user_detector.go This produces a standalone matcher function. ---- - ## Supported Code Generation Targets libfsm provides stable, “first-class” code generation for: +- High-level languages: C (via `-l vmc`), Go, Rust +- LLVM IR +- Native WebAssembly -| Category | Output | -| -------------------- | ------------------------------ | -| High-level languages | **C (via `-l vmc`), Go, Rust** | -| Toolchains | **LLVM IR** | -| Virtualization | **Native WebAssembly** | - -> Adding code generation for new languages is straightforward and is defined in [src/libfsm/print/](../src/libfsm/print/). - ---- +Adding code generation for new languages is straightforward and is defined in [src/libfsm/print/](../src/libfsm/print/). ## Workflow Overview @@ -70,11 +62,11 @@ libfsm provides two main tools: A recommended workflow when using libfsm is: -### 1. Validate the Regex +1. Validate the Regex Test behavior using any PCRE-compatible tool (e.g., [pcregrep(1)](https://man7.org/linux/man-pages/man1/pcregrep.1.html) on the CLI or [https://regex101.com/](https://regex101.com/) in the browser). -### 2. Verify libfsm Compatibility +2. Verify libfsm Compatibility ```bash re -r pcre -l ast 'x*?' @@ -87,13 +79,13 @@ rx -r pcre -l ast -d declined.txt 'x*?' If unsupported constructs exist, libfsm reports the failing location. -### 3. Generate Code +3. Generate Code ```bash re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs ``` -### 4. Multiple Patterns +4. Multiple Patterns ```bash # re - patterns from command line: @@ -109,8 +101,6 @@ Both tools: * Pattern ID is argument position for `re`, line number for `rx` * When encountering unsupported patterns: `rx` skips them to `-d` file and generates code with working patterns; `re` fails completely ---- - ### Flag Reference | Flag | Purpose | Common Options | Notes | | ---- | ---------------------------- | ------------------------------------------ | ---------------------------------------------------------------- | @@ -118,23 +108,22 @@ Both tools: | `-l` | Output language for printing | `go`, `rust`, `vmc`, `llvm`, `wasm`, `dot` | Use `vmc` for `C` code. Pipe `dot` into `idot` for visualization | | `-k` | Generated function I/O API | `str`, `getc`, `pair` | `str` takes string, `pair` takes byte array, `getc` uses callback for streaming | | `-p` | Print mode | *(no value)* | Abbrv. of `-l fsm`. Print the constructed fsm, rather than executing it. | -| `-d` | Declined | filename | Only applies to `rx` (batch mode) | +| `-d` | Declined patterns | filename | Only applies to `rx` (batch mode) | This is not exhausted list. For full flag details, see [include/fsm/options.h](../include/fsm/options.h) and the [man pages](../man). -The man pages can be built by running `bmake doc`, then view with `build/man/re.1/re.1`. - ---- +The man pages can be built by running `bmake -r doc`, then view with `build/man/re.1/re.1`. ## Writing Effective libfsm Patterns -### 1. Replace Broad Wildcards +1. Replace Broad Wildcards + +Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise. And although they look compact, libfsm must enumerate every possible byte and continuation. This quickly leads to large DFAs. -Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise and forces libfsm to build a large DFA. +For example, a double-quoted string should not use `".*"` because the content cannot contain an unescaped quote. Using `.*` forces libfsm to consider all characters -- including both the presence and absence of the closing `"` at every step. This greatly increases the number of states. -For example, a double-quoted string should not use `".*?"` because the content cannot contain an unescaped quote. -Instead, restrict it to the actual valid characters `"[^"\r\n]*"`, which matches only what is allowd and will keep the DFA more compact. +Instead, restrict it to the actual valid characters `"[^"\r\n]*"`, which matches only what is allowed and will keep the DFA more compact. -Use negated character classes: +Use negated character classes to match only the allowed content: | Avoid | Better | | ---------- | -------------- | @@ -143,45 +132,46 @@ Use negated character classes: | `price=.+` | `price=[0-9]+` | | `var\s.+=` | `var\s[^=]+=` | -> This is often the cause of an “explosion” in the size of the generated FSM. -> -> See [Compilation Takes Too Long](#compilation-takes-too-long) for more details. +The overlap between `.*` or `.+` and strings that follow is often the cause of an “explosion” in the size of the generated FSM. So when compilation is slow or generated output is large, look for `.*` and `.+` first and replace them with a narrower character class. ---- - -### 2. Anchor When Matching Full String +2. Anchor When Matching Full String When the intention is to match an entire string, use anchors. Use `^` at the beginning and `\z` for the true end of the string. ```regex # Correct: matches only this exact hostname +# Matches "web12.example.com" +# Does not match "foo-web12.example.com-bar" ^web\d+\.example\.com\z  # Incorrect: would match inside a larger string -web\d+\.example\.com # also matches "foo-web12.example.com-bar" +# Matches "web12.example.com" +# Also matches "foo-web12.example.com-bar" +web\d+\.example\.com ``` ---- - -### 3. Prefer `\z` Over `$` for End-of-String +3. Prefer `\z` Over `$` for End-of-String `\z` always matches the end of the string. `$` will also match a trailing newline at the end of the string, so if you use this in combination with capturing groups, you may not be capturing what you expect. -Also, `\z` is more efficient, so it is better to use it in places where `\n` cannot appear. +Also, `\z` produces a smaller FSM, so it is better to use it in places where `\n` cannot appear. ```regex -# Preferred -/foo\z - -# Risky: $ may allow an extra newline -/foo$ +# Preferred: matches only if the string ends with "bar" +# Matches "/foo/bar" +# Does NOT match "/foo/bar\n" +/bar\z + +# Incorrect: allows a trailing newline, +# which is usually unintended and adds unnecessary complexity +# Matches "/foo/bar" +# Also matches "/foo/bar\n" +/bar$ ``` ---- - -### 4. Escape Special Characters When Used As Literal +4. Escape Special Characters When Used As Literal Many characters have special meaning in regex (for example `.`, `+`, `*`, `?`, `[`, `(`). If you mean to match them literally, escape them: @@ -195,9 +185,7 @@ If you mean to match them literally, escape them: | `(test)` | `\(test\)` | `(` and `)` begin/end a group | | Markdown link `[t](u)` | `(\[[^]]*\]\([^)]*\))` | Matches `[text](url)` without crossing `]` or `)` | ---- - -### 5. Use Non-Capturing Groups +5. Use Non-Capturing Groups Capture groups are _currently_ not supported (coming soon!). If you need grouping for alternation or precedence, use non-capturing syntax `(?:...)`: @@ -210,8 +198,6 @@ If you need grouping for alternation or precedence, use non-capturing syntax `(? (private|no-store) ``` ---- - ## Byte Search Optimization (Optional) Patterns that start with an **uncommon character** can be accelerated using an initial byte scan before running the FSM. @@ -231,11 +217,7 @@ These prefixes (`#`, `@`, `[`, `{`, `'`, `"`) are rare in normal text, so a byte We found using `strings.IndexByte` before calling the generated matcher in Go code significantly improved performance when matching strings with a large (>5k) leading prefix. ---- - -## Troubleshooting - -### Pattern Matches Empty String Unintentionally +## Pattern Matches Empty String Unintentionally Pattern: @@ -251,14 +233,3 @@ This is only an issue if that is not what you intend. * Require at least one match: `\s+` * Anchor context: `^\s+$` or alternatively, use `-Fb` flag - -### Compilation Takes Too Long - -This is often caused by unrestricted wildcards (`.*`, `.+`). -Although they look compact, libfsm must enumerate every possible byte and every possible continuation, causing the state machine to grow quickly. - -For example, to match `var anything =`, a pattern such as `var\s.+=` looks simple, but `.+` forces libfsm to encode every possible byte -and every possible continuation -- including both the presence and absence of `=`. This drastically increases the number of states. - -When compilation is slow, look for broad wildcards and replace them with more specific character classes (as shown [above](#writing-effective-libfsm-patterns)), -such as: `var\s[^=]+=`. From 227f4ac04910ca95913c830858749adb1a310e18 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 27 Nov 2025 15:53:27 +0000 Subject: [PATCH 342/373] Naming. --- README.md | 9 +++------ doc/{GUIDE.md => advice.md} | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) rename doc/{GUIDE.md => advice.md} (99%) diff --git a/README.md b/README.md index d72b4fece..d30e405b1 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,15 @@ ; re -cb -pl dot '[Ll]ibf+(sm)*' '[Ll]ibre' | dot ![libfsm.svg](doc/tutorial/libfsm.svg) +libfsm is not a drop-in replacement for other regex engines, and it only supports patterns that can be compiled to deterministic FSMs. In return, supported patterns run in linear time. + Getting started: * See the [tutorial introduction](doc/tutorial/re.md) for a quick overview of the re(1) command line interface. * [Compilation phases](doc/tutorial/phases.md) for typical applications which compile regular expressions to code. + * [Advice on using libfsm](doc/advice.md) for suggestions around compilation time, unsupported features, common usage patterns, and examples. You get: @@ -24,12 +27,6 @@ lx is an attempt to produce a simple, expressive, and unobtrusive lexer generator which is good at lexing, does just lexing, is language independent, and has no other features. -### Performance and Requirements - -libfsm is not a drop-in replacement for other regex engines and it only supports patterns that can be compiled to deterministic FSMs. In return, supported patterns run in linear time. - -**→ See [doc/GUIDE.md](doc/GUIDE.md) for detailed unsupported features, usage patterns, and examples.** - ### Building from source Clone with submodules (contains required .mk files): diff --git a/doc/GUIDE.md b/doc/advice.md similarity index 99% rename from doc/GUIDE.md rename to doc/advice.md index 26ea155c6..d2dc402b6 100644 --- a/doc/GUIDE.md +++ b/doc/advice.md @@ -1,4 +1,4 @@ -# Using libfsm for High-Performance Pattern Matching +# Advice on using libfsm for high-performance pattern matching libfsm compiles regular expressions to deterministic finite state machines (FSMs) and generates executable code. FSM-based matching runs in **linear time O(n)** with **no backtracking**. From c4d4ffe9b08e70103db626552bb0d638932d07c3 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 27 Nov 2025 16:09:46 +0000 Subject: [PATCH 343/373] Markup. --- doc/advice.md | 187 +++++++++++++++++++++++++------------------------- 1 file changed, 95 insertions(+), 92 deletions(-) diff --git a/doc/advice.md b/doc/advice.md index d2dc402b6..dff982acc 100644 --- a/doc/advice.md +++ b/doc/advice.md @@ -19,7 +19,7 @@ As a result, libfsm avoids input-dependent slowdowns and is not susceptible to r - [Supported Code Generation Targets](#supported-code-generation-targets) - [Workflow Overview](#workflow-overview) - [Writing Effective libfsm Patterns](#writing-effective-libfsm-patterns) -- [Byte Search Optimization (Optional)](#byte-search-optimization-optional) +- [Byte Search Optimization](#byte-search-optimization-optional) - [Troubleshooting](#troubleshooting) - [Pattern Matches Empty String Unintentionally](#pattern-matches-empty-string-unintentionally) @@ -38,7 +38,7 @@ These PCRE features will not compile: Generate a matcher from a regex: -```bash +```sh # Generate a Go matcher re -p -r pcre -l go -k str 'user\d+' > user_detector.go ``` @@ -56,151 +56,154 @@ Adding code generation for new languages is straightforward and is defined in [s ## Workflow Overview -libfsm provides two main tools: - - **`re`** takes patterns from command line - - **`rx`** takes patterns from file +libfsm provides two main tools for pattern matching: + - **`re`** takes patterns from the command line + - **`rx`** takes patterns from a file A recommended workflow when using libfsm is: 1. Validate the Regex -Test behavior using any PCRE-compatible tool (e.g., [pcregrep(1)](https://man7.org/linux/man-pages/man1/pcregrep.1.html) on the CLI or [https://regex101.com/](https://regex101.com/) in the browser). + Test behavior using any PCRE-compatible tool (e.g., [pcregrep(1)](https://man7.org/linux/man-pages/man1/pcregrep.1.html) on the CLI or [https://regex101.com/](https://regex101.com/) in the browser). 2. Verify libfsm Compatibility -```bash -re -r pcre -l ast 'x*?' -# Output: /x*?/:3: Unsupported operator -# :3 indicates that the character at offset 3 in the pattern is rejected. + If unsupported constructs exist, libfsm reports the failing location: + ```sh + re -r pcre -l ast 'x*?' + # Output: /x*?/:3: Unsupported operator + ``` + In this example, `:3` indicates that the character at byte offset three in the pattern is an unsupported feature. -rx -r pcre -l ast -d declined.txt 'x*?' -# Unsupported character in declined.txt -``` + ```sh + # patterns with unsupported operators are output to declined.txt + rx -r pcre -l ast -d declined.txt 'x*?' + ``` -If unsupported constructs exist, libfsm reports the failing location. 3. Generate Code -```bash -re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs -``` + ```sh + re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs + ``` 4. Multiple Patterns -```bash -# re - patterns from command line: -re -p -r pcre -l go -k str '^x?a b+c$' '^x*def?$' '^x$' - -# rx - patterns from file: -rx -p -r pcre -l vmc -k str -d skipped.txt patterns.txt > detectors.c -``` + ```sh + # re - patterns from command line: + re -p -r pcre -l go -k str '^x?a b+c$' '^x*def?$' '^x$' + + # rx - patterns from file: + rx -p -r pcre -l vmc -k str -d skipped.txt patterns.txt > detectors.c + ``` Both tools: * Combine all patterns into one function (like using `|` to join them) -* Return `(bool, int)` - match status and pattern ID +* Generate code that can return `(bool, int)` for the match status and pattern ID * Pattern ID is argument position for `re`, line number for `rx` -* When encountering unsupported patterns: `rx` skips them to `-d` file and generates code with working patterns; `re` fails completely +* When encountering unsupported patterns: `rx` can decline them to `-d` file and generates code with working patterns only; `re` fails completely + +### Common Flags -### Flag Reference | Flag | Purpose | Common Options | Notes | -| ---- | ---------------------------- | ------------------------------------------ | ---------------------------------------------------------------- | +|:----:|:---------------------------- |:------------------------------------------ |:---------------------------------------------------------------- | | `-r` | Regex dialect | `pcre`, `literal`, `glob`, `native`, `sql` | `pcre` supports the widest set of features | | `-l` | Output language for printing | `go`, `rust`, `vmc`, `llvm`, `wasm`, `dot` | Use `vmc` for `C` code. Pipe `dot` into `idot` for visualization | | `-k` | Generated function I/O API | `str`, `getc`, `pair` | `str` takes string, `pair` takes byte array, `getc` uses callback for streaming | | `-p` | Print mode | *(no value)* | Abbrv. of `-l fsm`. Print the constructed fsm, rather than executing it. | | `-d` | Declined patterns | filename | Only applies to `rx` (batch mode) | -This is not exhausted list. For full flag details, see [include/fsm/options.h](../include/fsm/options.h) and the [man pages](../man). +This is not an exhaustive list. For full flag details, see [include/fsm/options.h](../include/fsm/options.h) and the [man pages](../man). The man pages can be built by running `bmake -r doc`, then view with `build/man/re.1/re.1`. ## Writing Effective libfsm Patterns 1. Replace Broad Wildcards -Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise. And although they look compact, libfsm must enumerate every possible byte and continuation. This quickly leads to large DFAs. + Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise. And although they look compact, libfsm must enumerate every possible byte and continuation. This quickly leads to large DFAs. -For example, a double-quoted string should not use `".*"` because the content cannot contain an unescaped quote. Using `.*` forces libfsm to consider all characters -- including both the presence and absence of the closing `"` at every step. This greatly increases the number of states. + For example, a double-quoted string should not use `".*"` because the content cannot contain an unescaped quote. Using `.*` forces libfsm to consider all characters -- including both the presence and absence of the closing `"` at every step. This greatly increases the number of states. -Instead, restrict it to the actual valid characters `"[^"\r\n]*"`, which matches only what is allowed and will keep the DFA more compact. + Instead, restrict it to the actual valid characters `"[^"\r\n]*"`, which matches only what is allowed and will keep the DFA more compact. -Use negated character classes to match only the allowed content: + Use negated character classes to match only the allowed content: -| Avoid | Better | -| ---------- | -------------- | -| `<.*>` | `<[^>]*>` | -| `\((.*)\)` | `\([^)]*\)` | -| `price=.+` | `price=[0-9]+` | -| `var\s.+=` | `var\s[^=]+=` | + | Avoid | Better | + | ---------- | -------------- | + | `<.*>` | `<[^>]*>` | + | `\((.*)\)` | `\([^)]*\)`| + | `price=.+` | `price=[0-9]+` | + | `var\s.+=` | `var\s[^=]+=` | -The overlap between `.*` or `.+` and strings that follow is often the cause of an “explosion” in the size of the generated FSM. So when compilation is slow or generated output is large, look for `.*` and `.+` first and replace them with a narrower character class. + The overlap between `.*` or `.+` and strings that follow is often the cause of an “explosion” in the size of the generated FSM. So when compilation is slow or generated output is large, look for `.*` and `.+` first and replace them with a narrower character class. 2. Anchor When Matching Full String -When the intention is to match an entire string, use anchors. -Use `^` at the beginning and `\z` for the true end of the string. - -```regex -# Correct: matches only this exact hostname -# Matches "web12.example.com" -# Does not match "foo-web12.example.com-bar" -^web\d+\.example\.com\z  - -# Incorrect: would match inside a larger string -# Matches "web12.example.com" -# Also matches "foo-web12.example.com-bar" -web\d+\.example\.com -``` + When the intention is to match an entire string, use anchors. + Use `^` at the beginning and `\z` for the true end of the string. + + ```regex + # Correct: matches only this exact hostname + # Matches "web12.example.com" + # Does not match "foo-web12.example.com-bar" + ^web\d+\.example\.com\z  + + # Incorrect: would match inside a larger string + # Matches "web12.example.com" + # Also matches "foo-web12.example.com-bar" + web\d+\.example\.com + ``` 3. Prefer `\z` Over `$` for End-of-String -`\z` always matches the end of the string. -`$` will also match a trailing newline at the end of the string, -so if you use this in combination with capturing groups, you may not be capturing what you expect. -Also, `\z` produces a smaller FSM, so it is better to use it in places where `\n` cannot appear. - -```regex -# Preferred: matches only if the string ends with "bar" -# Matches "/foo/bar" -# Does NOT match "/foo/bar\n" -/bar\z - -# Incorrect: allows a trailing newline, -# which is usually unintended and adds unnecessary complexity -# Matches "/foo/bar" -# Also matches "/foo/bar\n" -/bar$ -``` + `\z` always matches the end of the string. + `$` will also match a trailing newline at the end of the string, + so if you use this in combination with capturing groups, you may not be capturing what you expect. + Also, `\z` produces a smaller FSM, so it is better to use it in places where `\n` cannot appear. + + ```regex + # Preferred: matches only if the string ends with "bar" + # Matches "/foo/bar" + # Does NOT match "/foo/bar\n" + /bar\z + + # Incorrect: allows a trailing newline, + # which is usually unintended and adds unnecessary complexity + # Matches "/foo/bar" + # Also matches "/foo/bar\n" + /bar$ + ``` 4. Escape Special Characters When Used As Literal -Many characters have special meaning in regex (for example `.`, `+`, `*`, `?`, `[`, `(`). -If you mean to match them literally, escape them: + Many characters have special meaning in regex (for example `.`, `+`, `*`, `?`, `[`, `(`). + If you mean to match them literally, escape them: -| Literal You Want | Correct Regex | Explanation | -|----------------------------|-----------------------------|--------------------------------------------| -| `example.com` | `example\.com` | `.` matches any character unless escaped | -| `a+b` | `a\+b` | `+` means “one or more” | -| `price?` | `price\?` | `?` means “optional” | -| `[value]` | `\[value\]` | `[` and `]` start/end a character class | -| `(test)` | `\(test\)` | `(` and `)` begin/end a group | -| Markdown link `[t](u)` | `(\[[^]]*\]\([^)]*\))` | Matches `[text](url)` without crossing `]` or `)` | + | Literal You Want | Correct Regex | Explanation | + |----------------------------|-----------------------------|--------------------------------------------| + | `example.com` | `example\.com` | `.` matches any character unless escaped | + | `a+b` | `a\+b` | `+` means “one or more” | + | `price?` | `price\?` | `?` means “optional” | + | `[value]` | `\[value\]` | `[` and `]` start/end a character class | + | `(test)` | `\(test\)` | `(` and `)` begin/end a group | + | Markdown link `[t](u)` | `(\[[^]]*\]\([^)]*\))` | Matches `[text](url)` without crossing `]` or `)` | 5. Use Non-Capturing Groups -Capture groups are _currently_ not supported (coming soon!). -If you need grouping for alternation or precedence, use non-capturing syntax `(?:...)`: + Capture groups are _currently_ not supported (coming soon!). + If you need grouping for alternation or precedence, use non-capturing syntax `(?:...)`: -```regex -# Correct -(?:private|no-store) - -# Unsupported -(private|no-store) -``` + ```regex + # Correct + (?:private|no-store) + + # Unsupported + (private|no-store) + ``` -## Byte Search Optimization (Optional) +## Byte Search Optimization -Patterns that start with an **uncommon character** can be accelerated using an initial byte scan before running the FSM. +Patterns that start with an uncommon character can be accelerated using an initial byte scan before running the FSM. This quickly jumps to likely match positions instead of scanning every byte. Good candidates are patterns that start with uncommon prefix characters, for example: From bf867f2eb6c45f81c4e3bbb2b07330e5fded396b Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 27 Nov 2025 16:21:30 +0000 Subject: [PATCH 344/373] Blurb on calling the generated code. --- doc/advice.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/doc/advice.md b/doc/advice.md index dff982acc..38c61d8c9 100644 --- a/doc/advice.md +++ b/doc/advice.md @@ -62,11 +62,11 @@ libfsm provides two main tools for pattern matching: A recommended workflow when using libfsm is: -1. Validate the Regex +1. Validate the regex Test behavior using any PCRE-compatible tool (e.g., [pcregrep(1)](https://man7.org/linux/man-pages/man1/pcregrep.1.html) on the CLI or [https://regex101.com/](https://regex101.com/) in the browser). -2. Verify libfsm Compatibility +2. Verify libfsm compatibility If unsupported constructs exist, libfsm reports the failing location: ```sh @@ -81,13 +81,13 @@ A recommended workflow when using libfsm is: ``` -3. Generate Code +3. Generate code ```sh re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs ``` -4. Multiple Patterns +4. Multiple patterns ```sh # re - patterns from command line: @@ -97,6 +97,12 @@ A recommended workflow when using libfsm is: rx -p -r pcre -l vmc -k str -d skipped.txt patterns.txt > detectors.c ``` +5. Call the generated code from your program somehow + + You're on your own for this. `-k` controls the API for the generated code to read in data to match. Try different options for the language you're using and see which suits you. + + The generated API can also vary depending on how you want libfsm to handle ambiguities between different patterns. See the `AMBIG_*` flags in [include/fsm/options.h](../include/fsm/options.h) for different approaches there. + Both tools: * Combine all patterns into one function (like using `|` to join them) * Generate code that can return `(bool, int)` for the match status and pattern ID @@ -114,7 +120,7 @@ Both tools: | `-d` | Declined patterns | filename | Only applies to `rx` (batch mode) | This is not an exhaustive list. For full flag details, see [include/fsm/options.h](../include/fsm/options.h) and the [man pages](../man). -The man pages can be built by running `bmake -r doc`, then view with `build/man/re.1/re.1`. +The man pages can be built by running `bmake -r doc`, then view with `man build/man/re.1/re.1`. ## Writing Effective libfsm Patterns From 4351273fe990941fc4e80c231e1018140ffe177a Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 27 Nov 2025 16:50:16 +0000 Subject: [PATCH 345/373] Blurb on bounded repetition. (Contributed by Scott) --- doc/advice.md | 50 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/doc/advice.md b/doc/advice.md index 38c61d8c9..577388da5 100644 --- a/doc/advice.md +++ b/doc/advice.md @@ -87,7 +87,13 @@ A recommended workflow when using libfsm is: re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs ``` -4. Multiple patterns +4. Use multiple patterns + + Execution complexity for the generated code is proportional to the length of the text being matched, not to the number of patterns. + Assuming your generated code isn't too large to compile, this means you can have as many patterns as you want, + for the same time it takes to execute a single pattern. + + Take advantage of this. ```sh # re - patterns from command line: @@ -124,7 +130,19 @@ The man pages can be built by running `bmake -r doc`, then view with `man build/ ## Writing Effective libfsm Patterns -1. Replace Broad Wildcards +Generally, to keep generated code compact, stick to the least expressive subset of features. + +libfsm has no way to know in advance what text you'll be passing to its generated code. +For example, are you matching a string that you know will never contain a newline? +libfsm doesn't know that. +It has to generate code that's capable of handling any input. +You can help it out by making your patterns precise. + +Think about what you intend your pattern to match, and what it's actually capable of matching given arbitrary text. +This helps restrict the scope of your pattern from arbitrary text to exactly what you mean. +The following bits of advice illustrate various specific ways to bring down this scope. + +1. Replace broad wildcards Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise. And although they look compact, libfsm must enumerate every possible byte and continuation. This quickly leads to large DFAs. @@ -143,7 +161,14 @@ The man pages can be built by running `bmake -r doc`, then view with `man build/ The overlap between `.*` or `.+` and strings that follow is often the cause of an “explosion” in the size of the generated FSM. So when compilation is slow or generated output is large, look for `.*` and `.+` first and replace them with a narrower character class. -2. Anchor When Matching Full String +2. Take care with bounded repetition + + If you have the pattern ^x{3,5}$, libfsm's resulting DFA will be structured like "match an x, then match an x, then match an x, then match an x or skip it, then match an x or skip it, then report an overall match if at the end of input". It has to repeat the pattern, noting each time whether it's required or optional (beyond the lower count in {min,max}), because DFA execution doesn't have a counter, just the current state within the overall DFA. + + When the subexpression (represented by `x`) unintentionally matches too many things, they all have to be spelled out every time. + So pay especially close attention to tightening up subexpressions in bounded repetition clauses. + +3. Anchor when matching full string When the intention is to match an entire string, use anchors. Use `^` at the beginning and `\z` for the true end of the string. @@ -160,7 +185,7 @@ The man pages can be built by running `bmake -r doc`, then view with `man build/ web\d+\.example\.com ``` -3. Prefer `\z` Over `$` for End-of-String +4. Prefer `\z` over `$` for End-of-String `\z` always matches the end of the string. `$` will also match a trailing newline at the end of the string, @@ -180,7 +205,7 @@ The man pages can be built by running `bmake -r doc`, then view with `man build/ /bar$ ``` -4. Escape Special Characters When Used As Literal +5. Escape special characters when used as literals Many characters have special meaning in regex (for example `.`, `+`, `*`, `?`, `[`, `(`). If you mean to match them literally, escape them: @@ -194,16 +219,23 @@ The man pages can be built by running `bmake -r doc`, then view with `man build/ | `(test)` | `\(test\)` | `(` and `)` begin/end a group | | Markdown link `[t](u)` | `(\[[^]]*\]\([^)]*\))` | Matches `[text](url)` without crossing `]` or `)` | -5. Use Non-Capturing Groups + The `.` wildcard in particular is often mistakenly left unescaped in practice. + On testing, it will match a literal `.` as intended. But it will also match any other character. + This means that not only is your pattern incorrect (write negative test cases!), + but also this part of your FSM is 256 times larger than it should be. + +6. Use non-capturing groups Capture groups are _currently_ not supported (coming soon!). - If you need grouping for alternation or precedence, use non-capturing syntax `(?:...)`: + + If you don't need to capture things, don't use capture. + If you need grouping for alternation or precedence, use PCRE's non-capturing syntax `(?:...)`: ```regex # Correct (?:private|no-store) - # Unsupported + # Not what's intended (private|no-store) ``` @@ -214,7 +246,7 @@ This quickly jumps to likely match positions instead of scanning every byte. Good candidates are patterns that start with uncommon prefix characters, for example: -``` +```regex #tag-[a-z]+ @user-[0-9]+ \[section\] From d8ab92e5834bf8e3a5bcd0f7ac6e3b94ffe5a639 Mon Sep 17 00:00:00 2001 From: Kate F Date: Thu, 27 Nov 2025 17:02:33 +0000 Subject: [PATCH 346/373] Markup. --- doc/advice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/advice.md b/doc/advice.md index 577388da5..0eb429f94 100644 --- a/doc/advice.md +++ b/doc/advice.md @@ -163,7 +163,7 @@ The following bits of advice illustrate various specific ways to bring down this 2. Take care with bounded repetition - If you have the pattern ^x{3,5}$, libfsm's resulting DFA will be structured like "match an x, then match an x, then match an x, then match an x or skip it, then match an x or skip it, then report an overall match if at the end of input". It has to repeat the pattern, noting each time whether it's required or optional (beyond the lower count in {min,max}), because DFA execution doesn't have a counter, just the current state within the overall DFA. + If you have the pattern `^x{3,5}$`, libfsm's resulting DFA will be structured like "match an x, then match an x, then match an x, then match an x or skip it, then match an x or skip it, then report an overall match if at the end of input". It has to repeat the pattern, noting each time whether it's required or optional (beyond the lower count in `{min,max}`), because DFA execution doesn't have a counter, just the current state within the overall DFA. When the subexpression (represented by `x`) unintentionally matches too many things, they all have to be spelled out every time. So pay especially close attention to tightening up subexpressions in bounded repetition clauses. From e6683daeb5bd8a69e1e69d4546d9c3f244d34e4d Mon Sep 17 00:00:00 2001 From: Kate F Date: Fri, 16 Jan 2026 17:05:07 +0000 Subject: [PATCH 347/373] First cut at re_interpolate_groups() --- Makefile | 1 + include/re/groups.h | 46 ++++++ src/libre/Makefile | 1 + src/libre/libre.syms | 1 + src/libre/re_interpolate_groups.c | 147 ++++++++++++++++++ tests/re_interpolate_groups/Makefile | 23 +++ .../re_interpolate_groups0.c | 73 +++++++++ .../re_interpolate_groups1.c | 63 ++++++++ 8 files changed, 355 insertions(+) create mode 100644 include/re/groups.h create mode 100644 src/libre/re_interpolate_groups.c create mode 100644 tests/re_interpolate_groups/Makefile create mode 100644 tests/re_interpolate_groups/re_interpolate_groups0.c create mode 100644 tests/re_interpolate_groups/re_interpolate_groups1.c diff --git a/Makefile b/Makefile index ad16b1ec7..4065c28b0 100644 --- a/Makefile +++ b/Makefile @@ -147,6 +147,7 @@ SUBDIR += tests/sql SUBDIR += tests/queue SUBDIR += tests/aho_corasick SUBDIR += tests/retest +SUBDIR += tests/re_interpolate_groups SUBDIR += tests .if make(theft) || make(${BUILD}/theft/theft) SUBDIR += theft diff --git a/include/re/groups.h b/include/re/groups.h new file mode 100644 index 000000000..0c9a77fb3 --- /dev/null +++ b/include/re/groups.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#ifndef RE_GROUPS_H +#define RE_GROUPS_H + +struct re_pos; + +/* + * esc is the character for escaping group references, + * typically '\\' or '$'. + * + * group0 is passed separately for caller convenience, + * so you don't have to construct a single array for + * all groups. It's supposed to be the entire string + * that matched. group0 may not be NULL. + * + * groupv is 0-indexed meaning group $1 onwards. + * groupc is the count of elements in groupv. + * + * nonexistent is what to do about references to groups + * that are outside the bounds of the array. NULL means + * to error, otherwise the string value will be used. + * Typically this would be passed as "". + * + * You can distinguish compile-time errors (that is, + * syntax errors in the format string) vs. runtime errors + * (that is, nonexistent groups) by calling + * re_interpolate_groups() ahead of time with groupc = 0 + * and passing a non-NULL nonexistent value. + * + * The output string will always be less than or equal in + * length to the format string. The output is \0-terminated. + * outn includes the \0. + */ +bool +re_interpolate_groups(const char *fmt, char esc, + const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, + char *outs, size_t outn, + struct re_pos *pos); + +#endif + diff --git a/src/libre/Makefile b/src/libre/Makefile index 67617ab92..36601004a 100644 --- a/src/libre/Makefile +++ b/src/libre/Makefile @@ -10,6 +10,7 @@ SRC += src/libre/ast_new_from_fsm.c SRC += src/libre/ast_rewrite.c SRC += src/libre/ac.c SRC += src/libre/print.c +SRC += src/libre/re_interpolate_groups.c SRC += src/libre/re_strings.c # generated diff --git a/src/libre/libre.syms b/src/libre/libre.syms index 9d381cb0f..b04833777 100644 --- a/src/libre/libre.syms +++ b/src/libre/libre.syms @@ -4,6 +4,7 @@ re_flags re_strerror re_perror re_is_anchored +re_interpolate_groups ast_print ast_print_dot diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c new file mode 100644 index 000000000..5c9752a41 --- /dev/null +++ b/src/libre/re_interpolate_groups.c @@ -0,0 +1,147 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +// TODO +#define OUT_CHAR(c) do { if (outn < 1) { goto error; } *outs++ = (c); outn--; } while (0) +#define OUT_GROUP(s) do { if (outn < strlen((s))) { goto error; } outs += sprintf(outs, "%s", (s)); outn -= strlen((s)); } while (0) + +// TODO: return values: syntax error, nonexistent group error (digit overflow is the same thing), success +bool +re_interpolate_groups(const char *fmt, char esc, + const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, + char *outs, size_t outn, + struct re_pos *pos) +{ + unsigned group; // 0 meaning group0, 1 meaning groupv[0], etc + const char *p; + + enum { + STATE_LIT, + STATE_ESC, + STATE_DIGIT + } state; + + assert(esc != '\0'); + assert(group0 != NULL || groupc == 0); + assert(groupc < UINT_MAX / 10 - 1); + assert(outs != NULL); + + state = STATE_LIT; + group = 0; + + p = fmt; + do { + switch (state) { + case STATE_LIT: + if (*p == '\0') { + break; + } + + if (*p == esc) { + state = STATE_ESC; + continue; + } + + OUT_CHAR(*p); + continue; + + case STATE_ESC: + if (*p == '\0') { + goto error; + } + + if (*p == esc) { + OUT_CHAR(esc); + state = STATE_LIT; + continue; + } + + if (isdigit((unsigned char) *p)) { + group = *p - '0'; + state = STATE_DIGIT; + continue; + } + + goto error; + + case STATE_DIGIT: + if (isdigit((unsigned char) *p)) { + group *= 10; + group += *p - '0'; + +// TODO: explain this +// digit overflow, we cap to groupc + 1 +// groupc + 1 is always out of bounds +// this is a simple way to avoid needing to handle digit overflow for subsequent digits, +// assuming groupc *= 10 is <= UINT_MAX + if (group > groupc) { + group = groupc + 1; + } + continue; + } + + if (group == 0) { + OUT_GROUP(group0); + } else if (group <= groupc) { + assert(groupv[group - 1] != NULL); + OUT_GROUP(groupv[group - 1]); + } else if (nonexistent == NULL) { +// TODO: maybe want to indicate this independently from syntax errors +// TODO: no need, you can pre-check the entire syntax by running with 0 groups + goto error; + } else { + OUT_GROUP(nonexistent); + } + + group = 0; + state = STATE_LIT; + + if (*p == '\0') { + break; + } + + if (*p == esc) { + state = STATE_ESC; + continue; + } + + OUT_CHAR(*p); + continue; + + default: + assert(!"unreached"); + goto error; + } + } while (*p++); + + if (state != STATE_LIT) { + goto error; + } + + OUT_CHAR('\0'); + + return true; + +error: + + // TODO: track start,end independently + if (pos != NULL) { + pos->byte = p - fmt; + } + + return false; +} + diff --git a/tests/re_interpolate_groups/Makefile b/tests/re_interpolate_groups/Makefile new file mode 100644 index 000000000..41f9e2599 --- /dev/null +++ b/tests/re_interpolate_groups/Makefile @@ -0,0 +1,23 @@ +.include "../../share/mk/top.mk" + +TEST.tests/re_interpolate_groups != ls -1 tests/re_interpolate_groups/re_interpolate_groups*.c +TEST_SRCDIR.tests/re_interpolate_groups = tests/re_interpolate_groups +TEST_OUTDIR.tests/re_interpolate_groups = ${BUILD}/tests/re_interpolate_groups + +.for n in ${TEST.tests/re_interpolate_groups:T:R:C/^re_interpolate_groups//} +test:: ${TEST_OUTDIR.tests/re_interpolate_groups}/res${n} +SRC += ${TEST_SRCDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.c +#CFLAGS.${TEST_SRCDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.c = -UNDEBUG +CFLAGS.${TEST_SRCDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.c = -std=c99 + +${TEST_OUTDIR.tests/re_interpolate_groups}/run${n}: ${TEST_OUTDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/re_interpolate_groups}/run${n} ${TEST_OUTDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.o ${BUILD}/src/libre/re_interpolate_groups.o + +${TEST_OUTDIR.tests/re_interpolate_groups}/res${n}: ${TEST_OUTDIR.tests/re_interpolate_groups}/run${n} + ( ${TEST_OUTDIR.tests/re_interpolate_groups}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/re_interpolate_groups}/res${n} + +#.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +#${TEST_OUTDIR.tests/re_interpolate_groups}/run${n}: ${BUILD}/lib/${lib:R}.a +#.endfor +.endfor + diff --git a/tests/re_interpolate_groups/re_interpolate_groups0.c b/tests/re_interpolate_groups/re_interpolate_groups0.c new file mode 100644 index 000000000..28311afb9 --- /dev/null +++ b/tests/re_interpolate_groups/re_interpolate_groups0.c @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +static unsigned failed; + +static void +test(const char *fmt, size_t groupc, const char *groupv[], const char *expected) +{ + struct re_pos pos; + char outs[40]; + bool r; + + assert(fmt != NULL); + assert(expected != NULL); + + if (!re_interpolate_groups(fmt, '$', "", groupc, groupv, "", outs, sizeof outs, &pos)) { + printf("%s/%zu XXX\n", fmt, groupc); + failed++; + return; + } + + failed += r = 0 != strcmp(outs, expected); + + printf("%s/%zu => %s%s\n", fmt, groupc, outs, + r ? " XXX" : ""); +} + +int main(void) { + const char *gn[] = { "one", "two", "three", "four" }; + const char **g0 = NULL; + const char *ga[] = { "1" }; + const char *gb[] = { "" }; +// const char *gc[] = { NULL }; // XXX: not permitted + + test("", 0, g0, ""); + test("", 4, gn, ""); + + test("x", 0, g0, "x"); + test("x", 4, gn, "x"); + + test("\001", 0, g0, "\001"); + test("\001", 4, gn, "\001"); + + test("$0", 0, gn, ""); + test("x$000000000000000000000x", 0, gn, "xx"); + test("x$000000000000000000001x", 1, gn, "xonex"); + test("x$100000000000000000000x", 1, gn, "xx"); + + test("$$$1$1$2$1$3$4$3$2$1$$$$", 4, gn, "$oneonetwoonethreefourthreetwoone$$"); + + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 4, gn, "xyz_one..three;three,$.one-four="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 3, gn, "xyz_one..three;three,$.one-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 2, gn, "xyz_one..;,$.one-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, gn, "xyz_one..;,$.one-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 0, g0, "xyz_..;,$.-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, ga, "xyz_1..;,$.1-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, gb, "xyz_..;,$.-="); +// test("xyz_$1..$0003;$3,$$.$1-$4=$123", 0, gc, "xyz_..;,$.-="); + + return failed; +} + diff --git a/tests/re_interpolate_groups/re_interpolate_groups1.c b/tests/re_interpolate_groups/re_interpolate_groups1.c new file mode 100644 index 000000000..c75122e1d --- /dev/null +++ b/tests/re_interpolate_groups/re_interpolate_groups1.c @@ -0,0 +1,63 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +static unsigned failed; + +static void +test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, unsigned expected_pos) +{ + struct re_pos pos; + char outs[10]; + bool r; + + assert(fmt != NULL); + + /* for these tests we're expecting to error */ + if (re_interpolate_groups(fmt, '$', "", groupc, groupv, ne, outs, sizeof outs, &pos)) { + printf("%s/%zu XXX\n", fmt, groupc); + failed++; + return; + } + + failed += r = expected_pos != pos.byte; + + printf("%s/%zu => :%u :%u%s\n", fmt, groupc, + pos.byte, expected_pos, + r ? " XXX" : ""); +} + +int main(void) { + const char *ne = ""; + + const char *gn[] = { "one", "two", "three", "four" }; + const char **g0 = NULL; + const char *ga[] = { "1" }; + const char *gb[] = { "" }; +// const char *gc[] = { NULL }; // XXX: not permitted + + test_err("$", 0, g0, ne, 1); + test_err("$x", 0, g0, ne, 1); + test_err("$ ", 4, gn, ne, 1); + test_err("$\\01", 0, g0, ne, 1); + + test_err("$$$x", 4, gn, ne, 3); + + test_err("xyz$1", 0, gn, NULL, 5); + test_err("xyz$2", 1, gn, NULL, 5); + + test_err("01234567890", 1, gn, ne, 10); + + return failed; +} + From 2f0c7722929f0341f782dac1e230bc5a9907933b Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Jan 2026 01:40:27 +0000 Subject: [PATCH 348/373] Add start,end error reporting --- include/re/groups.h | 4 +- src/libre/re_interpolate_groups.c | 33 +++++++++++--- .../re_interpolate_groups0.c | 5 +-- .../re_interpolate_groups1.c | 43 +++++++++++-------- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/include/re/groups.h b/include/re/groups.h index 0c9a77fb3..6d8a0bf65 100644 --- a/include/re/groups.h +++ b/include/re/groups.h @@ -26,6 +26,8 @@ struct re_pos; * to error, otherwise the string value will be used. * Typically this would be passed as "". * + * start,end are only populated on error. + * * You can distinguish compile-time errors (that is, * syntax errors in the format string) vs. runtime errors * (that is, nonexistent groups) by calling @@ -40,7 +42,7 @@ bool re_interpolate_groups(const char *fmt, char esc, const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, char *outs, size_t outn, - struct re_pos *pos); + struct re_pos *start, struct re_pos *end); #endif diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c index 5c9752a41..8241647b3 100644 --- a/src/libre/re_interpolate_groups.c +++ b/src/libre/re_interpolate_groups.c @@ -15,15 +15,15 @@ #include // TODO -#define OUT_CHAR(c) do { if (outn < 1) { goto error; } *outs++ = (c); outn--; } while (0) -#define OUT_GROUP(s) do { if (outn < strlen((s))) { goto error; } outs += sprintf(outs, "%s", (s)); outn -= strlen((s)); } while (0) +#define OUT_CHAR(c) do { if (outn < 1) { goto overflow; } *outs++ = (c); outn--; } while (0) +#define OUT_GROUP(s) do { if (outn < strlen((s))) { goto overflow; } outs += sprintf(outs, "%s", (s)); outn -= strlen((s)); } while (0) // TODO: return values: syntax error, nonexistent group error (digit overflow is the same thing), success bool re_interpolate_groups(const char *fmt, char esc, const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, char *outs, size_t outn, - struct re_pos *pos) + struct re_pos *start, struct re_pos *end) { unsigned group; // 0 meaning group0, 1 meaning groupv[0], etc const char *p; @@ -42,7 +42,12 @@ re_interpolate_groups(const char *fmt, char esc, state = STATE_LIT; group = 0; + if (start != NULL) { + start->byte = 0; + } + p = fmt; + do { switch (state) { case STATE_LIT: @@ -51,6 +56,10 @@ re_interpolate_groups(const char *fmt, char esc, } if (*p == esc) { + if (start != NULL) { + start->byte = p - fmt; + } + state = STATE_ESC; continue; } @@ -114,6 +123,10 @@ re_interpolate_groups(const char *fmt, char esc, } if (*p == esc) { + if (start != NULL) { + start->byte = p - fmt; + } + state = STATE_ESC; continue; } @@ -125,7 +138,7 @@ re_interpolate_groups(const char *fmt, char esc, assert(!"unreached"); goto error; } - } while (*p++); + } while (*p != '\0' && p++); if (state != STATE_LIT) { goto error; @@ -135,11 +148,17 @@ re_interpolate_groups(const char *fmt, char esc, return true; +overflow: + + /* we're blaming the entire fmt string for overflow */ + if (start != NULL) { + start->byte = 0; + } + error: - // TODO: track start,end independently - if (pos != NULL) { - pos->byte = p - fmt; + if (end != NULL) { + end->byte = p - fmt; } return false; diff --git a/tests/re_interpolate_groups/re_interpolate_groups0.c b/tests/re_interpolate_groups/re_interpolate_groups0.c index 28311afb9..4b4e0f709 100644 --- a/tests/re_interpolate_groups/re_interpolate_groups0.c +++ b/tests/re_interpolate_groups/re_interpolate_groups0.c @@ -17,14 +17,13 @@ static unsigned failed; static void test(const char *fmt, size_t groupc, const char *groupv[], const char *expected) { - struct re_pos pos; char outs[40]; bool r; assert(fmt != NULL); assert(expected != NULL); - if (!re_interpolate_groups(fmt, '$', "", groupc, groupv, "", outs, sizeof outs, &pos)) { + if (!re_interpolate_groups(fmt, '$', "", groupc, groupv, "", outs, sizeof outs, NULL, NULL)) { printf("%s/%zu XXX\n", fmt, groupc); failed++; return; @@ -58,6 +57,7 @@ int main(void) { test("x$100000000000000000000x", 1, gn, "xx"); test("$$$1$1$2$1$3$4$3$2$1$$$$", 4, gn, "$oneonetwoonethreefourthreetwoone$$"); + test("$$$$$$$$$$$$$$$$$$$$", 4, gn, "$$$$$$$$$$"); test("xyz_$1..$0003;$3,$$.$1-$4=$123", 4, gn, "xyz_one..three;three,$.one-four="); test("xyz_$1..$0003;$3,$$.$1-$4=$123", 3, gn, "xyz_one..three;three,$.one-="); @@ -66,7 +66,6 @@ int main(void) { test("xyz_$1..$0003;$3,$$.$1-$4=$123", 0, g0, "xyz_..;,$.-="); test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, ga, "xyz_1..;,$.1-="); test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, gb, "xyz_..;,$.-="); -// test("xyz_$1..$0003;$3,$$.$1-$4=$123", 0, gc, "xyz_..;,$.-="); return failed; } diff --git a/tests/re_interpolate_groups/re_interpolate_groups1.c b/tests/re_interpolate_groups/re_interpolate_groups1.c index c75122e1d..7c8fbf54e 100644 --- a/tests/re_interpolate_groups/re_interpolate_groups1.c +++ b/tests/re_interpolate_groups/re_interpolate_groups1.c @@ -15,26 +15,29 @@ static unsigned failed; static void -test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, unsigned expected_pos) +test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, + unsigned expected_start, unsigned expected_end) { - struct re_pos pos; + struct re_pos start, end; char outs[10]; - bool r; + bool rs, re; assert(fmt != NULL); /* for these tests we're expecting to error */ - if (re_interpolate_groups(fmt, '$', "", groupc, groupv, ne, outs, sizeof outs, &pos)) { + if (re_interpolate_groups(fmt, '$', "", groupc, groupv, ne, outs, sizeof outs, &start, &end)) { printf("%s/%zu XXX\n", fmt, groupc); failed++; return; } - failed += r = expected_pos != pos.byte; + failed += rs = expected_start != start.byte; + failed += re = expected_end != end.byte; - printf("%s/%zu => :%u :%u%s\n", fmt, groupc, - pos.byte, expected_pos, - r ? " XXX" : ""); + printf("%s/%zu => :%u-%u :%u-%u%s\n", fmt, groupc, + start.byte, end.byte, + expected_start, expected_end, + (rs || re) ? " XXX" : ""); } int main(void) { @@ -42,21 +45,23 @@ int main(void) { const char *gn[] = { "one", "two", "three", "four" }; const char **g0 = NULL; - const char *ga[] = { "1" }; - const char *gb[] = { "" }; -// const char *gc[] = { NULL }; // XXX: not permitted - test_err("$", 0, g0, ne, 1); - test_err("$x", 0, g0, ne, 1); - test_err("$ ", 4, gn, ne, 1); - test_err("$\\01", 0, g0, ne, 1); + test_err("$", 0, g0, ne, 0, 1); + test_err("$x", 0, g0, ne, 0, 1); + test_err("$ ", 4, gn, ne, 0, 1); + test_err("$\\01", 0, g0, ne, 0, 1); - test_err("$$$x", 4, gn, ne, 3); + test_err("$0$", 0, g0, ne, 2, 3); + test_err("$$$x", 4, gn, ne, 2, 3); - test_err("xyz$1", 0, gn, NULL, 5); - test_err("xyz$2", 1, gn, NULL, 5); + test_err("xyz$1", 0, gn, NULL, 3, 5); + test_err("xyz$2", 1, gn, NULL, 3, 5); - test_err("01234567890", 1, gn, ne, 10); + test_err("01234567890", 1, gn, ne, 0, 10); + test_err("$$$$$$$$$$$$$$$$$$$$", 1, gn, ne, 0, 20); + test_err("$1$1$1$$", 1, gn, ne, 0, 8); + test_err("$1$1$1x", 1, gn, ne, 0, 7); + test_err("xxxyyyzzz$$", 1, gn, ne, 0, 11); return failed; } From 1a6f0078f2f46dbb76914ccbcd74292ca0d132ad Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Jan 2026 01:55:41 +0000 Subject: [PATCH 349/373] Clarification. --- src/libre/re_interpolate_groups.c | 44 +++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c index 8241647b3..e08a57b22 100644 --- a/src/libre/re_interpolate_groups.c +++ b/src/libre/re_interpolate_groups.c @@ -14,11 +14,19 @@ #include #include -// TODO -#define OUT_CHAR(c) do { if (outn < 1) { goto overflow; } *outs++ = (c); outn--; } while (0) -#define OUT_GROUP(s) do { if (outn < strlen((s))) { goto overflow; } outs += sprintf(outs, "%s", (s)); outn -= strlen((s)); } while (0) +#define OUT_CHAR(c) \ + do { \ + if (outn < 1) { goto overflow; } \ + *outs++ = (c); outn--; \ + } while (0) + +#define OUT_GROUP(s) \ + do { \ + if (outn < strlen((s))) { goto overflow; } \ + outs += sprintf(outs, "%s", (s)); \ + outn -= strlen((s)); \ + } while (0) -// TODO: return values: syntax error, nonexistent group error (digit overflow is the same thing), success bool re_interpolate_groups(const char *fmt, char esc, const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, @@ -91,11 +99,18 @@ re_interpolate_groups(const char *fmt, char esc, group *= 10; group += *p - '0'; -// TODO: explain this -// digit overflow, we cap to groupc + 1 -// groupc + 1 is always out of bounds -// this is a simple way to avoid needing to handle digit overflow for subsequent digits, -// assuming groupc *= 10 is <= UINT_MAX + /* + * We need to handle numeric overflow somehow here, + * as we would with using strtol() or similar. But + * we don't need to distinguish this as a special + * error code, semantically it's the same as a group + * that doesn't exist. + * + * groupc + 1 is always out of bounds. So we cap to that, + * using it as a simple way to avoid needing to handle + * numeric overflow for subsequent digits. This assumes + * groupc *= 10 is <= UINT_MAX. + */ if (group > groupc) { group = groupc + 1; } @@ -108,8 +123,15 @@ re_interpolate_groups(const char *fmt, char esc, assert(groupv[group - 1] != NULL); OUT_GROUP(groupv[group - 1]); } else if (nonexistent == NULL) { -// TODO: maybe want to indicate this independently from syntax errors -// TODO: no need, you can pre-check the entire syntax by running with 0 groups + /* + * We could indicate this independently from syntax errors, + * with some way to return different error codes. + * + * But there's no need, you can pre-check the fmt syntax + * by running ahead of time with groupc == 0 and pass + * nonexistent != NULL, because that eliminates the + * possibility for group-related errors. + */ goto error; } else { OUT_GROUP(nonexistent); From cf2bb0afb050d97a83c3c08874d1edf92a8c7b1d Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Jan 2026 02:02:52 +0000 Subject: [PATCH 350/373] Fill out placeholders for writing out output. --- src/libre/re_interpolate_groups.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c index e08a57b22..8e9760eec 100644 --- a/src/libre/re_interpolate_groups.c +++ b/src/libre/re_interpolate_groups.c @@ -14,17 +14,20 @@ #include #include -#define OUT_CHAR(c) \ - do { \ - if (outn < 1) { goto overflow; } \ - *outs++ = (c); outn--; \ +#define OUT_CHAR(c) \ + do { \ + if (outn < 1) goto overflow; \ + *outs++ = (c); \ + outn--; \ } while (0) -#define OUT_GROUP(s) \ - do { \ - if (outn < strlen((s))) { goto overflow; } \ - outs += sprintf(outs, "%s", (s)); \ - outn -= strlen((s)); \ +#define OUT_GROUP(s) \ + do { \ + size_t n = strlen((s)); \ + if (outn < n) goto overflow; \ + (void) memcpy(outs, s, n); \ + outs += n; \ + outn -= n; \ } while (0) bool From 3234a7cc782c550ce8684849b0ecdce04343a795 Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Jan 2026 03:17:25 +0000 Subject: [PATCH 351/373] Convincing myself string offsets are convenient --- tests/re_interpolate_groups/re_interpolate_groups1.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/re_interpolate_groups/re_interpolate_groups1.c b/tests/re_interpolate_groups/re_interpolate_groups1.c index 7c8fbf54e..5cd9a9da5 100644 --- a/tests/re_interpolate_groups/re_interpolate_groups1.c +++ b/tests/re_interpolate_groups/re_interpolate_groups1.c @@ -34,9 +34,10 @@ test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, failed += rs = expected_start != start.byte; failed += re = expected_end != end.byte; - printf("%s/%zu => :%u-%u :%u-%u%s\n", fmt, groupc, + printf("%s/%zu => :%u-%u :%u-%u '%.*s'%s\n", fmt, groupc, start.byte, end.byte, expected_start, expected_end, + (int) (end.byte - start.byte), fmt + start.byte, (rs || re) ? " XXX" : ""); } From 433c8b8a58f88897e269a335237a93f1941b27ad Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Jan 2026 13:53:25 +0000 Subject: [PATCH 352/373] Allow a NULL output string. --- include/re/groups.h | 3 ++ src/libre/re_interpolate_groups.c | 6 +-- .../re_interpolate_groups2.c | 40 +++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 tests/re_interpolate_groups/re_interpolate_groups2.c diff --git a/include/re/groups.h b/include/re/groups.h index 6d8a0bf65..57f01357a 100644 --- a/include/re/groups.h +++ b/include/re/groups.h @@ -37,6 +37,9 @@ struct re_pos; * The output string will always be less than or equal in * length to the format string. The output is \0-terminated. * outn includes the \0. + * + * outs may be NULL in which case outn must be 0, and no + * output is made. */ bool re_interpolate_groups(const char *fmt, char esc, diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c index 8e9760eec..0adef4bc3 100644 --- a/src/libre/re_interpolate_groups.c +++ b/src/libre/re_interpolate_groups.c @@ -15,14 +15,14 @@ #include #define OUT_CHAR(c) \ - do { \ + do if (outs != NULL) { \ if (outn < 1) goto overflow; \ *outs++ = (c); \ outn--; \ } while (0) #define OUT_GROUP(s) \ - do { \ + do if (outs != NULL) { \ size_t n = strlen((s)); \ if (outn < n) goto overflow; \ (void) memcpy(outs, s, n); \ @@ -48,7 +48,7 @@ re_interpolate_groups(const char *fmt, char esc, assert(esc != '\0'); assert(group0 != NULL || groupc == 0); assert(groupc < UINT_MAX / 10 - 1); - assert(outs != NULL); + assert(outs != NULL || outn == 0); state = STATE_LIT; group = 0; diff --git a/tests/re_interpolate_groups/re_interpolate_groups2.c b/tests/re_interpolate_groups/re_interpolate_groups2.c new file mode 100644 index 000000000..d33fe656a --- /dev/null +++ b/tests/re_interpolate_groups/re_interpolate_groups2.c @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +static unsigned failed; + +static void +test(const char *fmt, bool expected) +{ + bool r; + + assert(fmt != NULL); + + r = re_interpolate_groups(fmt, '$', "", 0, NULL, "", NULL, 0, NULL, NULL); + + failed += r != expected; + + printf("%s/%d => %d%s\n", fmt, 0, r, + r != expected ? " XXX" : ""); +} + +int main(void) { + test("", true); + test("abc", true); + test("$$", true); + test("$x", false); + + return failed; +} + From 4ae257ee32cb32695f6605ad481b17b0b22f593e Mon Sep 17 00:00:00 2001 From: Kate F Date: Mon, 26 Jan 2026 23:33:48 +0000 Subject: [PATCH 353/373] Clarification. Spotted by both June and Scott, thanks --- include/re/groups.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/re/groups.h b/include/re/groups.h index 57f01357a..b89a74013 100644 --- a/include/re/groups.h +++ b/include/re/groups.h @@ -35,8 +35,12 @@ struct re_pos; * and passing a non-NULL nonexistent value. * * The output string will always be less than or equal in - * length to the format string. The output is \0-terminated. - * outn includes the \0. + * length to the format string when all interpolated + * values are the empty string. That is, when groupc is 0 + * and nonexistent is the empty string, or when all groups + * used from groupv[] are the empty string. + * + * The output is \0-terminated. outn includes the \0. * * outs may be NULL in which case outn must be 0, and no * output is made. From 0d487a68b1a3ca83acaec1b61241a573e777f1ed Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 00:45:14 +0000 Subject: [PATCH 354/373] Defensively terminate the output buffer on error. Suggested by Scott --- include/re/groups.h | 3 +++ src/libre/re_interpolate_groups.c | 7 +++++++ tests/re_interpolate_groups/re_interpolate_groups1.c | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/include/re/groups.h b/include/re/groups.h index b89a74013..cddccf6fc 100644 --- a/include/re/groups.h +++ b/include/re/groups.h @@ -44,6 +44,9 @@ struct re_pos; * * outs may be NULL in which case outn must be 0, and no * output is made. + * + * On error the function returns false and the output + * buffer is indeterminate. */ bool re_interpolate_groups(const char *fmt, char esc, diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c index 0adef4bc3..52b61f116 100644 --- a/src/libre/re_interpolate_groups.c +++ b/src/libre/re_interpolate_groups.c @@ -37,6 +37,7 @@ re_interpolate_groups(const char *fmt, char esc, struct re_pos *start, struct re_pos *end) { unsigned group; // 0 meaning group0, 1 meaning groupv[0], etc + char *outs_orig; const char *p; enum { @@ -53,6 +54,8 @@ re_interpolate_groups(const char *fmt, char esc, state = STATE_LIT; group = 0; + outs_orig = outn > 0 ? outs : NULL; + if (start != NULL) { start->byte = 0; } @@ -186,6 +189,10 @@ re_interpolate_groups(const char *fmt, char esc, end->byte = p - fmt; } + if (outs_orig != NULL) { + *outs_orig = '\0'; + } + return false; } diff --git a/tests/re_interpolate_groups/re_interpolate_groups1.c b/tests/re_interpolate_groups/re_interpolate_groups1.c index 5cd9a9da5..01dc3b344 100644 --- a/tests/re_interpolate_groups/re_interpolate_groups1.c +++ b/tests/re_interpolate_groups/re_interpolate_groups1.c @@ -24,6 +24,8 @@ test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, assert(fmt != NULL); + outs[0] = 'x'; + /* for these tests we're expecting to error */ if (re_interpolate_groups(fmt, '$', "", groupc, groupv, ne, outs, sizeof outs, &start, &end)) { printf("%s/%zu XXX\n", fmt, groupc); @@ -31,6 +33,10 @@ test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, return; } + if (outs[0] != '\0') { + failed++; + } + failed += rs = expected_start != start.byte; failed += re = expected_end != end.byte; From 82473208cb7bba31e4e6e2a2b71e065cb65665f8 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 12:40:47 +0000 Subject: [PATCH 355/373] Update to actions/cache@v5 --- .github/workflows/ci.yml | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 157506a14..6b1c09e7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Cache checkout - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -52,7 +52,7 @@ jobs: steps: - name: Cache PCRE suite - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-pcre with: path: pcre-suite/${{ env.pcre2 }} @@ -70,7 +70,7 @@ jobs: chmod -R ug-w pcre-suite - name: Cache converted PCRE tests - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-cvtpcre with: path: ${{ env.cvtpcre }} @@ -78,7 +78,7 @@ jobs: - name: Fetch build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} @@ -158,14 +158,14 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} - name: Cache build - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} @@ -236,7 +236,7 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -244,7 +244,7 @@ jobs: # An arbitary build. - name: Fetch build - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} @@ -325,7 +325,7 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -347,7 +347,7 @@ jobs: ${{ matrix.cc }} --version - name: Fetch build - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} @@ -384,7 +384,7 @@ jobs: steps: - name: Fetch checkout - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -406,7 +406,7 @@ jobs: ${{ matrix.cc }} --version - name: Fetch build - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} @@ -516,14 +516,14 @@ jobs: go version - name: Fetch build - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Fetch converted PCRE tests - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-cvtpcre with: path: ${{ env.cvtpcre }} @@ -542,7 +542,7 @@ jobs: steps: - name: Cache docs - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-docs with: path: ${{ env.build }} @@ -557,7 +557,7 @@ jobs: - name: Fetch checkout if: steps.cache-docs.outputs.cache-hit != 'true' - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -597,7 +597,7 @@ jobs: steps: - name: Cache prefix - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-prefix with: path: ${{ env.prefix }} @@ -611,7 +611,7 @@ jobs: - name: Fetch checkout if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -619,7 +619,7 @@ jobs: - name: Fetch build if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} @@ -627,7 +627,7 @@ jobs: - name: Fetch docs if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-docs with: path: ${{ env.build }} @@ -671,7 +671,7 @@ jobs: fpm -v - name: Fetch prefix - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-prefix with: path: ${{ env.prefix }} From 6d1bd9bb14556686459a735a69247f91f8e4d68a Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 12:37:46 +0000 Subject: [PATCH 356/373] fail-on-cache-miss: for grabbing arbitrary builds. --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b1c09e7d..5898404f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,7 @@ jobs: with: path: ${{ env.build }} key: build-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitrary build, just for cvtpcre + fail-on-cache-miss: true - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -163,6 +164,7 @@ jobs: with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Cache build uses: actions/cache@v5 @@ -241,6 +243,7 @@ jobs: with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true # An arbitary build. - name: Fetch build @@ -249,6 +252,7 @@ jobs: with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: true # We don't need to build the entire repo to know that the makefiles work, # I'm just deleting a couple of .o files and rebuilding those instead. @@ -330,6 +334,7 @@ jobs: with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Dependencies (Ubuntu) if: matrix.os == 'ubuntu-22.04' @@ -389,6 +394,7 @@ jobs: with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Dependencies (Ubuntu) if: matrix.os == 'ubuntu-22.04' @@ -562,6 +568,7 @@ jobs: with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Get number of CPU cores if: steps.cache-docs.outputs.cache-hit != 'true' @@ -616,6 +623,7 @@ jobs: with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Fetch build if: steps.cache-prefix.outputs.cache-hit != 'true' From c1203e3a1c27fd4e176c4798e209e3d61753f8c0 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 13:26:48 +0000 Subject: [PATCH 357/373] Explicitly allow build cache miss for makefile tests. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5898404f6..8bfcf920f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,13 +246,17 @@ jobs: fail-on-cache-miss: true # An arbitary build. + # Failing to fetch this is not fatal, we're testing Makefiles here. + # Some combinations of our options (pmake, EXPENSIVE_CHECKS, whatever) + # won't exist in cache because we didn't build those. That's okay for + # the purposes of this step, building those is harmless. - name: Fetch build uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - fail-on-cache-miss: true + fail-on-cache-miss: false # We don't need to build the entire repo to know that the makefiles work, # I'm just deleting a couple of .o files and rebuilding those instead. From 3f69a7c0ef4e8219286a91274268989036615579 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 13:44:15 +0000 Subject: [PATCH 358/373] Explicitly fail-on-cache-miss for other things too. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bfcf920f..840bbb47d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,6 +361,7 @@ jobs: with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v2 @@ -421,6 +422,7 @@ jobs: with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: true # note we do the fuzzing unconditionally; each run adds to the corpus. # @@ -531,6 +533,7 @@ jobs: with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Fetch converted PCRE tests uses: actions/cache@v5 @@ -636,6 +639,7 @@ jobs: with: path: ${{ env.build }} key: build-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Fetch docs if: steps.cache-prefix.outputs.cache-hit != 'true' @@ -644,6 +648,7 @@ jobs: with: path: ${{ env.build }} key: docs-${{ github.sha }} + fail-on-cache-miss: true - name: Get number of CPU cores if: steps.cache-prefix.outputs.cache-hit != 'true' @@ -688,6 +693,7 @@ jobs: with: path: ${{ env.prefix }} key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Find version # TODO: would get a tag or branch name here From 661105e9e6acdbb9b92c962f837e64e81a892918 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 13:10:45 +0000 Subject: [PATCH 359/373] cache/restore where possible. --- .github/workflows/ci.yml | 56 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 840bbb47d..f86f04535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,9 +76,9 @@ jobs: path: ${{ env.cvtpcre }} key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - - name: Fetch build + - name: Restore build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' - uses: actions/cache@v5 + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} @@ -158,8 +158,8 @@ jobs: cc: gcc # -fsanitize=fuzzer is clang-only steps: - - name: Fetch checkout - uses: actions/cache@v5 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -237,8 +237,8 @@ jobs: make: pmake # not packaged steps: - - name: Fetch checkout - uses: actions/cache@v5 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -250,8 +250,8 @@ jobs: # Some combinations of our options (pmake, EXPENSIVE_CHECKS, whatever) # won't exist in cache because we didn't build those. That's okay for # the purposes of this step, building those is harmless. - - name: Fetch build - uses: actions/cache@v5 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} @@ -332,8 +332,8 @@ jobs: san: MSAN # not supported steps: - - name: Fetch checkout - uses: actions/cache@v5 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -355,8 +355,8 @@ jobs: brew install bmake pcre ${{ matrix.cc }} --version - - name: Fetch build - uses: actions/cache@v5 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} @@ -393,8 +393,8 @@ jobs: cc: gcc # it's clang anyway steps: - - name: Fetch checkout - uses: actions/cache@v5 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -416,8 +416,8 @@ jobs: brew install bmake ${{ matrix.cc }} --version - - name: Fetch build - uses: actions/cache@v5 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} @@ -527,8 +527,8 @@ jobs: sudo apt-get install golang go version - - name: Fetch build - uses: actions/cache@v5 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} @@ -568,9 +568,9 @@ jobs: sudo apt-get update sudo apt-get install bmake libxml2-utils xsltproc docbook-xml docbook-xsl - - name: Fetch checkout + - name: Restore checkout if: steps.cache-docs.outputs.cache-hit != 'true' - uses: actions/cache@v5 + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -623,27 +623,27 @@ jobs: uname -a sudo apt-get install bmake - - name: Fetch checkout + - name: Restore checkout if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v5 + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} fail-on-cache-miss: true - - name: Fetch build + - name: Restore build if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v5 + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} fail-on-cache-miss: true - - name: Fetch docs + - name: Restore docs if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v5 + uses: actions/cache/restore@v5 id: cache-docs with: path: ${{ env.build }} @@ -687,8 +687,8 @@ jobs: sudo gem install --no-document fpm fpm -v - - name: Fetch prefix - uses: actions/cache@v5 + - name: Restore prefix + uses: actions/cache/restore@v5 id: cache-prefix with: path: ${{ env.prefix }} From c798ec1b2796ae69fe3a9d80361c14284116cb5c Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 27 Jan 2026 16:19:26 +0000 Subject: [PATCH 360/373] Merge mishap, accidentally @v4 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f86f04535..347d5df38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -433,7 +433,7 @@ jobs: # still run fuzzing, just from empty, and do not save their seeds. - name: Restore seeds (mode ${{ matrix.mode }}) if: github.repository == 'katef/libfsm' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@v5 id: cache-seeds with: path: ${{ env.seeds }}-${{ matrix.mode }} @@ -470,7 +470,7 @@ jobs: # the same seeds for a given bug. # The explicit cache/restore and cache/save actions are just for that. - name: Save seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) - uses: actions/cache/save@v4 + uses: actions/cache/save@v5 if: always() with: path: ${{ env.seeds }}-${{ matrix.mode }} From a6e9c2aec2bd2c5ef9ca7331742037b16ed01893 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 9 Feb 2026 12:57:55 -0500 Subject: [PATCH 361/373] Makefile: grep for 'FAIL' should use -I to ignore binary files. I ran into a test failure: grep FAIL build/tests/*/*res*; [ $? -ne 0 ] grep: build/tests/eager_output/run_mixed_start_anchor_regression: binary file matches *** Error code 1 because tests/eager_output/eager_output_mixed_start_anchor_regression.c contains the substring "res" in "regression", and the binary file contains "FAIL: $ strings build/tests/eager_output/run_mixed_start_anchor_regression | grep FAIL VM_END_FAIL VM_FAIL This shouldn't matter for testing purposes, the 'test' target only cares about the test result files containing "PASS" or "FAIL". The change in cb42d58 to allow prefixed result files (e.g. "dyn-fdgetc-getc-res0") made this check ANY files with "res" in the name, but grep should ignore binary files. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4065c28b0..683e1d906 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,6 @@ STAGE_BUILD := ${STAGE_BUILD:Nbin/cvtpcre} .if make(test) .END:: - grep FAIL ${BUILD}/tests/*/*res*; [ $$? -ne 0 ] + grep -I FAIL ${BUILD}/tests/*/*res*; [ $$? -ne 0 ] .endif From 7ea54217e889f00e488c554afa6867380d3002a8 Mon Sep 17 00:00:00 2001 From: Kate F Date: Tue, 10 Feb 2026 03:10:40 +0000 Subject: [PATCH 362/373] Oops... missed installing --- include/re/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/include/re/Makefile b/include/re/Makefile index 9c216528d..9e649a526 100644 --- a/include/re/Makefile +++ b/include/re/Makefile @@ -3,4 +3,5 @@ STAGE_COPY += include/re/re.h STAGE_COPY += include/re/strings.h STAGE_COPY += include/re/literal.h +STAGE_COPY += include/re/groups.h From d376df61206821f51f9dd5704beb567d292b4b89 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jun 2026 11:09:28 -0400 Subject: [PATCH 363/373] Regen lx parser: call re_comp with `| RE_NOCAPTURE`. lx has its own way of handling how matches are associated with tokens, so currently it shouldn't use captures. --- src/lx/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lx/parser.c b/src/lx/parser.c index 4ab69ce21..5af0592d5 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -2017,7 +2017,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->alloc, (ZIf), &err); + (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->alloc, (ZIf) | RE_NOCAPTURE, &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ From c69578f3c043499889dc5482a73fe8a4b26fa9db Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jun 2026 11:10:20 -0400 Subject: [PATCH 364/373] captest.c: Instead of using `captest_getc` just use `fsm_sgetc`. There's no need to have a custom replacement here. It also had a bug, because it advanced the input position on EOF -- that meant that if re_comp internally requested more characters after reaching EOF, it would read past the end of the input string, causing tests to fail with bizarre systax errors. This previously wasn't an issue, but something in the merge changed a caller's lookahead/pushback behavior so that it needs to yield EOF forever once it reaches EOF. --- tests/capture/captest.c | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/tests/capture/captest.c b/tests/capture/captest.c index 810d2cabb..cfd0d8b05 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -8,20 +8,6 @@ /* for fsm_capvm_program_exec */ #include "../../src/libfsm/capture_vm.h" -struct captest_input { - const char *string; - size_t pos; -}; - -static int -captest_getc(void *opaque) -{ - struct captest_input *input = opaque; - int res = input->string[input->pos]; - input->pos++; - return res == 0 ? EOF : res; -} - static struct fsm_options options = { .group_edges = 1, }; @@ -53,12 +39,11 @@ captest_run_case(const struct captest_case_single *testcase, /* build regex */ const enum re_flags flags = 0; - struct captest_input comp_input = { - .string = testcase->regex, - }; + + const char *regex = testcase->regex; struct fsm *fsm = re_comp(RE_PCRE, - captest_getc, &comp_input, + fsm_sgetc, ®ex, NULL, flags, &err); if (testcase->match == SHOULD_REJECT_AS_UNSUPPORTED) { @@ -243,17 +228,15 @@ captest_run_case_multi(const struct captest_case_multi *testcase, /* compile each individually */ for (size_t i = 0; i < testcase->regex_count; i++) { - struct captest_input comp_input = { - .string = testcase->regexes[i], - }; + const char *regex = testcase->regexes[i]; if (verbosity > 1) { fprintf(stderr, "%s: compiling \"%s\"\n", - __func__, comp_input.string); + __func__, regex); } struct fsm *fsm = re_comp(RE_PCRE, - captest_getc, &comp_input, + fsm_sgetc, ®ex, NULL, flags, &err); assert(fsm != NULL); From 335cf589caa25bd2fc41149941f7420243428f25 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jun 2026 15:00:42 -0400 Subject: [PATCH 365/373] Consolidate AST_ANALYSIS_ERROR_UNSUPPORTED_* cases and RE_EUNSUP*. After discussing this with katef, we don't need distinct cases for when something is unsupported because of capture behavior or because of particular PCRE edge cases, the general unsupported error covers it. This fixes test failures for native dialect tests, beacuse the native dialect (understandably) doesn't handle the PCRE-specific error codes. --- include/re/re.h | 2 -- src/libre/ast_analysis.c | 22 ++++++++++------------ src/libre/ast_analysis.h | 3 +-- src/libre/re.c | 9 +++------ src/libre/strerror.c | 2 -- 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/include/re/re.h b/include/re/re.h index 3ec95bb62..aa08e3522 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -51,8 +51,6 @@ enum re_errno { RE_EERRNO = 1 | RE_MISC, RE_EBADDIALECT = 2 | RE_MISC, RE_EBADGROUP = 3 | RE_MISC, - RE_EUNSUPCAPTUR = 4 | RE_MISC, - RE_EUNSUPPPCRE = 5 | RE_MISC, RE_ENEGRANGE = 0 | RE_MARK | RE_GROUP, RE_ENEGCOUNT = 1 | RE_MARK | RE_GROUP, diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 66e7c3e86..cf29802df 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -426,7 +426,7 @@ analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_p * * An example input that triggers this is '^(($)|)+$' . */ set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } } @@ -537,7 +537,7 @@ analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_p && repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); LOG(3 - LOG_REPETITION_CASES, "%s: not yet implemented, skipping\n", __func__); - /* return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; */ + /* return AST_ANALYSIS_ERROR_UNSUPPORTED; */ } res = analysis_iter_repetition(n->u.group.e, outermost_repeat_parent, @@ -1044,8 +1044,8 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: LITERAL: rejecting non-optional newline match after $ as unsupported\n", __func__); set_flags(n, AST_FLAG_UNSATISFIABLE); - /* fprintf(stderr, "%s:%u: AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE\n", __func__, __LINE__); */ - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + /* fprintf(stderr, "%s:%u: AST_ANALYSIS_ERROR_UNSUPPORTED\n", __func__, __LINE__); */ + return AST_ANALYSIS_ERROR_UNSUPPORTED; } } break; @@ -1084,7 +1084,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (res != AST_ANALYSIS_OK && res != AST_ANALYSIS_UNSATISFIABLE) { /* unsat is handled below */ - if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { /* FIXME: check this */ assert(child->flags & AST_FLAG_UNSATISFIABLE); @@ -1261,8 +1261,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) all_set_past_always_consuming &= child_env.past_always_consuming; any_sat = 1; } - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE - || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { assert(child->flags & AST_FLAG_UNSATISFIABLE); continue; } else { @@ -1467,7 +1466,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: SUBTRACT: rejecting non-optional newline match after $ as unsupported\n", __func__); set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } if (res != AST_ANALYSIS_OK) { @@ -1563,7 +1562,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) LOG(3 - LOG_ANCHORING, "%s: END anchor & followed_by_consuming, returning UNSUPPORTED\n", __func__); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } else { LOG(3 - LOG_ANCHORING, "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", @@ -1730,8 +1729,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) any_sat = 1; } - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE - || res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED, bubbling up\n", __func__); assert(child->flags & AST_FLAG_UNSATISFIABLE); @@ -2054,7 +2052,7 @@ analysis_iter_captures(struct capture_env *env, struct ast_expr *n) case AST_EXPR_ANCHOR: if (env->use_captures && n->u.anchor.type == AST_ANCHOR_END && !n->u.anchor.is_end_nl) { set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE; + return AST_ANALYSIS_ERROR_UNSUPPORTED; } break; diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index 7a7fdc846..af3c5bf97 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -29,8 +29,7 @@ enum ast_analysis_res { AST_ANALYSIS_ERROR_NULL = -1, AST_ANALYSIS_ERROR_MEMORY = -2, - AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE = -3, - AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE = -4 + AST_ANALYSIS_ERROR_UNSUPPORTED = -3, }; enum ast_analysis_res diff --git a/src/libre/re.c b/src/libre/re.c index f744a2422..f6c1d0c02 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -123,10 +123,8 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, res = ast_analysis(ast, flags); if (res < 0) { if (err != NULL) { - if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE) { - err->e = RE_EUNSUPPPCRE; - } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE) { - err->e = RE_EUNSUPCAPTUR; + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + err->e = RE_EUNSUPPORTED; } else if (err->e == RE_ESUCCESS) { err->e = RE_EERRNO; } @@ -157,8 +155,7 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, } break; - case AST_ANALYSIS_ERROR_UNSUPPORTED_PCRE: - case AST_ANALYSIS_ERROR_UNSUPPORTED_CAPTURE: + case AST_ANALYSIS_ERROR_UNSUPPORTED: err->e = RE_EUNSUPPORTED; /* diff --git a/src/libre/strerror.c b/src/libre/strerror.c index d66e750a4..009d61df2 100644 --- a/src/libre/strerror.c +++ b/src/libre/strerror.c @@ -20,8 +20,6 @@ re_strerror(enum re_errno e) case RE_EERRNO: return strerror(errno); case RE_EBADDIALECT: return "Bad dialect"; case RE_EBADGROUP: return "Bad group"; - case RE_EUNSUPCAPTUR: return "Cannot support captures in this case"; - case RE_EUNSUPPPCRE: return "Unsupported PCRE edge case"; case RE_ENEGRANGE: return "Negative group range"; case RE_ENEGCOUNT: return "Negative count range"; From 39bb9368c12cffd59646588ba60fa91d411b6860 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jun 2026 15:02:22 -0400 Subject: [PATCH 366/373] Update native parser.sid: '$' is `ast-make-anchor-end-nl`. This fixes an incorrect `AST_ANALYSIS_ERROR_UNSUPPORTED` case in the native dialect, where '$' was being treated like `\z` in PCRE, then getting an unsupported error related to captures. This fixes the test failure in tests/endids/endids10_minimise_partial_overlap.c. --- src/libre/dialect/native/parser.sid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libre/dialect/native/parser.sid b/src/libre/dialect/native/parser.sid index aeb894c64..1ac7e5d64 100644 --- a/src/libre/dialect/native/parser.sid +++ b/src/libre/dialect/native/parser.sid @@ -106,8 +106,8 @@ !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); : () -> (:ast_expr); - : () -> (:ast_expr); - !:() -> (:ast_expr); + !: () -> (:ast_expr); + :() -> (:ast_expr); !: (:ast_expr, :ast_expr) -> (:ast_expr); : (:ast_expr) -> (:ast_expr); : (:endpoint, :pos, :endpoint, :pos) -> (:ast_expr); @@ -359,7 +359,7 @@ e = ; || END; - e = ; + e = ; || ANY; a = ; From 5766444cfb6fc408044aa4dd466826bef2235e61 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Fri, 5 Jun 2026 15:04:14 -0400 Subject: [PATCH 367/373] Regenerate the parser for the native dialect. --- src/libre/dialect/native/parser.c | 41 +++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 63c5f8cb8..a4dfb96d9 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -2665,18 +2665,21 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e case (TOK_END): { ADVANCE_LEXER; - /* BEGINNING OF ACTION: ast-make-anchor-end */ + /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 942 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } + if (!(*flags & RE_ANCHORED)) { + (ZIe)->u.anchor.is_end_nl = 1; + } -#line 2678 "src/libre/dialect/native/parser.c" +#line 2681 "src/libre/dialect/native/parser.c" } - /* END OF ACTION: ast-make-anchor-end */ + /* END OF ACTION: ast-make-anchor-end-nl */ } break; case (TOK_OPENSUB): @@ -2691,7 +2694,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIid) = act_state->group_id++; -#line 2695 "src/libre/dialect/native/parser.c" +#line 2698 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2708,7 +2711,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2712 "src/libre/dialect/native/parser.c" +#line 2715 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2732,7 +2735,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 2736 "src/libre/dialect/native/parser.c" +#line 2739 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2772,7 +2775,7 @@ ZL1:; } goto ZL2; -#line 2776 "src/libre/dialect/native/parser.c" +#line 2779 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ @@ -2784,7 +2787,7 @@ ZL1:; goto ZL2; } -#line 2788 "src/libre/dialect/native/parser.c" +#line 2791 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2813,7 +2816,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2817 "src/libre/dialect/native/parser.c" +#line 2820 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2831,7 +2834,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla (ZIlower).type = AST_ENDPOINT_NAMED; (ZIlower).u.named.class = (*ZI243); -#line 2835 "src/libre/dialect/native/parser.c" +#line 2838 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ p_154 (flags, lex_state, act_state, err); @@ -2847,7 +2850,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); -#line 2851 "src/libre/dialect/native/parser.c" +#line 2854 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ @@ -2881,7 +2884,7 @@ p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } -#line 2885 "src/libre/dialect/native/parser.c" +#line 2888 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2915,7 +2918,7 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ goto ZL1; } -#line 2919 "src/libre/dialect/native/parser.c" +#line 2922 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2949,7 +2952,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 2953 "src/libre/dialect/native/parser.c" +#line 2956 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2967,7 +2970,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI (ZIlower).type = AST_ENDPOINT_LITERAL; (ZIlower).u.literal.c = (unsigned char) (*ZI247); -#line 2971 "src/libre/dialect/native/parser.c" +#line 2974 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ p_154 (flags, lex_state, act_state, err); @@ -2983,7 +2986,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); -#line 2987 "src/libre/dialect/native/parser.c" +#line 2990 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ @@ -3017,7 +3020,7 @@ p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } -#line 3021 "src/libre/dialect/native/parser.c" +#line 3024 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3191,6 +3194,6 @@ ZL0:; return NULL; } -#line 3195 "src/libre/dialect/native/parser.c" +#line 3198 "src/libre/dialect/native/parser.c" /* END OF FILE */ From e34627e53a686328e2f477bdc88282e8ccbd7cc4 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 8 Jun 2026 16:29:14 -0400 Subject: [PATCH 368/373] ast_analysis: Fix ALT reverse anchoring case, likely merge error. This looks like a merge error. The proper behavior here is to skip unsupported subtrees and only fail as unsatisfiable if none are, but it was returning immediately. This broke the `a|_$[^b]` test case in `tests/capture/capture_test_case_list.c`. --- src/libre/ast_analysis.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index cf29802df..399ad147d 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -1732,8 +1732,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED, bubbling up\n", __func__); assert(child->flags & AST_FLAG_UNSATISFIABLE); - - return res; + continue; } else { return res; } From 1855f7bafb4a0c1359cdf2b5a5f9eb0242d20284 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Mon, 8 Jun 2026 17:11:46 -0400 Subject: [PATCH 369/373] Fix UB warning for signed int promotion with the negated flags. --- src/libre/ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libre/ast.c b/src/libre/ast.c index c9216172f..4f86d8cde 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -996,7 +996,7 @@ ast_expr_is_literal(const struct ast_expr *e, /* mask out AST flags that we don't care about; these may be present or not */ int ast_flags = nodes[i]->flags - & ~(AST_FLAG_FIRST | AST_FLAG_LAST | AST_FLAG_ALWAYS_CONSUMES | AST_FLAG_CAN_CONSUME); + & (unsigned)~(AST_FLAG_FIRST | AST_FLAG_LAST | AST_FLAG_ALWAYS_CONSUMES | AST_FLAG_CAN_CONSUME); /* we reject anything else */ if (ast_flags != 0) { @@ -1005,7 +1005,7 @@ ast_expr_is_literal(const struct ast_expr *e, /* mask out re flags that we don't handle here yet (the caller is responsible for these) */ int re_flags = nodes[i]->re_flags - & ~(RE_END_NL | RE_NOCAPTURE); + & (unsigned)~(RE_END_NL | RE_NOCAPTURE); /* and we don't permit any other re_flags */ if (re_flags != 0) { From 54780537537b9a86e02dc65b09a68400e0260422 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Tue, 9 Jun 2026 12:08:23 -0400 Subject: [PATCH 370/373] Update test case: We can handle this correctly now. Instead of rejecting this as unsupported, it should just prune the unsatisfiable side of the `|` and produce the same FSM as `^a`. --- tests/pcre/out48.err | 1 - tests/pcre/out48.fsm | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) delete mode 100644 tests/pcre/out48.err create mode 100644 tests/pcre/out48.fsm diff --git a/tests/pcre/out48.err b/tests/pcre/out48.err deleted file mode 100644 index d96141a09..000000000 --- a/tests/pcre/out48.err +++ /dev/null @@ -1 +0,0 @@ -tests/pcre/in48.re:1-11: Unsupported operator diff --git a/tests/pcre/out48.fsm b/tests/pcre/out48.fsm new file mode 100644 index 000000000..80ce4807e --- /dev/null +++ b/tests/pcre/out48.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; # e.g. "a" +1 -> 1 ?; + +start: 0; +end: 1 = [0]; From a99667721cdb4f2c32e3e1002f893e5eca341dd8 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 17 Jun 2026 13:14:36 -0400 Subject: [PATCH 371/373] ast_make_expr_alt: Ensure field is init'd to 0. --- src/libre/ast.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libre/ast.c b/src/libre/ast.c index 4f86d8cde..2a630a878 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -638,6 +638,7 @@ ast_make_expr_alt(struct ast_expr_pool **poolp, enum re_flags re_flags) res->re_flags = re_flags; res->u.alt.alloc = 8; /* arbitrary initial value */ res->u.alt.count = 0; + res->u.alt.nullable_alt_inside_plus_repeat = 0; res->u.alt.n = calloc(res->u.alt.alloc, sizeof *res->u.alt.n); if (res->u.alt.n == NULL) { From 74992916c60c63188bf0ae62ad7e30aea2cd5c6d Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 17 Jun 2026 13:17:53 -0400 Subject: [PATCH 372/373] re_capvm_compile.c: Comment changes: Rather than using numbered label names ("j2", etc.) use something more descriptive. Use "{x,x}" for repetition comment, to make bounds explicit. --- src/libre/re_capvm_compile.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index 0b46e95f2..b59d52e84 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -653,18 +653,18 @@ capvm_compile_iter(struct capvm_compile_env *env, * leading to generated code like: * * // note: trying each case in order, earlier cases are more greedy - * - split_cont j1 - * - split_new j2 - * j1: + * - split_greedy j_try_a + * - split_nongreedy j_skip_a + * j_try_a: * - * - jmp pos_after_all // or split pos_after_all, PLUS_BACKPATCH, see below - * j2: - * - split_cont j3 - * - split_new j4 - * j3: + * j_skip_a: + * - split_greedy j_try_b + * - split_nongreedy j_else_c + * j_try_b: * - * - jmp pos_after_all - * j4: + * j_else_c: * //// DO NOT EMIT split instructions here, treat like a final else * - * // fall through to pos_after_all @@ -788,6 +788,7 @@ capvm_compile_iter(struct capvm_compile_env *env, } const uint32_t pos_after_all = get_program_offset(p); + // backpatch the jmps to pos_after_all for (size_t i = 0; i < expr->u.alt.count - 1; i++) { const bool is_final_else_case = i == last_active; assert(flow_info[i].backpatch < p->used); @@ -938,7 +939,7 @@ capvm_compile_iter(struct capvm_compile_env *env, if (max == AST_COUNT_UNBOUNDED) { /* A repeat of {x,inf} should be treated like - * (?:subtree){x} (?:subtree)* , where any numbered + * (?:subtree){x,x} (?:subtree)* , where any numbered * capture groups inside have the same group ID in * both copies of the subtree. */ if (!compile_kleene_star(env, p, expr)) { From ee0b2f921c720b29450eaf29e8698a2593daab17 Mon Sep 17 00:00:00 2001 From: Scott Vokes Date: Wed, 17 Jun 2026 13:18:47 -0400 Subject: [PATCH 373/373] Bugfix: Only check flag on union's alt if it's an AST_EXPR_ALT. --- src/libre/re_capvm_compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c index b59d52e84..be9e92c95 100644 --- a/src/libre/re_capvm_compile.c +++ b/src/libre/re_capvm_compile.c @@ -757,7 +757,7 @@ capvm_compile_iter(struct capvm_compile_env *env, } /* Based on analysis, either emit a JMP or SPLIT. */ - if (n->u.alt.nullable_alt_inside_plus_repeat) { + if (n->type == AST_EXPR_ALT && n->u.alt.nullable_alt_inside_plus_repeat) { const uint32_t pos_split_after = reserve_program_opcode(p); flow_info[c_i].backpatch = pos_split_after; struct capvm_opcode *op_split_after = &p->ops[pos_split_after];